Esempio n. 1
0
    def __init__(self, path, device, partition, account, container, obj,
                 logger, disk_chunk_size=65536,
                 bytes_per_sync=(512 * 1024 * 1024),
                 iter_hook=None, threadpool=None, obj_dir='objects',
                 mount_check=False):
        if mount_check and not check_mount(path, device):
            raise DiskFileDeviceUnavailable()
        self.disk_chunk_size = disk_chunk_size
        self.bytes_per_sync = bytes_per_sync
        self.iter_hook = iter_hook
        self.name = '/' + '/'.join((account, container, obj))
        name_hash = hash_path(account, container, obj)
        self.datadir = join(
            path, device, storage_directory(obj_dir, partition, name_hash))
        self.device_path = join(path, device)
        self.tmpdir = join(path, device, 'tmp')
        self.logger = logger
        self._metadata = None
        self.data_file = None
        self._data_file_size = None
        self.fp = None
        self.iter_etag = None
        self.started_at_0 = False
        self.read_to_eof = False
        self.quarantined_dir = None
        self.suppress_file_closing = False
        self._verify_close = False
        self.threadpool = threadpool or ThreadPool(nthreads=0)

        # FIXME(clayg): this attribute is set after open and affects the
        # behavior of the class (i.e. public interface)
        self.keep_cache = False
Esempio n. 2
0
    def setup_bad_zero_byte(self, with_ts=False):
        self.auditor = auditor.ObjectAuditor(self.conf)
        self.auditor.log_time = 0
        ts_file_path = ''
        if with_ts:

            name_hash = hash_path('a', 'c', 'o')
            dir_path = os.path.join(
                self.devices, 'sda',
                storage_directory(DATADIR, '0', name_hash))
            ts_file_path = os.path.join(dir_path, '99999.ts')
            if not os.path.exists(dir_path):
                mkdirs(dir_path)
            fp = open(ts_file_path, 'w')
            fp.close()

        etag = md5()
        with self.disk_file.create() as writer:
            etag = etag.hexdigest()
            metadata = {
                'ETag': etag,
                'X-Timestamp': str(normalize_timestamp(time.time())),
                'Content-Length': 10,
            }
            writer.put(metadata)
            etag = md5()
            etag = etag.hexdigest()
            metadata['ETag'] = etag
            write_metadata(writer.fd, metadata)
        if self.disk_file.data_file:
            return self.disk_file.data_file
        return ts_file_path
Esempio n. 3
0
 def _get_account_broker(self, drive, part, account, **kwargs):
     hsh = hash_path(account)
     db_dir = storage_directory(DATADIR, part, hsh)
     db_path = os.path.join(self.root, drive, db_dir, hsh + '.db')
     kwargs.setdefault('account', account)
     kwargs.setdefault('logger', self.logger)
     return AccountBroker(db_path, **kwargs)
Esempio n. 4
0
 def _get_account_broker(self, drive, part, account, **kwargs):
     hsh = hash_path(account)
     db_dir = storage_directory(DATADIR, part, hsh)
     db_path = os.path.join(self.root, drive, db_dir, hsh + '.db')
     kwargs.setdefault('account', account)
     kwargs.setdefault('logger', self.logger)
     return AccountBroker(db_path, **kwargs)
Esempio n. 5
0
    def _get_container_broker(self, drive, part, account, container, **kwargs):
        """
        Get a DB broker for the container.

        :param drive: drive that holds the container
        :param part: partition the container is in
        :param account: account name
        :param container: container name
        :returns: ContainerBroker object
        """
        hsh = hash_path(account, container)
        db_dir = storage_directory(DATADIR, part, hsh)
        db_path = os.path.join(self.root, drive, db_dir, hsh + '.db')
        kwargs.setdefault('account', account)
        kwargs.setdefault('container', container)
        kwargs.setdefault('logger', self.logger)
        return ContainerBroker(db_path, **kwargs)
Esempio n. 6
0
    def _get_container_broker(self, drive, part, account, container, **kwargs):
        """
        Get a DB broker for the container.

        :param drive: drive that holds the container
        :param part: partition the container is in
        :param account: account name
        :param container: container name
        :returns: ContainerBroker object
        """
        hsh = hash_path(account, container)
        db_dir = storage_directory(DATADIR, part, hsh)
        db_path = os.path.join(self.root, drive, db_dir, hsh + '.db')
        kwargs.setdefault('account', account)
        kwargs.setdefault('container', container)
        kwargs.setdefault('logger', self.logger)
        return ContainerBroker(db_path, **kwargs)
Esempio n. 7
0
    def __init__(self,
                 path,
                 device,
                 partition,
                 account,
                 container,
                 obj,
                 logger,
                 disk_chunk_size=65536,
                 bytes_per_sync=(512 * 1024 * 1024),
                 iter_hook=None,
                 threadpool=None,
                 obj_dir='objects',
                 mount_check=False):
        if mount_check and not check_mount(path, device):
            raise DiskFileDeviceUnavailable()
        self.disk_chunk_size = disk_chunk_size
        self.bytes_per_sync = bytes_per_sync
        self.iter_hook = iter_hook
        self.name = '/' + '/'.join((account, container, obj))
        name_hash = hash_path(account, container, obj)
        self.datadir = join(path, device,
                            storage_directory(obj_dir, partition, name_hash))
        self.device_path = join(path, device)
        self.tmpdir = join(path, device, 'tmp')
        self.logger = logger
        self._metadata = None
        self.data_file = None
        self._data_file_size = None
        self.fp = None
        self.iter_etag = None
        self.started_at_0 = False
        self.read_to_eof = False
        self.quarantined_dir = None
        self.suppress_file_closing = False
        self._verify_close = False
        self.threadpool = threadpool or ThreadPool(nthreads=0)

        # FIXME(clayg): this attribute is set after open and affects the
        # behavior of the class (i.e. public interface)
        self.keep_cache = False
Esempio n. 8
0
 def dispatch(self, replicate_args, args):
     if not hasattr(args, 'pop'):
         return HTTPBadRequest(body='Invalid object type')
     op = args.pop(0)
     drive, partition, hsh = replicate_args
     if self.mount_check and \
             not os.path.ismount(os.path.join(self.root, drive)):
         return Response(status='507 %s is not mounted' % drive)
     db_file = os.path.join(self.root, drive,
                            storage_directory(self.datadir, partition, hsh),
                            hsh + '.db')
     if op == 'rsync_then_merge':
         return self.rsync_then_merge(drive, db_file, args)
     if op == 'complete_rsync':
         return self.complete_rsync(drive, db_file, args)
     else:
         # someone might be about to rsync a db to us,
         # make sure there's a tmp dir to receive it.
         mkdirs(os.path.join(self.root, drive, 'tmp'))
         if not os.path.exists(db_file):
             return HTTPNotFound()
         return getattr(self, op)(self.broker_class(db_file), args)
Esempio n. 9
0
 def dispatch(self, replicate_args, args):
     if not hasattr(args, 'pop'):
         return HTTPBadRequest(body='Invalid object type')
     op = args.pop(0)
     drive, partition, hsh = replicate_args
     if self.mount_check and \
             not os.path.ismount(os.path.join(self.root, drive)):
         return Response(status='507 %s is not mounted' % drive)
     db_file = os.path.join(self.root, drive,
                            storage_directory(self.datadir, partition, hsh),
                            hsh + '.db')
     if op == 'rsync_then_merge':
         return self.rsync_then_merge(drive, db_file, args)
     if op == 'complete_rsync':
         return self.complete_rsync(drive, db_file, args)
     else:
         # someone might be about to rsync a db to us,
         # make sure there's a tmp dir to receive it.
         mkdirs(os.path.join(self.root, drive, 'tmp'))
         if not os.path.exists(db_file):
             return HTTPNotFound()
         return getattr(self, op)(self.broker_class(db_file), args)
Esempio n. 10
0
 def test_storage_directory(self):
     self.assertEquals(ondisk.storage_directory('objects', '1', 'ABCDEF'),
                       'objects/1/DEF/ABCDEF')