Esempio n. 1
0
    def setup_bad_zero_byte(self, with_ts=False):
        self.auditor = auditor.ObjectAuditor(self.conf)
        self.auditor.log_time = 0
        ts_file_path = ''
        if with_ts:

            name_hash = hash_path('a', 'c', 'o')
            dir_path = os.path.join(self.devices, 'sda',
                                    storage_directory(DATADIR, '0', name_hash))
            ts_file_path = os.path.join(dir_path, '99999.ts')
            if not os.path.exists(dir_path):
                mkdirs(dir_path)
            fp = open(ts_file_path, 'w')
            fp.close()

        etag = md5()
        with self.disk_file.mkstemp() as (fd, tmppath):
            etag = etag.hexdigest()
            metadata = {
                'ETag': etag,
                'X-Timestamp': str(normalize_timestamp(time.time())),
                'Content-Length': 10,
            }
            self.disk_file.put(fd, tmppath, metadata)
            etag = md5()
            etag = etag.hexdigest()
            metadata['ETag'] = etag
            write_metadata(fd, metadata)
        if self.disk_file.data_file:
            return self.disk_file.data_file
        return ts_file_path
Esempio n. 2
0
    def setup_bad_zero_byte(self, with_ts=False):
        self.auditor = auditor.ObjectAuditor(self.conf)
        self.auditor.log_time = 0
        ts_file_path = ''
        if with_ts:

            name_hash = hash_path('a', 'c', 'o')
            dir_path = os.path.join(self.devices, 'sda',
                               storage_directory(DATADIR, '0', name_hash))
            ts_file_path = os.path.join(dir_path, '99999.ts')
            if not os.path.exists(dir_path):
                mkdirs(dir_path)
            fp = open(ts_file_path, 'w')
            fp.close()

        etag = md5()
        with self.disk_file.mkstemp() as (fd, tmppath):
            etag = etag.hexdigest()
            metadata = {
                'ETag': etag,
                'X-Timestamp': str(normalize_timestamp(time.time())),
                'Content-Length': 10,
            }
            self.disk_file.put(fd, tmppath, metadata)
            etag = md5()
            etag = etag.hexdigest()
            metadata['ETag'] = etag
            write_metadata(fd, metadata)
        if self.disk_file.data_file:
            return self.disk_file.data_file
        return ts_file_path
Esempio n. 3
0
 def __init__(self,
              path,
              device,
              partition,
              account,
              container,
              obj,
              logger,
              keep_data_fp=False,
              disk_chunk_size=65536):
     self.disk_chunk_size = disk_chunk_size
     self.name = '/' + '/'.join((account, container, obj))
     name_hash = hash_path(account, container, obj)
     self.datadir = os.path.join(
         path, device, storage_directory(DATADIR, partition, name_hash))
     self.device_path = os.path.join(path, device)
     self.tmpdir = os.path.join(path, device, 'tmp')
     self.logger = logger
     self.metadata = {}
     self.meta_file = None
     self.data_file = None
     self.fp = None
     self.iter_etag = None
     self.started_at_0 = False
     self.read_to_eof = False
     self.quarantined_dir = None
     self.keep_cache = False
     if not os.path.exists(self.datadir):
         return
     files = sorted(os.listdir(self.datadir), reverse=True)
     for file in files:
         if file.endswith('.ts'):
             self.data_file = self.meta_file = None
             self.metadata = {'deleted': True}
             return
         if file.endswith('.meta') and not self.meta_file:
             self.meta_file = os.path.join(self.datadir, file)
         if file.endswith('.data') and not self.data_file:
             self.data_file = os.path.join(self.datadir, file)
             break
     if not self.data_file:
         return
     self.fp = open(self.data_file, 'rb')
     self.metadata = read_metadata(self.fp)
     if not keep_data_fp:
         self.close(verify_file=False)
     if self.meta_file:
         with open(self.meta_file) as mfp:
             for key in self.metadata.keys():
                 if key.lower() not in DISALLOWED_HEADERS:
                     del self.metadata[key]
             self.metadata.update(read_metadata(mfp))
Esempio n. 4
0
    def _get_container_broker(self, drive, part, account, container):
        """
        Get a DB broker for the container.

        :param drive: drive that holds the container
        :param part: partition the container is in
        :param account: account name
        :param container: container name
        :returns: ContainerBroker object
        """
        hsh = hash_path(account, container)
        db_dir = storage_directory(DATADIR, part, hsh)
        db_path = os.path.join(self.root, drive, db_dir, hsh + '.db')
        return ContainerBroker(db_path, account=account, container=container,
                               logger=self.logger)
Esempio n. 5
0
    def _get_container_broker(self, drive, part, account, container):
        """
        Get a DB broker for the container.

        :param drive: drive that holds the container
        :param part: partition the container is in
        :param account: account name
        :param container: container name
        :returns: ContainerBroker object
        """
        hsh = hash_path(account, container)
        db_dir = storage_directory(DATADIR, part, hsh)
        db_path = os.path.join(self.root, drive, db_dir, hsh + '.db')
        return ContainerBroker(db_path,
                               account=account,
                               container=container,
                               logger=self.logger)
Esempio n. 6
0
 def __init__(self, path, device, partition, account, container, obj,
              logger, keep_data_fp=False, disk_chunk_size=65536):
     self.disk_chunk_size = disk_chunk_size
     self.name = '/' + '/'.join((account, container, obj))
     name_hash = hash_path(account, container, obj)
     self.datadir = os.path.join(path, device,
                 storage_directory(DATADIR, partition, name_hash))
     self.device_path = os.path.join(path, device)
     self.tmpdir = os.path.join(path, device, 'tmp')
     self.logger = logger
     self.metadata = {}
     self.meta_file = None
     self.data_file = None
     self.fp = None
     self.iter_etag = None
     self.started_at_0 = False
     self.read_to_eof = False
     self.quarantined_dir = None
     self.keep_cache = False
     if not os.path.exists(self.datadir):
         return
     files = sorted(os.listdir(self.datadir), reverse=True)
     for file in files:
         if file.endswith('.ts'):
             self.data_file = self.meta_file = None
             self.metadata = {'deleted': True}
             return
         if file.endswith('.meta') and not self.meta_file:
             self.meta_file = os.path.join(self.datadir, file)
         if file.endswith('.data') and not self.data_file:
             self.data_file = os.path.join(self.datadir, file)
             break
     if not self.data_file:
         return
     self.fp = open(self.data_file, 'rb')
     self.metadata = read_metadata(self.fp)
     if not keep_data_fp:
         self.close(verify_file=False)
     if self.meta_file:
         with open(self.meta_file) as mfp:
             for key in self.metadata.keys():
                 if key.lower() not in DISALLOWED_HEADERS:
                     del self.metadata[key]
             self.metadata.update(read_metadata(mfp))
Esempio n. 7
0
 def dispatch(self, replicate_args, args):
     if not hasattr(args, 'pop'):
         return HTTPBadRequest(body='Invalid object type')
     op = args.pop(0)
     drive, partition, hsh = replicate_args
     if self.mount_check and \
             not os.path.ismount(os.path.join(self.root, drive)):
         return Response(status='507 %s is not mounted' % drive)
     db_file = os.path.join(self.root, drive,
             storage_directory(self.datadir, partition, hsh), hsh + '.db')
     if op == 'rsync_then_merge':
         return self.rsync_then_merge(drive, db_file, args)
     if op == 'complete_rsync':
         return self.complete_rsync(drive, db_file, args)
     else:
         # someone might be about to rsync a db to us,
         # make sure there's a tmp dir to receive it.
         mkdirs(os.path.join(self.root, drive, 'tmp'))
         if not os.path.exists(db_file):
             return HTTPNotFound()
         return getattr(self, op)(self.broker_class(db_file), args)
Esempio n. 8
0
 def dispatch(self, replicate_args, args):
     if not hasattr(args, 'pop'):
         return HTTPBadRequest(body='Invalid object type')
     op = args.pop(0)
     drive, partition, hsh = replicate_args
     if self.mount_check and \
             not os.path.ismount(os.path.join(self.root, drive)):
         return Response(status='507 %s is not mounted' % drive)
     db_file = os.path.join(self.root, drive,
                            storage_directory(self.datadir, partition, hsh),
                            hsh + '.db')
     if op == 'rsync_then_merge':
         return self.rsync_then_merge(drive, db_file, args)
     if op == 'complete_rsync':
         return self.complete_rsync(drive, db_file, args)
     else:
         # someone might be about to rsync a db to us,
         # make sure there's a tmp dir to receive it.
         mkdirs(os.path.join(self.root, drive, 'tmp'))
         if not os.path.exists(db_file):
             return HTTPNotFound()
         return getattr(self, op)(self.broker_class(db_file), args)
Esempio n. 9
0
 def _get_account_broker(self, drive, part, account):
     hsh = hash_path(account)
     db_dir = storage_directory(DATADIR, part, hsh)
     db_path = os.path.join(self.root, drive, db_dir, hsh + '.db')
     return AccountBroker(db_path, account=account, logger=self.logger)
Esempio n. 10
0
 def _get_account_broker(self, drive, part, account):
     hsh = hash_path(account)
     db_dir = storage_directory(DATADIR, part, hsh)
     db_path = os.path.join(self.root, drive, db_dir, hsh + '.db')
     return AccountBroker(db_path, account=account, logger=self.logger)
Esempio n. 11
0
 def test_storage_directory(self):
     self.assertEquals(utils.storage_directory('objects', '1', 'ABCDEF'),
                       'objects/1/DEF/ABCDEF')
Esempio n. 12
0
 def test_storage_directory(self):
     self.assertEquals(utils.storage_directory('objects', '1', 'ABCDEF'),
             'objects/1/DEF/ABCDEF')