def test_DELETE(self): """ Test swift.object_server.ObjectController.DELETE """ req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'DELETE'}) resp = self.object_controller.DELETE(req) self.assertEquals(resp.status_int, 400) req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'DELETE'}) resp = self.object_controller.DELETE(req) self.assertEquals(resp.status_int, 400) # self.assertRaises(KeyError, self.object_controller.DELETE, req) timestamp = normalize_timestamp(time()) req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'DELETE'}, headers={'X-Timestamp': timestamp}) resp = self.object_controller.DELETE(req) self.assertEquals(resp.status_int, 404) sleep(.00001) timestamp = normalize_timestamp(time()) req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={ 'X-Timestamp': timestamp, 'Content-Type': 'application/octet-stream', 'Content-Length': '4', }) req.body = 'test' resp = self.object_controller.PUT(req) self.assertEquals(resp.status_int, 201) timestamp = normalize_timestamp(float(timestamp) - 1) req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'DELETE'}, headers={'X-Timestamp': timestamp}) resp = self.object_controller.DELETE(req) self.assertEquals(resp.status_int, 204) objfile = os.path.join(self.testdir, 'sda1', storage_directory(object_server.DATADIR, 'p', hash_path('a', 'c', 'o')), timestamp + '.ts') self.assert_(os.path.isfile(objfile)) sleep(.00001) timestamp = normalize_timestamp(time()) req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'DELETE'}, headers={'X-Timestamp': timestamp}) resp = self.object_controller.DELETE(req) self.assertEquals(resp.status_int, 204) objfile = os.path.join(self.testdir, 'sda1', storage_directory(object_server.DATADIR, 'p', hash_path('a', 'c', 'o')), timestamp + '.ts') self.assert_(os.path.isfile(objfile))
def test_hash_path(self): # Yes, these tests are deliberately very fragile. We want to make sure # that if someones changes the results hash_path produces, they know it self.assertEquals(utils.hash_path("a"), "1c84525acb02107ea475dcd3d09c2c58") self.assertEquals(utils.hash_path("a", "c"), "33379ecb053aa5c9e356c68997cbb59e") self.assertEquals(utils.hash_path("a", "c", "o"), "06fbf0b514e5199dfc4e00f42eb5ea83") self.assertEquals(utils.hash_path("a", "c", "o", raw_digest=False), "06fbf0b514e5199dfc4e00f42eb5ea83") self.assertEquals( utils.hash_path("a", "c", "o", raw_digest=True), "\x06\xfb\xf0\xb5\x14\xe5\x19\x9d\xfcN\x00\xf4.\xb5\xea\x83", ) self.assertRaises(ValueError, utils.hash_path, "a", object="o")
def get_nodes(self, account, container=None, obj=None): """ Get the partition and nodes for an account/container/object. If a node is responsible for more than one replica, it will only appear in the output once. :param account: account name :param container: container name :param obj: object name :returns: a tuple of (partition, list of node dicts) Each node dict will have at least the following keys: ====== =============================================================== id unique integer identifier amongst devices weight a float of the relative weight of this device as compared to others; this indicates how many partitions the builder will try to assign to this device zone integer indicating which zone the device is in; a given partition will not be assigned to multiple devices within the same zone ip the ip address of the device port the tcp port of the device device the device's name on disk (sdb1, for example) meta general use 'extra' field; for example: the online date, the hardware description ====== =============================================================== """ key = hash_path(account, container, obj, raw_digest=True) if time() > self._rtime: self._reload() part = unpack_from('>I', key)[0] >> self._part_shift seen_ids = set() return part, [self.devs[r[part]] for r in self._replica2part2dev_id if not (r[part] in seen_ids or seen_ids.add(r[part]))]
def _get_account_broker(self, drive, part, account, **kwargs): hsh = hash_path(account) db_dir = storage_directory(DATADIR, part, hsh) db_path = os.path.join(self.root, drive, db_dir, hsh + ".db") kwargs.setdefault("account", account) kwargs.setdefault("logger", self.logger) return AccountBroker(db_path, **kwargs)
def print_ring_locations(ring, datadir, account, container=None): """ print out ring locations of specified type :param ring: ring instance :param datadir: high level directory to store account/container/objects :param account: account name :param container: container name """ if ring is None or datadir is None or account is None: raise ValueError('None type') storage_type = 'account' if container: storage_type = 'container' try: part, nodes = ring.get_nodes(account, container, None) except (ValueError, AttributeError): raise ValueError('Ring error') else: path_hash = hash_path(account, container, None) print '\nRing locations:' for node in nodes: print (' %s:%s - /srv/node/%s/%s/%s.db' % (node['ip'], node['port'], node['device'], storage_directory(datadir, part, path_hash), path_hash)) print '\nnote: /srv/node is used as default value of `devices`, the ' \ 'real value is set in the %s config file on each storage node.' % \ storage_type
def setup_bad_zero_byte(self, with_ts=False): self.auditor = auditor.ObjectAuditor(self.conf) self.auditor.log_time = 0 ts_file_path = '' if with_ts: name_hash = hash_path('a', 'c', 'o') dir_path = os.path.join( self.devices, 'sda', storage_directory(get_data_dir(0), '0', name_hash)) ts_file_path = os.path.join(dir_path, '99999.ts') if not os.path.exists(dir_path): mkdirs(dir_path) fp = open(ts_file_path, 'w') write_metadata(fp, {'X-Timestamp': '99999', 'name': '/a/c/o'}) fp.close() etag = md5() with self.disk_file.create() as writer: etag = etag.hexdigest() metadata = { 'ETag': etag, 'X-Timestamp': str(normalize_timestamp(time.time())), 'Content-Length': 10, } writer.put(metadata) etag = md5() etag = etag.hexdigest() metadata['ETag'] = etag write_metadata(writer._fd, metadata) return ts_file_path
def test_run_once(self): replicator = object_replicator.ObjectReplicator( dict(swift_dir=self.testdir, devices=self.devices, mount_check='false', timeout='300', stats_interval='1')) was_connector = object_replicator.http_connect object_replicator.http_connect = mock_http_connect(200) cur_part = '0' df = DiskFile(self.devices, 'sda', cur_part, 'a', 'c', 'o', FakeLogger()) mkdirs(df.datadir) f = open(os.path.join(df.datadir, normalize_timestamp(time.time()) + '.data'), 'wb') f.write('1234567890') f.close() ohash = hash_path('a', 'c', 'o') data_dir = ohash[-3:] whole_path_from = os.path.join(self.objects, cur_part, data_dir) process_arg_checker = [] nodes = [node for node in self.ring.get_part_nodes(int(cur_part)) if node['ip'] not in _ips()] for node in nodes: rsync_mod = '%s::object/sda/objects/%s' % (node['ip'], cur_part) process_arg_checker.append( (0, '', ['rsync', whole_path_from, rsync_mod])) with _mock_process(process_arg_checker): replicator.run_once() self.assertFalse(process_errors) object_replicator.http_connect = was_connector
def test_run_once_1(self): conf = dict(swift_dir=self.testdir, devices=self.devices, mount_check='false', timeout='300', stats_interval='1') replicator = object_replicator.ObjectReplicator(conf) was_connector = object_replicator.http_connect object_replicator.http_connect = mock_http_connect(200) cur_part = '0' df = self.df_mgr.get_diskfile('sda', cur_part, 'a', 'c', 'o', policy_idx=1) mkdirs(df._datadir) f = open(os.path.join(df._datadir, normalize_timestamp(time.time()) + '.data'), 'wb') f.write('1234567890') f.close() ohash = hash_path('a', 'c', 'o') data_dir = ohash[-3:] whole_path_from = os.path.join(self.objects_1, cur_part, data_dir) process_arg_checker = [] ring = replicator.get_object_ring(1) nodes = [node for node in ring.get_part_nodes(int(cur_part)) if node['ip'] not in _ips()] rsync_mods = tuple(['%s::object/sda/objects-1/%s' % (node['ip'], cur_part) for node in nodes]) for node in nodes: process_arg_checker.append( (0, '', ['rsync', whole_path_from, rsync_mods])) with _mock_process(process_arg_checker): replicator.run_once() self.assertFalse(process_errors) object_replicator.http_connect = was_connector
def test_run_once(self): replicator = object_replicator.ObjectReplicator( dict(swift_dir=self.testdir, devices=self.devices, mount_check="false", timeout="300", stats_interval="1") ) was_connector = object_replicator.http_connect object_replicator.http_connect = mock_http_connect(200) cur_part = "0" df = DiskFile(self.devices, "sda", cur_part, "a", "c", "o", FakeLogger()) mkdirs(df.datadir) f = open(os.path.join(df.datadir, normalize_timestamp(time.time()) + ".data"), "wb") f.write("1234567890") f.close() ohash = hash_path("a", "c", "o") data_dir = ohash[-3:] whole_path_from = os.path.join(self.objects, cur_part, data_dir) process_arg_checker = [] nodes = [node for node in self.ring.get_part_nodes(int(cur_part)) if node["ip"] not in _ips()] for node in nodes: rsync_mod = "%s::object/sda/objects/%s" % (node["ip"], cur_part) process_arg_checker.append((0, "", ["rsync", whole_path_from, rsync_mod])) with _mock_process(process_arg_checker): replicator.run_once() self.assertFalse(process_errors) object_replicator.http_connect = was_connector
def test_run_once_recover_from_failure(self): replicator = object_replicator.ObjectReplicator( dict(swift_dir=self.testdir, devices=self.devices, mount_check="false", timeout="300", stats_interval="1") ) was_connector = object_replicator.http_connect try: object_replicator.http_connect = mock_http_connect(200) # Write some files into '1' and run replicate- they should be moved # to the other partitoins and then node should get deleted. cur_part = "1" df = DiskFile(self.devices, "sda", cur_part, "a", "c", "o", FakeLogger()) mkdirs(df.datadir) f = open(os.path.join(df.datadir, normalize_timestamp(time.time()) + ".data"), "wb") f.write("1234567890") f.close() ohash = hash_path("a", "c", "o") data_dir = ohash[-3:] whole_path_from = os.path.join(self.objects, cur_part, data_dir) process_arg_checker = [] nodes = [node for node in self.ring.get_part_nodes(int(cur_part)) if node["ip"] not in _ips()] for node in nodes: rsync_mod = "%s::object/sda/objects/%s" % (node["ip"], cur_part) process_arg_checker.append((0, "", ["rsync", whole_path_from, rsync_mod])) self.assertTrue(os.access(os.path.join(self.objects, "1", data_dir, ohash), os.F_OK)) with _mock_process(process_arg_checker): replicator.run_once() self.assertFalse(process_errors) for i, result in [("0", True), ("1", False), ("2", True), ("3", True)]: self.assertEquals( os.access(os.path.join(self.objects, i, object_replicator.HASH_FILE), os.F_OK), result ) finally: object_replicator.http_connect = was_connector
def __init__(self, path, device, partition, account, container, obj, logger, disk_chunk_size=65536, bytes_per_sync=(512 * 1024 * 1024), iter_hook=None, threadpool=None, obj_dir='objects', mount_check=False): if mount_check and not check_mount(path, device): raise DiskFileDeviceUnavailable() self.disk_chunk_size = disk_chunk_size self.bytes_per_sync = bytes_per_sync self.iter_hook = iter_hook self.name = '/' + '/'.join((account, container, obj)) name_hash = hash_path(account, container, obj) self.datadir = join( path, device, storage_directory(obj_dir, partition, name_hash)) self.device_path = join(path, device) self.tmpdir = join(path, device, 'tmp') self.logger = logger self._metadata = None self.data_file = None self._data_file_size = None self.fp = None self.iter_etag = None self.started_at_0 = False self.read_to_eof = False self.quarantined_dir = None self.suppress_file_closing = False self._verify_close = False self.threadpool = threadpool or ThreadPool(nthreads=0) # FIXME(clayg): this attribute is set after open and affects the # behavior of the class (i.e. public interface) self.keep_cache = False
def test_update_send_delete(self): device = 'dev' part = '9' object_parts = ('a', 'c', 'o') df = self._make_open_diskfile(device, part, *object_parts) object_hash = utils.hash_path(*object_parts) delete_timestamp = utils.normalize_timestamp(time.time()) df.delete(delete_timestamp) self.sender.connection = FakeConnection() self.sender.job = {'device': device, 'partition': part} self.sender.node = {} self.sender.send_list = [object_hash] self.sender.response = FakeResponse( chunk_body=( ':UPDATES: START\r\n' ':UPDATES: END\r\n')) self.sender.updates() self.assertEqual( ''.join(self.sender.connection.sent), '11\r\n:UPDATES: START\r\n\r\n' '30\r\n' 'DELETE /a/c/o\r\n' 'X-Timestamp: %s\r\n\r\n\r\n' 'f\r\n:UPDATES: END\r\n\r\n' % delete_timestamp )
def setup_bad_zero_byte(self, with_ts=False): self.auditor = auditor.ObjectAuditor(self.conf) self.auditor.log_time = 0 ts_file_path = '' if with_ts: name_hash = hash_path('a', 'c', 'o') dir_path = os.path.join(self.devices, 'sda', storage_directory(DATADIR, '0', name_hash)) ts_file_path = os.path.join(dir_path, '99999.ts') if not os.path.exists(dir_path): mkdirs(dir_path) fp = open(ts_file_path, 'w') fp.close() etag = md5() with self.disk_file.mkstemp() as (fd, tmppath): etag = etag.hexdigest() metadata = { 'ETag': etag, 'X-Timestamp': str(normalize_timestamp(time.time())), 'Content-Length': 10, } self.disk_file.put(fd, tmppath, metadata) etag = md5() etag = etag.hexdigest() metadata['ETag'] = etag write_metadata(fd, metadata) if self.disk_file.data_file: return self.disk_file.data_file return ts_file_path
def print_ring_locations(ring, datadir, account, container=None): """ print out ring locations of specified type :param ring: ring instance :param datadir: high level directory to store account/container/objects :param acount: account name :param container: container name """ if ring is None or datadir is None or account is None: raise ValueError("None type") storage_type = "account" if container: storage_type = "container" try: part, nodes = ring.get_nodes(account, container, None) except (ValueError, AttributeError): raise ValueError("Ring error") else: path_hash = hash_path(account, container, None) print "\nRing locations:" for node in nodes: print ( " %s:%s - /srv/node/%s/%s/%s.db" % (node["ip"], node["port"], node["device"], storage_directory(datadir, part, path_hash), path_hash) ) print "\nnote: /srv/node is used as default value of `devices`, the " "real value is set in the %s config file on each storage node." % storage_type
def test_hash_suffix_multi_file_two(self): df = diskfile.DiskFile(self.devices, 'sda', '0', 'a', 'c', 'o', FakeLogger()) mkdirs(df.datadir) for tdiff in [1, 50, 100, 500]: suffs = ['.meta', '.data'] if tdiff > 50: suffs.append('.ts') for suff in suffs: f = open( os.path.join( df.datadir, normalize_timestamp(int(time()) - tdiff) + suff), 'wb') f.write('1234567890') f.close() ohash = hash_path('a', 'c', 'o') data_dir = ohash[-3:] whole_path_from = os.path.join(self.objects, '0', data_dir) hsh_path = os.listdir(whole_path_from)[0] whole_hsh_path = os.path.join(whole_path_from, hsh_path) diskfile.hash_suffix(whole_path_from, 99) # only the meta and data should be left self.assertEquals(len(os.listdir(whole_hsh_path)), 2)
def test_updates_storage_policy_index(self): device = 'dev' part = '9' object_parts = ('a', 'c', 'o') df = self._make_open_diskfile(device, part, *object_parts, policy_idx=1) object_hash = utils.hash_path(*object_parts) expected = df.get_metadata() self.sender.connection = FakeConnection() self.sender.job = {'device': device, 'partition': part, 'policy_idx': 1} self.sender.node = {} self.sender.send_list = [object_hash] self.sender.send_delete = mock.MagicMock() self.sender.send_put = mock.MagicMock() self.sender.response = FakeResponse( chunk_body=( ':UPDATES: START\r\n' ':UPDATES: END\r\n')) self.sender.updates() args, _kwargs = self.sender.send_put.call_args path, df = args self.assertEqual(path, '/a/c/o') self.assert_(isinstance(df, diskfile.DiskFile)) self.assertEqual(expected, df.get_metadata()) self.assertEqual(os.path.join(self.testdir, 'dev/objects-1/9/', object_hash[-3:], object_hash), df._datadir)
def test_delete_partition_1(self): with mock.patch('swift.obj.replicator.http_connect', mock_http_connect(200)): df = self.df_mgr.get_diskfile('sda', '1', 'a', 'c', 'o', policy_idx=1) mkdirs(df._datadir) f = open(os.path.join(df._datadir, normalize_timestamp(time.time()) + '.data'), 'wb') f.write('1234567890') f.close() ohash = hash_path('a', 'c', 'o') data_dir = ohash[-3:] whole_path_from = os.path.join(self.objects_1, '1', data_dir) part_path = os.path.join(self.objects_1, '1') self.assertTrue(os.access(part_path, os.F_OK)) ring = self.replicator.get_object_ring(1) nodes = [node for node in ring.get_part_nodes(1) if node['ip'] not in _ips()] process_arg_checker = [] for node in nodes: rsync_mod = '%s::object/sda/objects-1/%s' % (node['ip'], 1) process_arg_checker.append( (0, '', ['rsync', whole_path_from, rsync_mod])) with _mock_process(process_arg_checker): self.replicator.replicate() self.assertFalse(os.access(part_path, os.F_OK))
def test_delete_partition_with_handoff_delete(self): with mock.patch('swift.obj.replicator.http_connect', mock_http_connect(200)): self.replicator.handoff_delete = 2 df = diskfile.DiskFile(self.devices, 'sda', '1', 'a', 'c', 'o', FakeLogger()) mkdirs(df.datadir) print df.datadir f = open(os.path.join(df.datadir, normalize_timestamp(time.time()) + '.data'), 'wb') f.write('1234567890') f.close() ohash = hash_path('a', 'c', 'o') data_dir = ohash[-3:] whole_path_from = os.path.join(self.objects, '1', data_dir) part_path = os.path.join(self.objects, '1') self.assertTrue(os.access(part_path, os.F_OK)) nodes = [node for node in self.ring.get_part_nodes(1) if node['ip'] not in _ips()] process_arg_checker = [] for i, node in enumerate(nodes): rsync_mod = '%s::object/sda/objects/%s' % (node['ip'], 1) if i == 0: # force one of the rsync calls to fail ret_code = 1 else: ret_code = 0 process_arg_checker.append( (ret_code, '', ['rsync', whole_path_from, rsync_mod])) with _mock_process(process_arg_checker): self.replicator.replicate() self.assertFalse(os.access(part_path, os.F_OK))
def test_delete_partition_with_handoff_delete_failures(self): with mock.patch('swift.obj.replicator.http_connect', mock_http_connect(200)): self.replicator.handoff_delete = 2 df = self.df_mgr.get_diskfile('sda', '1', 'a', 'c', 'o') mkdirs(df._datadir) f = open(os.path.join(df._datadir, normalize_timestamp(time.time()) + '.data'), 'wb') f.write('1234567890') f.close() ohash = hash_path('a', 'c', 'o') data_dir = ohash[-3:] whole_path_from = os.path.join(self.objects, '1', data_dir) part_path = os.path.join(self.objects, '1') self.assertTrue(os.access(part_path, os.F_OK)) ring = self.replicator.get_object_ring(0) nodes = [node for node in ring.get_part_nodes(1) if node['ip'] not in _ips()] process_arg_checker = [] for i, node in enumerate(nodes): rsync_mod = '%s::object/sda/objects/%s' % (node['ip'], 1) if i in (0, 1): # force two of the rsync calls to fail ret_code = 1 else: ret_code = 0 process_arg_checker.append( (ret_code, '', ['rsync', whole_path_from, rsync_mod])) with _mock_process(process_arg_checker): self.replicator.replicate() # The file should still exist self.assertTrue(os.access(part_path, os.F_OK))
def get_reconciler_broker(self, timestamp): """ Get a local instance of the reconciler container broker that is appropriate to enqueue the given timestamp. :param timestamp: the timestamp of the row to be enqueued :returns: a local reconciler broker """ container = get_reconciler_container_name(timestamp) if self.reconciler_containers and \ container in self.reconciler_containers: return self.reconciler_containers[container][1] account = MISPLACED_OBJECTS_ACCOUNT part = self.ring.get_part(account, container) node = self.find_local_handoff_for_part(part) if not node: raise DeviceUnavailable( 'No mounted devices found suitable to Handoff reconciler ' 'container %s in partition %s' % (container, part)) hsh = hash_path(account, container) db_dir = storage_directory(DATADIR, part, hsh) db_path = os.path.join(self.root, node['device'], db_dir, hsh + '.db') broker = ContainerBroker(db_path, account=account, container=container) if not os.path.exists(broker.db_file): try: broker.initialize(timestamp, 0) except DatabaseAlreadyExists: pass if self.reconciler_containers is not None: self.reconciler_containers[container] = part, broker, node['id'] return broker
def test_invalidate_hash(self): def assertFileData(file_path, data): with open(file_path, 'r') as fp: fdata = fp.read() self.assertEquals(pickle.loads(fdata), pickle.loads(data)) df = DiskFile(self.devices, 'sda', '0', 'a', 'c', 'o', FakeLogger()) mkdirs(df.datadir) ohash = hash_path('a', 'c', 'o') data_dir = ohash[-3:] whole_path_from = os.path.join(self.objects, '0', data_dir) hashes_file = os.path.join(self.objects, '0', object_replicator.HASH_FILE) # test that non existant file except caught self.assertEquals(object_replicator.invalidate_hash(whole_path_from), None) # test that hashes get cleared check_pickle_data = pickle.dumps({data_dir: None}, object_replicator.PICKLE_PROTOCOL) for data_hash in [{data_dir: None}, {data_dir: 'abcdefg'}]: with open(hashes_file, 'wb') as fp: pickle.dump(data_hash, fp, object_replicator.PICKLE_PROTOCOL) object_replicator.invalidate_hash(whole_path_from) assertFileData(hashes_file, check_pickle_data)
def test_PUT_overwrite(self): req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={'X-Timestamp': normalize_timestamp(time()), 'Content-Length': '6', 'Content-Type': 'application/octet-stream'}) req.body = 'VERIFY' resp = self.object_controller.PUT(req) self.assertEquals(resp.status_int, 201) sleep(.00001) timestamp = normalize_timestamp(time()) req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={'X-Timestamp': timestamp, 'Content-Type': 'text/plain', 'Content-Encoding': 'gzip'}) req.body = 'VERIFY TWO' resp = self.object_controller.PUT(req) self.assertEquals(resp.status_int, 201) objfile = os.path.join(self.testdir, 'sda1', storage_directory(object_server.DATADIR, 'p', hash_path('a', 'c', 'o')), timestamp + '.data') self.assert_(os.path.isfile(objfile)) self.assertEquals(open(objfile).read(), 'VERIFY TWO') self.assertEquals(pickle.loads(getxattr(objfile, object_server.METADATA_KEY)), {'X-Timestamp': timestamp, 'Content-Length': '10', 'ETag': 'b381a4c5dab1eaa1eb9711fa647cd039', 'Content-Type': 'text/plain', 'name': '/a/c/o', 'Content-Encoding': 'gzip'})
def test_updates_put(self): device = 'dev' part = '9' object_parts = ('a', 'c', 'o') df = self._make_open_diskfile(device, part, *object_parts) object_hash = utils.hash_path(*object_parts) expected = df.get_metadata() self.sender.connection = FakeConnection() self.sender.job = {'device': device, 'partition': part} self.sender.node = {} self.sender.send_list = [object_hash] self.sender.send_delete = mock.MagicMock() self.sender.send_put = mock.MagicMock() self.sender.response = FakeResponse( chunk_body=( ':UPDATES: START\r\n' ':UPDATES: END\r\n')) self.sender.updates() self.assertEqual(self.sender.send_delete.mock_calls, []) self.assertEqual(1, len(self.sender.send_put.mock_calls)) args, _kwargs = self.sender.send_put.call_args path, df = args self.assertEqual(path, '/a/c/o') self.assert_(isinstance(df, diskfile.DiskFile)) self.assertEqual(expected, df.get_metadata()) # note that the put line isn't actually sent since we mock send_put; # send_put is tested separately. self.assertEqual( ''.join(self.sender.connection.sent), '11\r\n:UPDATES: START\r\n\r\n' 'f\r\n:UPDATES: END\r\n\r\n')
def test_PUT_user_metadata(self): timestamp = normalize_timestamp(time()) req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={'X-Timestamp': timestamp, 'Content-Type': 'text/plain', 'ETag': 'b114ab7b90d9ccac4bd5d99cc7ebb568', 'X-Object-Meta-1': 'One', 'X-Object-Meta-Two': 'Two'}) req.body = 'VERIFY THREE' resp = self.object_controller.PUT(req) self.assertEquals(resp.status_int, 201) objfile = os.path.join(self.testdir, 'sda1', storage_directory(object_server.DATADIR, 'p', hash_path('a', 'c', 'o')), timestamp + '.data') self.assert_(os.path.isfile(objfile)) self.assertEquals(open(objfile).read(), 'VERIFY THREE') self.assertEquals(pickle.loads(getxattr(objfile, object_server.METADATA_KEY)), {'X-Timestamp': timestamp, 'Content-Length': '12', 'ETag': 'b114ab7b90d9ccac4bd5d99cc7ebb568', 'Content-Type': 'text/plain', 'name': '/a/c/o', 'X-Object-Meta-1': 'One', 'X-Object-Meta-Two': 'Two'})
def test_updates_is_deleted(self): device = 'dev' part = '9' object_parts = ('a', 'c', 'o') df = self._make_open_diskfile(device, part, *object_parts) object_hash = utils.hash_path(*object_parts) delete_timestamp = utils.normalize_timestamp(time.time()) df.delete(delete_timestamp) self.sender.connection = FakeConnection() self.sender.job = {'device': device, 'partition': part} self.sender.node = {} self.sender.send_list = [object_hash] self.sender.send_delete = mock.MagicMock() self.sender.send_put = mock.MagicMock() self.sender.response = FakeResponse( chunk_body=( ':UPDATES: START\r\n' ':UPDATES: END\r\n')) self.sender.updates() self.sender.send_delete.assert_called_once_with( '/a/c/o', delete_timestamp) self.assertEqual(self.sender.send_put.mock_calls, []) # note that the delete line isn't actually sent since we mock # send_delete; send_delete is tested separately. self.assertEqual( ''.join(self.sender.connection.sent), '11\r\n:UPDATES: START\r\n\r\n' 'f\r\n:UPDATES: END\r\n\r\n')
def setup_bad_zero_byte(self, with_ts=False): self.auditor = auditor.ObjectAuditor(self.conf) self.auditor.log_time = 0 ts_file_path = "" if with_ts: name_hash = hash_path("a", "c", "o") dir_path = os.path.join(self.devices, "sda", storage_directory(DATADIR, "0", name_hash)) ts_file_path = os.path.join(dir_path, "99999.ts") if not os.path.exists(dir_path): mkdirs(dir_path) fp = open(ts_file_path, "w") fp.close() etag = md5() with self.disk_file.mkstemp() as (fd, tmppath): etag = etag.hexdigest() metadata = {"ETag": etag, "X-Timestamp": str(normalize_timestamp(time.time())), "Content-Length": 10} self.disk_file.put(fd, tmppath, metadata) etag = md5() etag = etag.hexdigest() metadata["ETag"] = etag write_metadata(fd, metadata) if self.disk_file.data_file: return self.disk_file.data_file return ts_file_path
def test_run_once_recover_from_timeout(self): replicator = object_replicator.ObjectReplicator( dict(swift_dir=self.testdir, devices=self.devices, mount_check='false', timeout='300', stats_interval='1')) was_connector = object_replicator.http_connect was_get_hashes = object_replicator.get_hashes was_execute = tpool.execute self.get_hash_count = 0 try: def fake_get_hashes(*args, **kwargs): self.get_hash_count += 1 if self.get_hash_count == 3: # raise timeout on last call to get hashes raise Timeout() return 2, {'abc': 'def'} def fake_exc(tester, *args, **kwargs): if 'Error syncing partition' in args[0]: tester.i_failed = True self.i_failed = False object_replicator.http_connect = mock_http_connect(200) object_replicator.get_hashes = fake_get_hashes replicator.logger.exception = \ lambda *args, **kwargs: fake_exc(self, *args, **kwargs) # Write some files into '1' and run replicate- they should be moved # to the other partitions and then node should get deleted. cur_part = '1' df = DiskFile(self.devices, 'sda', cur_part, 'a', 'c', 'o', FakeLogger()) mkdirs(df.datadir) f = open(os.path.join(df.datadir, normalize_timestamp(time.time()) + '.data'), 'wb') f.write('1234567890') f.close() ohash = hash_path('a', 'c', 'o') data_dir = ohash[-3:] whole_path_from = os.path.join(self.objects, cur_part, data_dir) process_arg_checker = [] nodes = [node for node in self.ring.get_part_nodes(int(cur_part)) if node['ip'] not in _ips()] for node in nodes: rsync_mod = '%s::object/sda/objects/%s' % (node['ip'], cur_part) process_arg_checker.append( (0, '', ['rsync', whole_path_from, rsync_mod])) self.assertTrue(os.access(os.path.join(self.objects, '1', data_dir, ohash), os.F_OK)) with _mock_process(process_arg_checker): replicator.run_once() self.assertFalse(process_errors) self.assertFalse(self.i_failed) finally: object_replicator.http_connect = was_connector object_replicator.get_hashes = was_get_hashes tpool.execute = was_execute
def __get_partition__(self, account, container, obj, part_shift): """ Internal method to calculate partition with MD5 hash """ key = hash_path(account, container, obj, raw_digest=True) part = unpack_from('>I', key)[0] >> part_shift return part
def get_enc_key(self, account, container, obj): """ Generate a unique, random key for encrypting this file. - Maybe based on the user passwords, and/or credentials - Need chained keys to support tenants/group access in the above case. """ enc_key = sha1(hash_path(account, container, obj)).hexdigest() return enc_key[:16]
def _get_account_broker(self, drive, part, account): if self.fs_object: return DiskAccount(self.root, account, self.fs_object) hsh = hash_path(account) db_dir = storage_directory(DATADIR, part, hsh) db_path = os.path.join(self.root, drive, db_dir, hsh + '.db') return AccountBroker(db_path, account=account, logger=self.logger)
def __init__(self, path, device, partition, account, container, obj, logger, keep_data_fp=False, disk_chunk_size=65536, bytes_per_sync=(512 * 1024 * 1024), iter_hook=None): self.disk_chunk_size = disk_chunk_size self.bytes_per_sync = bytes_per_sync self.iter_hook = iter_hook self.name = '/' + '/'.join((account, container, obj)) name_hash = hash_path(account, container, obj) self.datadir = os.path.join( path, device, storage_directory(DATADIR, partition, name_hash)) self.device_path = os.path.join(path, device) self.tmpdir = os.path.join(path, device, 'tmp') self.logger = logger self.metadata = {} self.meta_file = None self.data_file = None self.fp = None self.iter_etag = None self.started_at_0 = False self.read_to_eof = False self.quarantined_dir = None self.keep_cache = False self.suppress_file_closing = False if not os.path.exists(self.datadir): return files = sorted(os.listdir(self.datadir), reverse=True) for afile in files: if afile.endswith('.ts'): self.data_file = self.meta_file = None self.metadata = {'deleted': True} return if afile.endswith('.meta') and not self.meta_file: self.meta_file = os.path.join(self.datadir, afile) if afile.endswith('.data') and not self.data_file: self.data_file = os.path.join(self.datadir, afile) break if not self.data_file: return self.fp = open(self.data_file, 'rb') self.metadata = read_metadata(self.fp) if not keep_data_fp: self.close(verify_file=False) if self.meta_file: with open(self.meta_file) as mfp: for key in self.metadata.keys(): if key.lower() not in DISALLOWED_HEADERS: del self.metadata[key] self.metadata.update(read_metadata(mfp)) if 'name' in self.metadata: if self.metadata['name'] != self.name: self.logger.error( _('Client path %(client)s does not match ' 'path stored in object metadata %(meta)s'), { 'client': self.name, 'meta': self.metadata['name'] }) raise DiskFileCollision('Client path does not match path ' 'stored in object metadata')
def container_sync(self, path): """ Checks the given path for a container database, determines if syncing is turned on for that database and, if so, sends any updates to the other container. :param path: the path to a container db """ broker = None try: broker = ContainerBroker(path) info = broker.get_info() x, nodes = self.container_ring.get_nodes(info['account'], info['container']) for ordinal, node in enumerate(nodes): if node['ip'] in self._myips and node['port'] == self._myport: break else: return if not broker.is_deleted(): sync_to = None sync_key = None sync_point1 = info['x_container_sync_point1'] sync_point2 = info['x_container_sync_point2'] for key, (value, timestamp) in broker.metadata.iteritems(): if key.lower() == 'x-container-sync-to': sync_to = value elif key.lower() == 'x-container-sync-key': sync_key = value if not sync_to or not sync_key: self.container_skips += 1 self.logger.increment('skips') return sync_to = sync_to.rstrip('/') err = validate_sync_to(sync_to, self.allowed_sync_hosts) if err: self.logger.info( _('ERROR %(db_file)s: %(validate_sync_to_err)s'), {'db_file': broker.db_file, 'validate_sync_to_err': err}) self.container_failures += 1 self.logger.increment('failures') return stop_at = time() + self.container_time next_sync_point = None while time() < stop_at and sync_point2 < sync_point1: rows = broker.get_items_since(sync_point2, 1) if not rows: break row = rows[0] if row['ROWID'] > sync_point1: break key = hash_path(info['account'], info['container'], row['name'], raw_digest=True) # This node will only initially sync out one third of the # objects (if 3 replicas, 1/4 if 4, etc.) and will skip # problematic rows as needed in case of faults. # This section will attempt to sync previously skipped # rows in case the previous attempts by any of the nodes # didn't succeed. if not self.container_sync_row(row, sync_to, sync_key, broker, info): if not next_sync_point: next_sync_point = sync_point2 sync_point2 = row['ROWID'] broker.set_x_container_sync_points(None, sync_point2) if next_sync_point: broker.set_x_container_sync_points(None, next_sync_point) while time() < stop_at: rows = broker.get_items_since(sync_point1, 1) if not rows: break row = rows[0] key = hash_path(info['account'], info['container'], row['name'], raw_digest=True) # This node will only initially sync out one third of the # objects (if 3 replicas, 1/4 if 4, etc.). It'll come back # around to the section above and attempt to sync # previously skipped rows in case the other nodes didn't # succeed or in case it failed to do so the first time. if unpack_from('>I', key)[0] % \ len(nodes) == ordinal: self.container_sync_row(row, sync_to, sync_key, broker, info) sync_point1 = row['ROWID'] broker.set_x_container_sync_points(sync_point1, None) self.container_syncs += 1 self.logger.increment('syncs') except (Exception, Timeout) as err: self.container_failures += 1 self.logger.increment('failures') self.logger.exception(_('ERROR Syncing %s'), broker.db_file if broker else path)
def async_update(self, op, account, container, obj, host, partition, contdevice, headers_out, objdevice): """ Sends or saves an async update. :param op: operation performed (ex: 'PUT', or 'DELETE') :param account: account name for the object :param container: container name for the object :param obj: object name :param host: host that the container is on :param partition: partition that the container is on :param contdevice: device name that the container is on :param headers_out: dictionary of headers to send in the container request :param objdevice: device name that the object is in """ full_path = '/%s/%s/%s' % (account, container, obj) if all([host, partition, contdevice]): try: with ConnectionTimeout(self.conn_timeout): ip, port = host.rsplit(':', 1) conn = http_connect(ip, port, contdevice, partition, op, full_path, headers_out) with Timeout(self.node_timeout): response = conn.getresponse() response.read() if is_success(response.status): return else: self.logger.error( _('ERROR Container update failed ' '(saving for async update later): %(status)d ' 'response from %(ip)s:%(port)s/%(dev)s'), { 'status': response.status, 'ip': ip, 'port': port, 'dev': contdevice }) except (Exception, Timeout): self.logger.exception( _('ERROR container update failed with ' '%(ip)s:%(port)s/%(dev)s (saving for async update later)' ), { 'ip': ip, 'port': port, 'dev': contdevice }) async_dir = os.path.join(self.devices, objdevice, ASYNCDIR) ohash = hash_path(account, container, obj) self.logger.increment('async_pendings') write_pickle( { 'op': op, 'account': account, 'container': container, 'obj': obj, 'headers': headers_out }, os.path.join( async_dir, ohash[-3:], ohash + '-' + normalize_timestamp(headers_out['x-timestamp'])), os.path.join(self.devices, objdevice, 'tmp'))
def print_db_info_metadata(db_type, info, metadata): """ print out data base info/metadata based on its type :param db_type: database type, account or container :param info: dict of data base info :param metadata: dict of data base metadata """ if info is None: raise ValueError('DB info is None') if db_type not in ['container', 'account']: raise ValueError('Wrong DB type') try: account = info['account'] container = None if db_type == 'container': container = info['container'] path = '/%s/%s' % (account, container) else: path = '/%s' % account print 'Path: %s' % path print ' Account: %s' % account if db_type == 'container': print ' Container: %s' % container path_hash = hash_path(account, container) if db_type == 'container': print ' Container Hash: %s' % path_hash else: print ' Account Hash: %s' % path_hash print 'Metadata:' print(' Created at: %s (%s)' % (Timestamp(info['created_at']).isoformat, info['created_at'])) print(' Put Timestamp: %s (%s)' % (Timestamp( info['put_timestamp']).isoformat, info['put_timestamp'])) print(' Delete Timestamp: %s (%s)' % (Timestamp( info['delete_timestamp']).isoformat, info['delete_timestamp'])) print(' Status Timestamp: %s (%s)' % (Timestamp( info['status_changed_at']).isoformat, info['status_changed_at'])) if db_type == 'account': print ' Container Count: %s' % info['container_count'] print ' Object Count: %s' % info['object_count'] print ' Bytes Used: %s' % info['bytes_used'] if db_type == 'container': try: policy_name = POLICIES[info['storage_policy_index']].name except KeyError: policy_name = 'Unknown' print(' Storage Policy: %s (%s)' % (policy_name, info['storage_policy_index'])) print(' Reported Put Timestamp: %s (%s)' % (Timestamp(info['reported_put_timestamp']).isoformat, info['reported_put_timestamp'])) print(' Reported Delete Timestamp: %s (%s)' % (Timestamp(info['reported_delete_timestamp']).isoformat, info['reported_delete_timestamp'])) print ' Reported Object Count: %s' % info['reported_object_count'] print ' Reported Bytes Used: %s' % info['reported_bytes_used'] print ' Chexor: %s' % info['hash'] print ' UUID: %s' % info['id'] except KeyError as e: raise ValueError('Info is incomplete: %s' % e) meta_prefix = 'x_' + db_type + '_' for key, value in info.iteritems(): if key.lower().startswith(meta_prefix): title = key.replace('_', '-').title() print ' %s: %s' % (title, value) user_metadata = {} sys_metadata = {} for key, (value, timestamp) in metadata.iteritems(): if is_user_meta(db_type, key): user_metadata[strip_user_meta_prefix(db_type, key)] = value elif is_sys_meta(db_type, key): sys_metadata[strip_sys_meta_prefix(db_type, key)] = value else: title = key.replace('_', '-').title() print ' %s: %s' % (title, value) if sys_metadata: print ' System Metadata: %s' % sys_metadata else: print 'No system metadata found in db file' if user_metadata: print ' User Metadata: %s' % user_metadata else: print 'No user metadata found in db file'
def print_ring_locations(ring, datadir, account, container=None, obj=None, tpart=None, all_nodes=False, policy_index=None): """ print out ring locations of specified type :param ring: ring instance :param datadir: name of directory where things are stored. Usually one of "accounts", "containers", "objects", or "objects-N". :param account: account name :param container: container name :param obj: object name :param tpart: target partition in ring :param all_nodes: include all handoff nodes. If false, only the N primary nodes and first N handoffs will be printed. :param policy_index: include policy_index in curl headers """ if not ring: raise ValueError("No ring specified") if not datadir: raise ValueError("No datadir specified") if tpart is None and not account: raise ValueError("No partition or account/container/object specified") if not account and (container or obj): raise ValueError("Container/object specified without account") if obj and not container: raise ValueError('Object specified without container') if obj: target = '%s/%s/%s' % (account, container, obj) elif container: target = '%s/%s' % (account, container) else: target = '%s' % (account) if tpart: part = int(tpart) else: part = ring.get_part(account, container, obj) primary_nodes = ring.get_part_nodes(part) handoff_nodes = ring.get_more_nodes(part) if not all_nodes: handoff_nodes = itertools.islice(handoff_nodes, len(primary_nodes)) handoff_nodes = list(handoff_nodes) if account and not tpart: path_hash = hash_path(account, container, obj) else: path_hash = None print 'Partition\t%s' % part print 'Hash \t%s\n' % path_hash for node in primary_nodes: print 'Server:Port Device\t%s:%s %s' % (node['ip'], node['port'], node['device']) for node in handoff_nodes: print 'Server:Port Device\t%s:%s %s\t [Handoff]' % ( node['ip'], node['port'], node['device']) print "\n" for node in primary_nodes: cmd = 'curl -I -XHEAD "http://%s:%s/%s/%s/%s"' \ % (node['ip'], node['port'], node['device'], part, urllib.quote(target)) if policy_index is not None: cmd += ' -H "%s: %s"' % ('X-Backend-Storage-Policy-Index', policy_index) print cmd for node in handoff_nodes: cmd = 'curl -I -XHEAD "http://%s:%s/%s/%s/%s"' \ % (node['ip'], node['port'], node['device'], part, urllib.quote(target)) if policy_index is not None: cmd += ' -H "%s: %s"' % ('X-Backend-Storage-Policy-Index', policy_index) cmd += ' # [Handoff]' print cmd print "\n\nUse your own device location of servers:" print "such as \"export DEVICE=/srv/node\"" if path_hash: for node in primary_nodes: print('ssh %s "ls -lah ${DEVICE:-/srv/node*}/%s/%s"' % (node['ip'], node['device'], storage_directory(datadir, part, path_hash))) for node in handoff_nodes: print('ssh %s "ls -lah ${DEVICE:-/srv/node*}/%s/%s" # [Handoff]' % (node['ip'], node['device'], storage_directory(datadir, part, path_hash))) else: for node in primary_nodes: print('ssh %s "ls -lah ${DEVICE:-/srv/node*}/%s/%s/%d"' % (node['ip'], node['device'], datadir, part)) for node in handoff_nodes: print( 'ssh %s "ls -lah ${DEVICE:-/srv/node*}/%s/%s/%d"' ' # [Handoff]' % (node['ip'], node['device'], datadir, part)) print '\nnote: `/srv/node*` is used as default value of `devices`, the ' \ 'real value is set in the config file on each storage node.'
def _get_storage_dir(self, part, node, account=None, container=None): # TODO: fix hard-coded path datadir = os.path.join('/srv/1/node/sdb1', 'containers') container_hash = swift_utils.hash_path(account, container) return (swift_utils.storage_directory(datadir, part, container_hash), container_hash)
def test_sweep_logs(self): asyncdir = os.path.join(self.sda1, ASYNCDIR_BASE) prefix_dir = os.path.join(asyncdir, 'abc') mkpath(prefix_dir) for o, t in [('abc', 123), ('def', 234), ('ghi', 345), ('jkl', 456), ('mno', 567)]: ohash = hash_path('account', 'container', o) o_path = os.path.join(prefix_dir, ohash + '-' + normalize_timestamp(t)) write_pickle({}, o_path) class MockObjectUpdater(object_updater.ObjectUpdater): def process_object_update(self, update_path, device, policy): os.unlink(update_path) self.stats.successes += 1 self.stats.unlinks += 1 logger = FakeLogger() ou = MockObjectUpdater( { 'devices': self.devices_dir, 'mount_check': 'false', 'swift_dir': self.testdir, 'interval': '1', 'concurrency': '1', 'report_interval': '10.0', 'node_timeout': '5' }, logger=logger) now = [time()] def mock_time_function(): rv = now[0] now[0] += 5 return rv # With 10s between updates, time() advancing 5s every time we look, # and 5 async_pendings on disk, we should get at least two progress # lines. with mock.patch('swift.obj.updater.time', mock.MagicMock(time=mock_time_function)): ou.object_sweep(self.sda1) info_lines = logger.get_lines_for_level('info') self.assertEqual(4, len(info_lines)) self.assertIn("sweep starting", info_lines[0]) self.assertIn(self.sda1, info_lines[0]) self.assertIn("sweep progress", info_lines[1]) # the space ensures it's a positive number self.assertIn( "2 successes, 0 failures, 0 quarantines, 2 unlinks, 0 errors, " "0 redirects", info_lines[1]) self.assertIn(self.sda1, info_lines[1]) self.assertIn("sweep progress", info_lines[2]) self.assertIn( "4 successes, 0 failures, 0 quarantines, 4 unlinks, 0 errors, " "0 redirects", info_lines[2]) self.assertIn(self.sda1, info_lines[2]) self.assertIn("sweep complete", info_lines[3]) self.assertIn( "5 successes, 0 failures, 0 quarantines, 5 unlinks, 0 errors, " "0 redirects", info_lines[3]) self.assertIn(self.sda1, info_lines[3])
def check_with_idx(index, warn, should_skip): if int(index) > 0: asyncdir = os.path.join(self.sda1, ASYNCDIR_BASE + "-" + index) else: asyncdir = os.path.join(self.sda1, ASYNCDIR_BASE) prefix_dir = os.path.join(asyncdir, 'abc') mkpath(prefix_dir) # A non-directory where directory is expected should just be # skipped, but should not stop processing of subsequent # directories. not_dirs = (os.path.join(self.sda1, 'not_a_dir'), os.path.join(self.sda1, ASYNCDIR_BASE + '-' + 'twentington'), os.path.join( self.sda1, ASYNCDIR_BASE + '-' + str(int(index) + 100))) for not_dir in not_dirs: with open(not_dir, 'w'): pass objects = { 'a': [1089.3, 18.37, 12.83, 1.3], 'b': [49.4, 49.3, 49.2, 49.1], 'c': [109984.123], } expected = set() for o, timestamps in objects.iteritems(): ohash = hash_path('account', 'container', o) for t in timestamps: o_path = os.path.join(prefix_dir, ohash + '-' + normalize_timestamp(t)) if t == timestamps[0]: expected.add((o_path, int(index))) write_pickle({}, o_path) seen = set() class MockObjectUpdater(object_updater.ObjectUpdater): def process_object_update(self, update_path, device, idx): seen.add((update_path, idx)) os.unlink(update_path) cu = MockObjectUpdater({ 'devices': self.devices_dir, 'mount_check': 'false', 'swift_dir': self.testdir, 'interval': '1', 'concurrency': '1', 'node_timeout': '5' }) cu.logger = mock_logger = mock.MagicMock() cu.object_sweep(self.sda1) self.assertEquals(mock_logger.warn.call_count, warn) self.assert_(os.path.exists(os.path.join(self.sda1, 'not_a_dir'))) if should_skip: # if we were supposed to skip over the dir, we didn't process # anything at all self.assertTrue(os.path.exists(prefix_dir)) self.assertEqual(set(), seen) else: self.assert_(not os.path.exists(prefix_dir)) self.assertEqual(expected, seen) # test cleanup: the tempdir gets cleaned up between runs, but this # way we can be called multiple times in a single test method for not_dir in not_dirs: os.unlink(not_dir)
def test_run_once_recover_from_timeout(self): replicator = object_replicator.ObjectReplicator( dict(swift_dir=self.testdir, devices=self.devices, mount_check='false', timeout='300', stats_interval='1')) was_connector = object_replicator.http_connect was_get_hashes = object_replicator.get_hashes was_execute = tpool.execute self.get_hash_count = 0 try: def fake_get_hashes(*args, **kwargs): self.get_hash_count += 1 if self.get_hash_count == 3: # raise timeout on last call to get hashes raise Timeout() return 2, {'abc': 'def'} def fake_exc(tester, *args, **kwargs): if 'Error syncing partition' in args[0]: tester.i_failed = True self.i_failed = False object_replicator.http_connect = mock_http_connect(200) object_replicator.get_hashes = fake_get_hashes replicator.logger.exception = \ lambda *args, **kwargs: fake_exc(self, *args, **kwargs) # Write some files into '1' and run replicate- they should be moved # to the other partitions and then node should get deleted. cur_part = '1' df = DiskFile(self.devices, 'sda', cur_part, 'a', 'c', 'o', FakeLogger()) mkdirs(df.datadir) f = open( os.path.join(df.datadir, normalize_timestamp(time.time()) + '.data'), 'wb') f.write('1234567890') f.close() ohash = hash_path('a', 'c', 'o') data_dir = ohash[-3:] whole_path_from = os.path.join(self.objects, cur_part, data_dir) process_arg_checker = [] nodes = [node for node in self.ring.get_part_nodes(int(cur_part)) \ if node['ip'] not in _ips()] for node in nodes: rsync_mod = '%s::object/sda/objects/%s' % (node['ip'], cur_part) process_arg_checker.append( (0, '', ['rsync', whole_path_from, rsync_mod])) self.assertTrue( os.access(os.path.join(self.objects, '1', data_dir, ohash), os.F_OK)) with _mock_process(process_arg_checker): replicator.run_once() self.assertFalse(process_errors) self.assertFalse(self.i_failed) finally: object_replicator.http_connect = was_connector object_replicator.get_hashes = was_get_hashes tpool.execute = was_execute
def test_run_once(self, mock_ismount): mock_ismount.return_value = True cu = object_updater.ObjectUpdater( { 'devices': self.devices_dir, 'mount_check': 'false', 'swift_dir': self.testdir, 'interval': '1', 'concurrency': '1', 'node_timeout': '15' }, logger=self.logger) cu.run_once() async_dir = os.path.join(self.sda1, get_async_dir(0)) os.mkdir(async_dir) cu.run_once() self.assert_(os.path.exists(async_dir)) # mount_check == False means no call to ismount self.assertEqual([], mock_ismount.mock_calls) cu = object_updater.ObjectUpdater( { 'devices': self.devices_dir, 'mount_check': 'TrUe', 'swift_dir': self.testdir, 'interval': '1', 'concurrency': '1', 'node_timeout': '15' }, logger=self.logger) odd_dir = os.path.join(async_dir, 'not really supposed ' 'to be here') os.mkdir(odd_dir) cu.run_once() self.assert_(os.path.exists(async_dir)) self.assert_(not os.path.exists(odd_dir)) # mount_check == True means ismount was checked self.assertEqual([ mock.call(self.sda1), ], mock_ismount.mock_calls) ohash = hash_path('a', 'c', 'o') odir = os.path.join(async_dir, ohash[-3:]) mkdirs(odir) older_op_path = os.path.join( odir, '%s-%s' % (ohash, normalize_timestamp(time() - 1))) op_path = os.path.join(odir, '%s-%s' % (ohash, normalize_timestamp(time()))) for path in (op_path, older_op_path): with open(path, 'wb') as async_pending: pickle.dump( { 'op': 'PUT', 'account': 'a', 'container': 'c', 'obj': 'o', 'headers': { 'X-Container-Timestamp': normalize_timestamp(0) } }, async_pending) cu.run_once() self.assert_(not os.path.exists(older_op_path)) self.assert_(os.path.exists(op_path)) self.assertEqual(cu.logger.get_increment_counts(), { 'failures': 1, 'unlinks': 1 }) self.assertEqual(None, pickle.load(open(op_path)).get('successes')) bindsock = listen(('127.0.0.1', 0)) def accepter(sock, return_code): try: with Timeout(3): inc = sock.makefile('rb') out = sock.makefile('wb') out.write('HTTP/1.1 %d OK\r\nContent-Length: 0\r\n\r\n' % return_code) out.flush() self.assertEquals(inc.readline(), 'PUT /sda1/0/a/c/o HTTP/1.1\r\n') headers = swob.HeaderKeyDict() line = inc.readline() while line and line != '\r\n': headers[line.split(':')[0]] = \ line.split(':')[1].strip() line = inc.readline() self.assertTrue('x-container-timestamp' in headers) self.assertTrue( 'X-Backend-Storage-Policy-Index' in headers) except BaseException as err: return err return None def accept(return_codes): codes = iter(return_codes) try: events = [] for x in xrange(len(return_codes)): with Timeout(3): sock, addr = bindsock.accept() events.append(spawn(accepter, sock, codes.next())) for event in events: err = event.wait() if err: raise err except BaseException as err: return err return None event = spawn(accept, [201, 500, 500]) for dev in cu.get_container_ring().devs: if dev is not None: dev['port'] = bindsock.getsockname()[1] cu.logger._clear() cu.run_once() err = event.wait() if err: raise err self.assert_(os.path.exists(op_path)) self.assertEqual(cu.logger.get_increment_counts(), {'failures': 1}) self.assertEqual([0], pickle.load(open(op_path)).get('successes')) event = spawn(accept, [404, 500]) cu.logger._clear() cu.run_once() err = event.wait() if err: raise err self.assert_(os.path.exists(op_path)) self.assertEqual(cu.logger.get_increment_counts(), {'failures': 1}) self.assertEqual([0, 1], pickle.load(open(op_path)).get('successes')) event = spawn(accept, [201]) cu.logger._clear() cu.run_once() err = event.wait() if err: raise err self.assert_(not os.path.exists(op_path)) self.assertEqual(cu.logger.get_increment_counts(), { 'unlinks': 1, 'successes': 1 })
def test_run_once(self): cu = object_updater.ObjectUpdater({ 'devices': self.devices_dir, 'mount_check': 'false', 'swift_dir': self.testdir, 'interval': '1', 'concurrency': '1', 'node_timeout': '15'}) cu.run_once() async_dir = os.path.join(self.sda1, object_server.ASYNCDIR) os.mkdir(async_dir) cu.run_once() self.assert_(os.path.exists(async_dir)) odd_dir = os.path.join(async_dir, 'not really supposed to be here') os.mkdir(odd_dir) cu.run_once() self.assert_(os.path.exists(async_dir)) self.assert_(not os.path.exists(odd_dir)) ohash = hash_path('a', 'c', 'o') odir = os.path.join(async_dir, ohash[-3:]) mkdirs(odir) older_op_path = os.path.join( odir, '%s-%s' % (ohash, normalize_timestamp(time() - 1))) op_path = os.path.join( odir, '%s-%s' % (ohash, normalize_timestamp(time()))) for path in (op_path, older_op_path): with open(path, 'wb') as async_pending: pickle.dump({'op': 'PUT', 'account': 'a', 'container': 'c', 'obj': 'o', 'headers': { 'X-Container-Timestamp': normalize_timestamp(0)}}, async_pending) cu.logger = FakeLogger() cu.run_once() self.assert_(not os.path.exists(older_op_path)) self.assert_(os.path.exists(op_path)) self.assertEqual(cu.logger.get_increment_counts(), {'failures': 1, 'unlinks': 1}) bindsock = listen(('127.0.0.1', 0)) def accepter(sock, return_code): try: with Timeout(3): inc = sock.makefile('rb') out = sock.makefile('wb') out.write('HTTP/1.1 %d OK\r\nContent-Length: 0\r\n\r\n' % return_code) out.flush() self.assertEquals(inc.readline(), 'PUT /sda1/0/a/c/o HTTP/1.1\r\n') headers = {} line = inc.readline() while line and line != '\r\n': headers[line.split(':')[0].lower()] = \ line.split(':')[1].strip() line = inc.readline() self.assert_('x-container-timestamp' in headers) except BaseException, err: return err return None
def _get_account_broker(self, drive, part, account): hsh = hash_path(account) db_dir = storage_directory(DATADIR, part, hsh) db_path = os.path.join(self.root, drive, db_dir, hsh + '.db') return AccountBroker(db_path, account=account, logger=self.logger)
def container_update(self, op, account, container, obj, headers_in, headers_out, objdevice): """ Update the container when objects are updated. :param op: operation performed (ex: 'PUT', or 'DELETE') :param account: account name for the object :param container: container name for the object :param obj: object name :param headers_in: dictionary of headers from the original request :param headers_out: dictionary of headers to send in the container request :param objdevice: device name that the object is in """ host = headers_in.get('X-Container-Host', None) partition = headers_in.get('X-Container-Partition', None) contdevice = headers_in.get('X-Container-Device', None) if not all([host, partition, contdevice]): return full_path = '/%s/%s/%s' % (account, container, obj) try: with ConnectionTimeout(self.conn_timeout): ip, port = host.rsplit(':', 1) conn = http_connect(ip, port, contdevice, partition, op, full_path, headers_out) with Timeout(self.node_timeout): response = conn.getresponse() response.read() if 200 <= response.status < 300: return else: self.logger.error( _('ERROR Container update failed ' '(saving for async update later): %(status)d ' 'response from %(ip)s:%(port)s/%(dev)s'), { 'status': response.status, 'ip': ip, 'port': port, 'dev': contdevice }) except (Exception, TimeoutError): self.logger.exception( _('ERROR container update failed with ' '%(ip)s:%(port)s/%(dev)s (saving for async update later)'), { 'ip': ip, 'port': port, 'dev': contdevice }) async_dir = os.path.join(self.devices, objdevice, ASYNCDIR) ohash = hash_path(account, container, obj) write_pickle( { 'op': op, 'account': account, 'container': container, 'obj': obj, 'headers': headers_out }, os.path.join( async_dir, ohash[-3:], ohash + '-' + normalize_timestamp(headers_out['x-timestamp'])), os.path.join(self.devices, objdevice, 'tmp'))
def container_sync(self, path): """ Checks the given path for a container database, determines if syncing is turned on for that database and, if so, sends any updates to the other container. :param path: the path to a container db """ broker = None try: broker = ContainerBroker(path) # The path we pass to the ContainerBroker is a real path of # a container DB. If we get here, however, it means that this # path is linked from the sync_containers dir. In rare cases # of race or processes failures the link can be stale and # the get_info below will raise a DB doesn't exist exception # In this case we remove the stale link and raise an error # since in most cases the db should be there. try: info = broker.get_info() except DatabaseConnectionError as db_err: if str(db_err).endswith("DB doesn't exist"): self.sync_store.remove_synced_container(broker) raise x, nodes = self.container_ring.get_nodes(info['account'], info['container']) for ordinal, node in enumerate(nodes): if is_local_device(self._myips, self._myport, node['ip'], node['port']): break else: return if not broker.is_deleted(): sync_to = None user_key = None sync_point1 = info['x_container_sync_point1'] sync_point2 = info['x_container_sync_point2'] for key, (value, timestamp) in broker.metadata.items(): if key.lower() == 'x-container-sync-to': sync_to = value elif key.lower() == 'x-container-sync-key': user_key = value if not sync_to or not user_key: self.container_skips += 1 self.logger.increment('skips') return err, sync_to, realm, realm_key = validate_sync_to( sync_to, self.allowed_sync_hosts, self.realms_conf) if err: self.logger.info( _('ERROR %(db_file)s: %(validate_sync_to_err)s'), { 'db_file': str(broker), 'validate_sync_to_err': err }) self.container_failures += 1 self.logger.increment('failures') return start_at = time() stop_at = start_at + self.container_time next_sync_point = None sync_stage_time = start_at try: while time() < stop_at and sync_point2 < sync_point1: rows = broker.get_items_since(sync_point2, 1) if not rows: break row = rows[0] if row['ROWID'] > sync_point1: break # This node will only initially sync out one third # of the objects (if 3 replicas, 1/4 if 4, etc.) # and will skip problematic rows as needed in case of # faults. # This section will attempt to sync previously skipped # rows in case the previous attempts by any of the # nodes didn't succeed. if not self.container_sync_row(row, sync_to, user_key, broker, info, realm, realm_key): if not next_sync_point: next_sync_point = sync_point2 sync_point2 = row['ROWID'] broker.set_x_container_sync_points(None, sync_point2) if next_sync_point: broker.set_x_container_sync_points( None, next_sync_point) else: next_sync_point = sync_point2 sync_stage_time = time() while sync_stage_time < stop_at: rows = broker.get_items_since(sync_point1, 1) if not rows: break row = rows[0] key = hash_path(info['account'], info['container'], row['name'], raw_digest=True) # This node will only initially sync out one third of # the objects (if 3 replicas, 1/4 if 4, etc.). # It'll come back around to the section above # and attempt to sync previously skipped rows in case # the other nodes didn't succeed or in case it failed # to do so the first time. if unpack_from('>I', key)[0] % \ len(nodes) == ordinal: self.container_sync_row(row, sync_to, user_key, broker, info, realm, realm_key) sync_point1 = row['ROWID'] broker.set_x_container_sync_points(sync_point1, None) sync_stage_time = time() self.container_syncs += 1 self.logger.increment('syncs') except Exception as ex: raise ex finally: self.container_report(start_at, sync_stage_time, sync_point1, next_sync_point, info, broker.get_max_row()) except (Exception, Timeout): self.container_failures += 1 self.logger.increment('failures') self.logger.exception(_('ERROR Syncing %s'), broker if broker else path)
def print_db_info_metadata(db_type, info, metadata, drop_prefixes=False): """ print out data base info/metadata based on its type :param db_type: database type, account or container :param info: dict of data base info :param metadata: dict of data base metadata :param drop_prefixes: if True, strip "X-Account-Meta-", "X-Container-Meta-", "X-Account-Sysmeta-", and "X-Container-Sysmeta-" when displaying User Metadata and System Metadata dicts """ if info is None: raise ValueError('DB info is None') if db_type not in ['container', 'account']: raise ValueError('Wrong DB type') try: account = info['account'] container = None if db_type == 'container': container = info['container'] path = '/%s/%s' % (account, container) else: path = '/%s' % account print('Path: %s' % path) print(' Account: %s' % account) if db_type == 'container': print(' Container: %s' % container) path_hash = hash_path(account, container) if db_type == 'container': print(' Container Hash: %s' % path_hash) else: print(' Account Hash: %s' % path_hash) print('Metadata:') print(' Created at: %s (%s)' % (Timestamp(info['created_at']).isoformat, info['created_at'])) print(' Put Timestamp: %s (%s)' % (Timestamp( info['put_timestamp']).isoformat, info['put_timestamp'])) print(' Delete Timestamp: %s (%s)' % (Timestamp( info['delete_timestamp']).isoformat, info['delete_timestamp'])) print(' Status Timestamp: %s (%s)' % (Timestamp( info['status_changed_at']).isoformat, info['status_changed_at'])) if db_type == 'account': print(' Container Count: %s' % info['container_count']) print(' Object Count: %s' % info['object_count']) print(' Bytes Used: %s' % info['bytes_used']) if db_type == 'container': try: policy_name = POLICIES[info['storage_policy_index']].name except KeyError: policy_name = 'Unknown' print(' Storage Policy: %s (%s)' % (policy_name, info['storage_policy_index'])) print(' Reported Put Timestamp: %s (%s)' % (Timestamp(info['reported_put_timestamp']).isoformat, info['reported_put_timestamp'])) print(' Reported Delete Timestamp: %s (%s)' % (Timestamp(info['reported_delete_timestamp']).isoformat, info['reported_delete_timestamp'])) print(' Reported Object Count: %s' % info['reported_object_count']) print(' Reported Bytes Used: %s' % info['reported_bytes_used']) print(' Chexor: %s' % info['hash']) print(' UUID: %s' % info['id']) except KeyError as e: raise ValueError('Info is incomplete: %s' % e) meta_prefix = 'x_' + db_type + '_' for key, value in info.items(): if key.lower().startswith(meta_prefix): title = key.replace('_', '-').title() print(' %s: %s' % (title, value)) user_metadata = {} sys_metadata = {} for key, (value, timestamp) in metadata.items(): if is_user_meta(db_type, key): if drop_prefixes: key = strip_user_meta_prefix(db_type, key) user_metadata[key] = value elif is_sys_meta(db_type, key): if drop_prefixes: key = strip_sys_meta_prefix(db_type, key) sys_metadata[key] = value else: title = key.replace('_', '-').title() print(' %s: %s' % (title, value)) if sys_metadata: print(' System Metadata: %s' % sys_metadata) else: print('No system metadata found in db file') if user_metadata: print(' User Metadata: %s' % user_metadata) else: print('No user metadata found in db file') if db_type == 'container': print('Sharding Metadata:') shard_type = 'root' if info['is_root'] else 'shard' print(' Type: %s' % shard_type) print(' State: %s' % info['db_state']) if info.get('shard_ranges'): print('Shard Ranges (%d):' % len(info['shard_ranges'])) for srange in info['shard_ranges']: srange = dict(srange, state_text=srange.state_text) print(' Name: %(name)s' % srange) print(' lower: %(lower)r, upper: %(upper)r' % srange) print(' Object Count: %(object_count)d, Bytes Used: ' '%(bytes_used)d, State: %(state_text)s (%(state)d)' % srange) print(' Created at: %s (%s)' % (Timestamp( srange['timestamp']).isoformat, srange['timestamp'])) print(' Meta Timestamp: %s (%s)' % (Timestamp( srange['meta_timestamp']).isoformat, srange['meta_timestamp']))
def print_obj_metadata(metadata): """ Print out basic info and metadata from object, as returned from :func:`swift.obj.diskfile.read_metadata`. Metadata should include the keys: name, Content-Type, and X-Timestamp. Additional metadata is displayed unmodified. :param metadata: dict of object metadata :raises: ValueError """ user_metadata = {} sys_metadata = {} other_metadata = {} if not metadata: raise ValueError('Metadata is None') path = metadata.pop('name', '') content_type = metadata.pop('Content-Type', '') ts = Timestamp(metadata.pop('X-Timestamp', 0)) account = container = obj = obj_hash = None if path: try: account, container, obj = path.split('/', 3)[1:] except ValueError: raise ValueError('Path is invalid for object %r' % path) else: obj_hash = hash_path(account, container, obj) print('Path: %s' % path) print(' Account: %s' % account) print(' Container: %s' % container) print(' Object: %s' % obj) print(' Object hash: %s' % obj_hash) else: print('Path: Not found in metadata') if content_type: print('Content-Type: %s' % content_type) else: print('Content-Type: Not found in metadata') if ts: print('Timestamp: %s (%s)' % (ts.isoformat, ts.internal)) else: print('Timestamp: Not found in metadata') for key, value in metadata.items(): if is_user_meta('Object', key): user_metadata[key] = value elif is_sys_meta('Object', key): sys_metadata[key] = value else: other_metadata[key] = value def print_metadata(title, items): print(title) if items: for meta_key in sorted(items): print(' %s: %s' % (meta_key, items[meta_key])) else: print(' No metadata found') print_metadata('System Metadata:', sys_metadata) print_metadata('User Metadata:', user_metadata) print_metadata('Other Metadata:', other_metadata)
def test_run_once(self, mock_check_drive): mock_check_drive.side_effect = lambda r, d, mc: os.path.join(r, d) ou = object_updater.ObjectUpdater( { 'devices': self.devices_dir, 'mount_check': 'false', 'swift_dir': self.testdir, 'interval': '1', 'concurrency': '1', 'node_timeout': '15' }, logger=self.logger) ou.run_once() async_dir = os.path.join(self.sda1, get_async_dir(POLICIES[0])) os.mkdir(async_dir) ou.run_once() self.assertTrue(os.path.exists(async_dir)) # each run calls check_device self.assertEqual([ mock.call(self.devices_dir, 'sda1', False), mock.call(self.devices_dir, 'sda1', False), ], mock_check_drive.mock_calls) mock_check_drive.reset_mock() ou = object_updater.ObjectUpdater( { 'devices': self.devices_dir, 'mount_check': 'TrUe', 'swift_dir': self.testdir, 'interval': '1', 'concurrency': '1', 'node_timeout': '15' }, logger=self.logger) odd_dir = os.path.join(async_dir, 'not really supposed ' 'to be here') os.mkdir(odd_dir) ou.run_once() self.assertTrue(os.path.exists(async_dir)) self.assertEqual([ mock.call(self.devices_dir, 'sda1', True), ], mock_check_drive.mock_calls) ohash = hash_path('a', 'c', 'o') odir = os.path.join(async_dir, ohash[-3:]) mkdirs(odir) older_op_path = os.path.join( odir, '%s-%s' % (ohash, normalize_timestamp(time() - 1))) op_path = os.path.join(odir, '%s-%s' % (ohash, normalize_timestamp(time()))) for path in (op_path, older_op_path): with open(path, 'wb') as async_pending: pickle.dump( { 'op': 'PUT', 'account': 'a', 'container': 'c', 'obj': 'o', 'headers': { 'X-Container-Timestamp': normalize_timestamp(0) } }, async_pending) ou.run_once() self.assertTrue(not os.path.exists(older_op_path)) self.assertTrue(os.path.exists(op_path)) self.assertEqual(ou.logger.get_increment_counts(), { 'failures': 1, 'unlinks': 1 }) self.assertIsNone(pickle.load(open(op_path, 'rb')).get('successes')) bindsock = listen_zero() def accepter(sock, return_code): try: with Timeout(3): inc = sock.makefile('rb') out = sock.makefile('wb') out.write(b'HTTP/1.1 %d OK\r\nContent-Length: 0\r\n\r\n' % return_code) out.flush() self.assertEqual(inc.readline(), b'PUT /sda1/0/a/c/o HTTP/1.1\r\n') headers = HeaderKeyDict() line = inc.readline() while line and line != b'\r\n': headers[line.split(b':')[0]] = \ line.split(b':')[1].strip() line = inc.readline() self.assertIn(b'x-container-timestamp', headers) self.assertIn(b'X-Backend-Storage-Policy-Index', headers) except BaseException as err: return err return None def accept(return_codes): try: events = [] for code in return_codes: with Timeout(3): sock, addr = bindsock.accept() events.append(spawn(accepter, sock, code)) for event in events: err = event.wait() if err: raise err except BaseException as err: return err return None event = spawn(accept, [201, 500, 500]) for dev in ou.get_container_ring().devs: if dev is not None: dev['port'] = bindsock.getsockname()[1] ou.logger._clear() ou.run_once() err = event.wait() if err: raise err self.assertTrue(os.path.exists(op_path)) self.assertEqual(ou.logger.get_increment_counts(), {'failures': 1}) self.assertEqual([0], pickle.load(open(op_path, 'rb')).get('successes')) event = spawn(accept, [404, 201]) ou.logger._clear() ou.run_once() err = event.wait() if err: raise err self.assertTrue(os.path.exists(op_path)) self.assertEqual(ou.logger.get_increment_counts(), {'failures': 1}) self.assertEqual([0, 2], pickle.load(open(op_path, 'rb')).get('successes')) event = spawn(accept, [201]) ou.logger._clear() ou.run_once() err = event.wait() if err: raise err # we remove the async_pending and its containing suffix dir, but not # anything above that self.assertFalse(os.path.exists(op_path)) self.assertFalse(os.path.exists(os.path.dirname(op_path))) self.assertTrue( os.path.exists(os.path.dirname(os.path.dirname(op_path)))) self.assertEqual(ou.logger.get_increment_counts(), { 'unlinks': 1, 'successes': 1 })
def print_obj_metadata(metadata, drop_prefixes=False): """ Print out basic info and metadata from object, as returned from :func:`swift.obj.diskfile.read_metadata`. Metadata should include the keys: name, Content-Type, and X-Timestamp. Additional metadata is displayed unmodified. :param metadata: dict of object metadata :param drop_prefixes: if True, strip "X-Object-Meta-", "X-Object-Sysmeta-", and "X-Object-Transient-Sysmeta-" when displaying User Metadata, System Metadata, and Transient System Metadata entries :raises ValueError: """ user_metadata = {} sys_metadata = {} transient_sys_metadata = {} other_metadata = {} if not metadata: raise ValueError('Metadata is None') path = metadata.pop('name', '') content_type = metadata.pop('Content-Type', '') ts = Timestamp(metadata.pop('X-Timestamp', 0)) account = container = obj = obj_hash = None if path: try: account, container, obj = path.split('/', 3)[1:] except ValueError: raise ValueError('Path is invalid for object %r' % path) else: obj_hash = hash_path(account, container, obj) print('Path: %s' % path) print(' Account: %s' % account) print(' Container: %s' % container) print(' Object: %s' % obj) print(' Object hash: %s' % obj_hash) else: print('Path: Not found in metadata') if content_type: print('Content-Type: %s' % content_type) else: print('Content-Type: Not found in metadata') if ts: print('Timestamp: %s (%s)' % (ts.isoformat, ts.internal)) else: print('Timestamp: Not found in metadata') for key, value in metadata.items(): if is_user_meta('Object', key): if drop_prefixes: key = strip_user_meta_prefix('Object', key) user_metadata[key] = value elif is_sys_meta('Object', key): if drop_prefixes: key = strip_sys_meta_prefix('Object', key) sys_metadata[key] = value elif is_object_transient_sysmeta(key): if drop_prefixes: key = strip_object_transient_sysmeta_prefix(key) transient_sys_metadata[key] = value else: other_metadata[key] = value def print_metadata(title, items): print(title) if items: for key, value in sorted(items.items()): print(' %s: %s' % (key, value)) else: print(' No metadata found') print_metadata('System Metadata:', sys_metadata) print_metadata('Transient System Metadata:', transient_sys_metadata) print_metadata('User Metadata:', user_metadata) print_metadata('Other Metadata:', other_metadata) for label, meta in [ ('Data crypto details', sys_metadata.get('X-Object-Sysmeta-Crypto-Body-Meta')), ('Metadata crypto details', transient_sys_metadata.get('X-Object-Transient-Sysmeta-Crypto-Meta')), ]: if meta is None: continue print('%s: %s' % (label, json.dumps(load_crypto_meta(meta, b64decode=False), indent=2, sort_keys=True, separators=(',', ': '))))
def get_broker(self, account, container, part, node): db_hash = hash_path(account.encode('utf-8'), container.encode('utf-8')) db_dir = storage_directory(DATADIR, part, db_hash) db_path = os.path.join(self.root, node['device'], db_dir, db_hash + '.db') return ContainerBroker(db_path, account=account, container=container)
def __init__(self, path, device, partition, account, container, obj, logger, keep_data_fp=False, disk_chunk_size=65536, origin_disk_chunk_size=65536, iter_hook=None, encryption_context=None, crypto_driver=None): self.disk_chunk_size = disk_chunk_size self.origin_disk_chunk_size = origin_disk_chunk_size self.iter_hook = iter_hook self.name = '/' + '/'.join((account, container, obj)) name_hash = hash_path(account, container, obj) self.datadir = os.path.join( path, device, storage_directory(DATADIR, partition, name_hash)) self.device_path = os.path.join(path, device) self.tmpdir = os.path.join(path, device, 'tmp') self.tmppath = None self.logger = logger self.metadata = {} self.meta_file = None self.data_file = None self.fp = None self.iter_etag = None self.started_at_0 = False self.read_to_eof = False self.quarantined_dir = None self.keep_cache = False self.suppress_file_closing = False self.encryption_context = encryption_context self.crypto_driver = crypto_driver if not os.path.exists(self.datadir): return files = sorted(os.listdir(self.datadir), reverse=True) for file in files: if file.endswith('.ts'): self.data_file = self.meta_file = None self.metadata = {'deleted': True} return if file.endswith('.meta') and not self.meta_file: self.meta_file = os.path.join(self.datadir, file) if file.endswith('.data') and not self.data_file: self.data_file = os.path.join(self.datadir, file) break if not self.data_file: return self.fp = open(self.data_file, 'rb') self.metadata = read_metadata(self.fp) if not keep_data_fp: self.close(verify_file=False) if self.meta_file: with open(self.meta_file) as mfp: for key in self.metadata.keys(): if key.lower() not in DISALLOWED_HEADERS: del self.metadata[key] self.metadata.update(read_metadata(mfp)) if self.crypto_driver and not self.encryption_context: key_id = self.metadata.get('X-Object-Meta-Key-Id') self.encryption_context = \ self.crypto_driver.encryption_context(key_id)