def test_strip_user_meta_prefix(self): mt = 'meta' for st in server_types: self.assertEqual(strip_user_meta_prefix(st, 'x-%s-%s-a' % (st, mt)), 'a') mt = 'not-meta' for st in server_types: with self.assertRaises(ValueError): strip_sys_meta_prefix(st, 'x-%s-%s-a' % (st, mt))
def test_strip_user_meta_prefix(self): mt = 'meta' for st in server_types: self.assertEqual( rh.strip_user_meta_prefix(st, 'x-%s-%s-a' % (st, mt)), 'a') mt = 'not-meta' for st in server_types: with self.assertRaises(ValueError): rh.strip_sys_meta_prefix(st, 'x-%s-%s-a' % (st, mt))
def _prep_headers_to_info(headers, server_type): """ Helper method that iterates once over a dict of headers, converting all keys to lower case and separating into subsets containing user metadata, system metadata and other headers. """ meta = {} sysmeta = {} other = {} for key, val in dict(headers).iteritems(): lkey = key.lower() if is_user_meta(server_type, lkey): meta[strip_user_meta_prefix(server_type, lkey)] = val elif is_sys_meta(server_type, lkey): sysmeta[strip_sys_meta_prefix(server_type, lkey)] = val else: other[lkey] = val return other, meta, sysmeta
def print_db_info_metadata(db_type, info, metadata): """ print out data base info/metadata based on its type :param db_type: database type, account or container :param info: dict of data base info :param metadata: dict of data base metadata """ if info is None: raise ValueError('DB info is None') if db_type not in ['container', 'account']: raise ValueError('Wrong DB type') try: account = info['account'] container = None if db_type == 'container': container = info['container'] path = '/%s/%s' % (account, container) else: path = '/%s' % account print 'Path: %s' % path print ' Account: %s' % account if db_type == 'container': print ' Container: %s' % container path_hash = hash_path(account, container) if db_type == 'container': print ' Container Hash: %s' % path_hash else: print ' Account Hash: %s' % path_hash print 'Metadata:' print (' Created at: %s (%s)' % (Timestamp(info['created_at']).isoformat, info['created_at'])) print (' Put Timestamp: %s (%s)' % (Timestamp(info['put_timestamp']).isoformat, info['put_timestamp'])) print (' Delete Timestamp: %s (%s)' % (Timestamp(info['delete_timestamp']).isoformat, info['delete_timestamp'])) print (' Status Timestamp: %s (%s)' % (Timestamp(info['status_changed_at']).isoformat, info['status_changed_at'])) if db_type == 'account': print ' Container Count: %s' % info['container_count'] print ' Object Count: %s' % info['object_count'] print ' Bytes Used: %s' % info['bytes_used'] if db_type == 'container': try: policy_name = POLICIES[info['storage_policy_index']].name except KeyError: policy_name = 'Unknown' print (' Storage Policy: %s (%s)' % ( policy_name, info['storage_policy_index'])) print (' Reported Put Timestamp: %s (%s)' % (Timestamp(info['reported_put_timestamp']).isoformat, info['reported_put_timestamp'])) print (' Reported Delete Timestamp: %s (%s)' % (Timestamp(info['reported_delete_timestamp']).isoformat, info['reported_delete_timestamp'])) print ' Reported Object Count: %s' % info['reported_object_count'] print ' Reported Bytes Used: %s' % info['reported_bytes_used'] print ' Chexor: %s' % info['hash'] print ' UUID: %s' % info['id'] except KeyError as e: raise ValueError('Info is incomplete: %s' % e) meta_prefix = 'x_' + db_type + '_' for key, value in info.iteritems(): if key.lower().startswith(meta_prefix): title = key.replace('_', '-').title() print ' %s: %s' % (title, value) user_metadata = {} sys_metadata = {} for key, (value, timestamp) in metadata.iteritems(): if is_user_meta(db_type, key): user_metadata[strip_user_meta_prefix(db_type, key)] = value elif is_sys_meta(db_type, key): sys_metadata[strip_sys_meta_prefix(db_type, key)] = value else: title = key.replace('_', '-').title() print ' %s: %s' % (title, value) if sys_metadata: print ' System Metadata: %s' % sys_metadata else: print 'No system metadata found in db file' if user_metadata: print ' User Metadata: %s' % user_metadata else: print 'No user metadata found in db file'
def print_obj_metadata(metadata, drop_prefixes=False): """ Print out basic info and metadata from object, as returned from :func:`swift.obj.diskfile.read_metadata`. Metadata should include the keys: name, Content-Type, and X-Timestamp. Additional metadata is displayed unmodified. :param metadata: dict of object metadata :param drop_prefixes: if True, strip "X-Object-Meta-", "X-Object-Sysmeta-", and "X-Object-Transient-Sysmeta-" when displaying User Metadata, System Metadata, and Transient System Metadata entries :raises ValueError: """ user_metadata = {} sys_metadata = {} transient_sys_metadata = {} other_metadata = {} if not metadata: raise ValueError('Metadata is None') path = metadata.pop('name', '') content_type = metadata.pop('Content-Type', '') ts = Timestamp(metadata.pop('X-Timestamp', 0)) account = container = obj = obj_hash = None if path: try: account, container, obj = path.split('/', 3)[1:] except ValueError: raise ValueError('Path is invalid for object %r' % path) else: obj_hash = hash_path(account, container, obj) print('Path: %s' % path) print(' Account: %s' % account) print(' Container: %s' % container) print(' Object: %s' % obj) print(' Object hash: %s' % obj_hash) else: print('Path: Not found in metadata') if content_type: print('Content-Type: %s' % content_type) else: print('Content-Type: Not found in metadata') if ts: print('Timestamp: %s (%s)' % (ts.isoformat, ts.internal)) else: print('Timestamp: Not found in metadata') for key, value in metadata.items(): if is_user_meta('Object', key): if drop_prefixes: key = strip_user_meta_prefix('Object', key) user_metadata[key] = value elif is_sys_meta('Object', key): if drop_prefixes: key = strip_sys_meta_prefix('Object', key) sys_metadata[key] = value elif is_object_transient_sysmeta(key): if drop_prefixes: key = strip_object_transient_sysmeta_prefix(key) transient_sys_metadata[key] = value else: other_metadata[key] = value def print_metadata(title, items): print(title) if items: for key, value in sorted(items.items()): print(' %s: %s' % (key, value)) else: print(' No metadata found') print_metadata('System Metadata:', sys_metadata) print_metadata('Transient System Metadata:', transient_sys_metadata) print_metadata('User Metadata:', user_metadata) print_metadata('Other Metadata:', other_metadata)
def print_db_info_metadata(db_type, info, metadata, drop_prefixes=False): """ print out data base info/metadata based on its type :param db_type: database type, account or container :param info: dict of data base info :param metadata: dict of data base metadata :param drop_prefixes: if True, strip "X-Account-Meta-", "X-Container-Meta-", "X-Account-Sysmeta-", and "X-Container-Sysmeta-" when displaying User Metadata and System Metadata dicts """ if info is None: raise ValueError('DB info is None') if db_type not in ['container', 'account']: raise ValueError('Wrong DB type') try: account = info['account'] container = None if db_type == 'container': container = info['container'] path = '/%s/%s' % (account, container) else: path = '/%s' % account print('Path: %s' % path) print(' Account: %s' % account) if db_type == 'container': print(' Container: %s' % container) path_hash = hash_path(account, container) if db_type == 'container': print(' Container Hash: %s' % path_hash) else: print(' Account Hash: %s' % path_hash) print('Metadata:') print(' Created at: %s (%s)' % (Timestamp(info['created_at']).isoformat, info['created_at'])) print(' Put Timestamp: %s (%s)' % (Timestamp(info['put_timestamp']).isoformat, info['put_timestamp'])) print(' Delete Timestamp: %s (%s)' % (Timestamp(info['delete_timestamp']).isoformat, info['delete_timestamp'])) print(' Status Timestamp: %s (%s)' % (Timestamp(info['status_changed_at']).isoformat, info['status_changed_at'])) if db_type == 'account': print(' Container Count: %s' % info['container_count']) print(' Object Count: %s' % info['object_count']) print(' Bytes Used: %s' % info['bytes_used']) if db_type == 'container': try: policy_name = POLICIES[info['storage_policy_index']].name except KeyError: policy_name = 'Unknown' print(' Storage Policy: %s (%s)' % ( policy_name, info['storage_policy_index'])) print(' Reported Put Timestamp: %s (%s)' % (Timestamp(info['reported_put_timestamp']).isoformat, info['reported_put_timestamp'])) print(' Reported Delete Timestamp: %s (%s)' % (Timestamp(info['reported_delete_timestamp']).isoformat, info['reported_delete_timestamp'])) print(' Reported Object Count: %s' % info['reported_object_count']) print(' Reported Bytes Used: %s' % info['reported_bytes_used']) print(' Chexor: %s' % info['hash']) print(' UUID: %s' % info['id']) except KeyError as e: raise ValueError('Info is incomplete: %s' % e) meta_prefix = 'x_' + db_type + '_' for key, value in info.items(): if key.lower().startswith(meta_prefix): title = key.replace('_', '-').title() print(' %s: %s' % (title, value)) user_metadata = {} sys_metadata = {} for key, (value, timestamp) in metadata.items(): if is_user_meta(db_type, key): if drop_prefixes: key = strip_user_meta_prefix(db_type, key) user_metadata[key] = value elif is_sys_meta(db_type, key): if drop_prefixes: key = strip_sys_meta_prefix(db_type, key) sys_metadata[key] = value else: title = key.replace('_', '-').title() print(' %s: %s' % (title, value)) if sys_metadata: print(' System Metadata: %s' % sys_metadata) else: print('No system metadata found in db file') if user_metadata: print(' User Metadata: %s' % user_metadata) else: print('No user metadata found in db file') if db_type == 'container': print('Sharding Metadata:') shard_type = 'root' if info['is_root'] else 'shard' print(' Type: %s' % shard_type) print(' State: %s' % info['db_state']) if info.get('shard_ranges'): print('Shard Ranges (%d):' % len(info['shard_ranges'])) for srange in info['shard_ranges']: srange = dict(srange, state_text=srange.state_text) print(' Name: %(name)s' % srange) print(' lower: %(lower)r, upper: %(upper)r' % srange) print(' Object Count: %(object_count)d, Bytes Used: ' '%(bytes_used)d, State: %(state_text)s (%(state)d)' % srange) print(' Created at: %s (%s)' % (Timestamp(srange['timestamp']).isoformat, srange['timestamp'])) print(' Meta Timestamp: %s (%s)' % (Timestamp(srange['meta_timestamp']).isoformat, srange['meta_timestamp']))
def test_strip_sys_meta_prefix(self): mt = 'sysmeta' for st in server_types: self.assertEquals(strip_sys_meta_prefix(st, 'x-%s-%s-a' % (st, mt)), 'a')
def print_obj_metadata(metadata, drop_prefixes=False): """ Print out basic info and metadata from object, as returned from :func:`swift.obj.diskfile.read_metadata`. Metadata should include the keys: name, Content-Type, and X-Timestamp. Additional metadata is displayed unmodified. :param metadata: dict of object metadata :param drop_prefixes: if True, strip "X-Object-Meta-", "X-Object-Sysmeta-", and "X-Object-Transient-Sysmeta-" when displaying User Metadata, System Metadata, and Transient System Metadata entries :raises ValueError: """ user_metadata = {} sys_metadata = {} transient_sys_metadata = {} other_metadata = {} if not metadata: raise ValueError('Metadata is None') path = metadata.pop('name', '') content_type = metadata.pop('Content-Type', '') ts = Timestamp(metadata.pop('X-Timestamp', 0)) account = container = obj = obj_hash = None if path: try: account, container, obj = path.split('/', 3)[1:] except ValueError: raise ValueError('Path is invalid for object %r' % path) else: obj_hash = hash_path(account, container, obj) print('Path: %s' % path) print(' Account: %s' % account) print(' Container: %s' % container) print(' Object: %s' % obj) print(' Object hash: %s' % obj_hash) else: print('Path: Not found in metadata') if content_type: print('Content-Type: %s' % content_type) else: print('Content-Type: Not found in metadata') if ts: print('Timestamp: %s (%s)' % (ts.isoformat, ts.internal)) else: print('Timestamp: Not found in metadata') for key, value in metadata.items(): if is_user_meta('Object', key): if drop_prefixes: key = strip_user_meta_prefix('Object', key) user_metadata[key] = value elif is_sys_meta('Object', key): if drop_prefixes: key = strip_sys_meta_prefix('Object', key) sys_metadata[key] = value elif is_object_transient_sysmeta(key): if drop_prefixes: key = strip_object_transient_sysmeta_prefix(key) transient_sys_metadata[key] = value else: other_metadata[key] = value def print_metadata(title, items): print(title) if items: for key, value in sorted(items.items()): print(' %s: %s' % (key, value)) else: print(' No metadata found') print_metadata('System Metadata:', sys_metadata) print_metadata('Transient System Metadata:', transient_sys_metadata) print_metadata('User Metadata:', user_metadata) print_metadata('Other Metadata:', other_metadata) for label, meta in [ ('Data crypto details', sys_metadata.get('X-Object-Sysmeta-Crypto-Body-Meta')), ('Metadata crypto details', transient_sys_metadata.get('X-Object-Transient-Sysmeta-Crypto-Meta')), ]: if meta is None: continue print('%s: %s' % ( label, json.dumps(load_crypto_meta(meta, b64decode=False), indent=2, sort_keys=True, separators=(',', ': '))))
def print_db_info_metadata(db_type, info, metadata, drop_prefixes=False, verbose=False): """ print out data base info/metadata based on its type :param db_type: database type, account or container :param info: dict of data base info :param metadata: dict of data base metadata :param drop_prefixes: if True, strip "X-Account-Meta-", "X-Container-Meta-", "X-Account-Sysmeta-", and "X-Container-Sysmeta-" when displaying User Metadata and System Metadata dicts """ if info is None: raise ValueError('DB info is None') if db_type not in ['container', 'account']: raise ValueError('Wrong DB type') try: account = info['account'] container = None if db_type == 'container': container = info['container'] path = '/%s/%s' % (account, container) else: path = '/%s' % account print('Path: %s' % path) print(' Account: %s' % account) if db_type == 'container': print(' Container: %s' % container) print(' Deleted: %s' % info['is_deleted']) path_hash = hash_path(account, container) if db_type == 'container': print(' Container Hash: %s' % path_hash) else: print(' Account Hash: %s' % path_hash) print('Metadata:') print(' Created at: %s (%s)' % (Timestamp(info['created_at']).isoformat, info['created_at'])) print(' Put Timestamp: %s (%s)' % (Timestamp(info['put_timestamp']).isoformat, info['put_timestamp'])) print(' Delete Timestamp: %s (%s)' % (Timestamp(info['delete_timestamp']).isoformat, info['delete_timestamp'])) print(' Status Timestamp: %s (%s)' % (Timestamp(info['status_changed_at']).isoformat, info['status_changed_at'])) if db_type == 'account': print(' Container Count: %s' % info['container_count']) print(' Object Count: %s' % info['object_count']) print(' Bytes Used: %s' % info['bytes_used']) if db_type == 'container': try: policy_name = POLICIES[info['storage_policy_index']].name except KeyError: policy_name = 'Unknown' print(' Storage Policy: %s (%s)' % ( policy_name, info['storage_policy_index'])) print(' Reported Put Timestamp: %s (%s)' % (Timestamp(info['reported_put_timestamp']).isoformat, info['reported_put_timestamp'])) print(' Reported Delete Timestamp: %s (%s)' % (Timestamp(info['reported_delete_timestamp']).isoformat, info['reported_delete_timestamp'])) print(' Reported Object Count: %s' % info['reported_object_count']) print(' Reported Bytes Used: %s' % info['reported_bytes_used']) print(' Chexor: %s' % info['hash']) print(' UUID: %s' % info['id']) except KeyError as e: raise ValueError('Info is incomplete: %s' % e) meta_prefix = 'x_' + db_type + '_' for key, value in info.items(): if key.lower().startswith(meta_prefix): title = key.replace('_', '-').title() print(' %s: %s' % (title, value)) user_metadata = {} sys_metadata = {} for key, (value, timestamp) in metadata.items(): if is_user_meta(db_type, key): if drop_prefixes: key = strip_user_meta_prefix(db_type, key) user_metadata[key] = value elif is_sys_meta(db_type, key): if drop_prefixes: key = strip_sys_meta_prefix(db_type, key) sys_metadata[key] = value else: title = key.replace('_', '-').title() print(' %s: %s' % (title, value)) if sys_metadata: print(' System Metadata: %s' % sys_metadata) else: print('No system metadata found in db file') if user_metadata: print(' User Metadata: %s' % user_metadata) else: print('No user metadata found in db file') if db_type == 'container': print('Sharding Metadata:') shard_type = 'root' if info['is_root'] else 'shard' print(' Type: %s' % shard_type) print(' State: %s' % info['db_state']) if info.get('shard_ranges'): num_shards = len(info['shard_ranges']) print('Shard Ranges (%d):' % num_shards) count_by_state = defaultdict(int) for srange in info['shard_ranges']: count_by_state[(srange.state, srange.state_text)] += 1 print(' States:') for key_state, count in sorted(count_by_state.items()): key, state = key_state print(' %9s: %s' % (state, count)) if verbose: for srange in info['shard_ranges']: srange = dict(srange, state_text=srange.state_text) print(' Name: %(name)s' % srange) print(' lower: %(lower)r, upper: %(upper)r' % srange) print(' Object Count: %(object_count)d, Bytes Used: ' '%(bytes_used)d, State: %(state_text)s (%(state)d)' % srange) print(' Created at: %s (%s)' % (Timestamp(srange['timestamp']).isoformat, srange['timestamp'])) print(' Meta Timestamp: %s (%s)' % (Timestamp(srange['meta_timestamp']).isoformat, srange['meta_timestamp'])) else: print('(Use -v/--verbose to show more Shard Ranges details)')
def print_db_info_metadata(db_type, info, metadata): """ print out data base info/metadata based on its type :param db_type: database type, account or container :param info: dict of data base info :param metadata: dict of data base metadata """ if info is None: raise ValueError("DB info is None") if db_type not in ["container", "account"]: raise ValueError("Wrong DB type") try: account = info["account"] container = None if db_type == "container": container = info["container"] path = "/%s/%s" % (account, container) else: path = "/%s" % account print "Path: %s" % path print " Account: %s" % account if db_type == "container": print " Container: %s" % container path_hash = hash_path(account, container) if db_type == "container": print " Container Hash: %s" % path_hash else: print " Account Hash: %s" % path_hash print "Metadata:" print (" Created at: %s (%s)" % (datetime.utcfromtimestamp(float(info["created_at"])), info["created_at"])) print ( " Put Timestamp: %s (%s)" % (datetime.utcfromtimestamp(float(info["put_timestamp"])), info["put_timestamp"]) ) print ( " Delete Timestamp: %s (%s)" % (datetime.utcfromtimestamp(float(info["delete_timestamp"])), info["delete_timestamp"]) ) if db_type == "account": print " Container Count: %s" % info["container_count"] print " Object Count: %s" % info["object_count"] print " Bytes Used: %s" % info["bytes_used"] if db_type == "container": print ( " Reported Put Timestamp: %s (%s)" % (datetime.utcfromtimestamp(float(info["reported_put_timestamp"])), info["reported_put_timestamp"]) ) print ( " Reported Delete Timestamp: %s (%s)" % ( datetime.utcfromtimestamp(float(info["reported_delete_timestamp"])), info["reported_delete_timestamp"], ) ) print " Reported Object Count: %s" % info["reported_object_count"] print " Reported Bytes Used: %s" % info["reported_bytes_used"] print " Chexor: %s" % info["hash"] print " UUID: %s" % info["id"] except KeyError: raise ValueError("Info is incomplete") meta_prefix = "x_" + db_type + "_" for key, value in info.iteritems(): if key.lower().startswith(meta_prefix): title = key.replace("_", "-").title() print " %s: %s" % (title, value) user_metadata = {} sys_metadata = {} for key, (value, timestamp) in metadata.iteritems(): if is_user_meta(db_type, key): user_metadata[strip_user_meta_prefix(db_type, key)] = value elif is_sys_meta(db_type, key): sys_metadata[strip_sys_meta_prefix(db_type, key)] = value else: title = key.replace("_", "-").title() print " %s: %s" % (title, value) if sys_metadata: print " System Metadata: %s" % sys_metadata else: print "No system metadata found in db file" if user_metadata: print " User Metadata: %s" % user_metadata else: print "No user metadata found in db file"