def test_chexor(self):
broker = AccountBroker(':memory:', account='a')
broker.initialize(Timestamp('1').internal)
broker.put_container('a', Timestamp(1).internal,
Timestamp(0).internal, 0, 0,
POLICIES.default.idx)
broker.put_container('b', Timestamp(2).internal,
Timestamp(0).internal, 0, 0,
POLICIES.default.idx)
hasha = hashlib.md5(
'%s-%s' % ('a', "%s-%s-%s-%s" % (
Timestamp(1).internal, Timestamp(0).internal, 0, 0))
).digest()
hashb = hashlib.md5(
'%s-%s' % ('b', "%s-%s-%s-%s" % (
Timestamp(2).internal, Timestamp(0).internal, 0, 0))
).digest()
hashc = \
''.join(('%02x' % (ord(a) ^ ord(b)) for a, b in zip(hasha, hashb)))
self.assertEqual(broker.get_info()['hash'], hashc)
broker.put_container('b', Timestamp(3).internal,
Timestamp(0).internal, 0, 0,
POLICIES.default.idx)
hashb = hashlib.md5(
'%s-%s' % ('b', "%s-%s-%s-%s" % (
Timestamp(3).internal, Timestamp(0).internal, 0, 0))
).digest()
hashc = \
''.join(('%02x' % (ord(a) ^ ord(b)) for a, b in zip(hasha, hashb)))
self.assertEqual(broker.get_info()['hash'], hashc)
def test_chexor(self):
broker = AccountBroker(':memory:', account='a')
broker.initialize(Timestamp('1').internal)
broker.put_container('a',
Timestamp(1).internal,
Timestamp(0).internal, 0, 0, POLICIES.default.idx)
broker.put_container('b',
Timestamp(2).internal,
Timestamp(0).internal, 0, 0, POLICIES.default.idx)
hasha = hashlib.md5(
'%s-%s' %
('a', "%s-%s-%s-%s" %
(Timestamp(1).internal, Timestamp(0).internal, 0, 0))).digest()
hashb = hashlib.md5(
'%s-%s' %
('b', "%s-%s-%s-%s" %
(Timestamp(2).internal, Timestamp(0).internal, 0, 0))).digest()
hashc = \
''.join(('%02x' % (ord(a) ^ ord(b)) for a, b in zip(hasha, hashb)))
self.assertEqual(broker.get_info()['hash'], hashc)
broker.put_container('b',
Timestamp(3).internal,
Timestamp(0).internal, 0, 0, POLICIES.default.idx)
hashb = hashlib.md5(
'%s-%s' %
('b', "%s-%s-%s-%s" %
(Timestamp(3).internal, Timestamp(0).internal, 0, 0))).digest()
hashc = \
''.join(('%02x' % (ord(a) ^ ord(b)) for a, b in zip(hasha, hashb)))
self.assertEqual(broker.get_info()['hash'], hashc)
def print_info(db_type, db_file, swift_dir='/etc/swift'):
if db_type not in ('account', 'container'):
print "Unrecognized DB type: internal error"
raise InfoSystemExit()
if not os.path.exists(db_file) or not db_file.endswith('.db'):
print "DB file doesn't exist"
raise InfoSystemExit()
if not db_file.startswith(('/', './')):
db_file = './' + db_file # don't break if the bare db file is given
if db_type == 'account':
broker = AccountBroker(db_file)
datadir = ABDATADIR
else:
broker = ContainerBroker(db_file)
datadir = CBDATADIR
try:
info = broker.get_info()
except sqlite3.OperationalError as err:
if 'no such table' in str(err):
print "Does not appear to be a DB of type \"%s\": %s" % (
db_type, db_file)
raise InfoSystemExit()
raise
account = info['account']
container = info['container'] if db_type == 'container' else None
print_db_info_metadata(db_type, info, broker.metadata)
try:
ring = Ring(swift_dir, ring_name=db_type)
except Exception:
ring = None
else:
print_ring_locations(ring, datadir, account, container)
def _gen_account_stat(self):
stat = db_stats_collector.AccountStatsCollector(self.conf)
output_data = set()
for i in range(10):
account_db = AccountBroker("%s/stats-201001010%s-%s.db" %
(self.accounts, i, uuid.uuid4().hex),
account='test_acc_%s' % i)
account_db.initialize()
account_db.put_container('test_container', time.time(),
None, 10, 1000)
# this will "commit" the data
account_db.get_info()
output_data.add('''"test_acc_%s",1,10,1000''' % i),
self.assertEqual(len(output_data), 10)
return stat, output_data
def test_merge_items(self):
broker1 = AccountBroker(':memory:', account='a')
broker1.initialize(Timestamp('1').internal)
broker2 = AccountBroker(':memory:', account='a')
broker2.initialize(Timestamp('1').internal)
broker1.put_container('a',
Timestamp(1).internal, 0, 0, 0,
POLICIES.default.idx)
broker1.put_container('b',
Timestamp(2).internal, 0, 0, 0,
POLICIES.default.idx)
id = broker1.get_info()['id']
broker2.merge_items(
broker1.get_items_since(broker2.get_sync(id), 1000), id)
items = broker2.get_items_since(-1, 1000)
self.assertEqual(len(items), 2)
self.assertEqual(['a', 'b'], sorted([rec['name'] for rec in items]))
broker1.put_container('c',
Timestamp(3).internal, 0, 0, 0,
POLICIES.default.idx)
broker2.merge_items(
broker1.get_items_since(broker2.get_sync(id), 1000), id)
items = broker2.get_items_since(-1, 1000)
self.assertEqual(len(items), 3)
self.assertEqual(['a', 'b', 'c'],
sorted([rec['name'] for rec in items]))
def _gen_account_stat(self):
stat = db_stats_collector.AccountStatsCollector(self.conf)
output_data = set()
for i in range(10):
account_db = AccountBroker("%s/stats-201001010%s-%s.db" %
(self.accounts, i, uuid.uuid4().hex),
account='test_acc_%s' % i)
account_db.initialize()
account_db.put_container('test_container', time.time(), None, 10,
1000, 1)
# this will "commit" the data
account_db.get_info()
output_data.add('''"test_acc_%s",1,10,1000''' % i),
self.assertEqual(len(output_data), 10)
return stat, output_data
def print_info(db_type, db_file, swift_dir="/etc/swift"):
if db_type not in ("account", "container"):
print "Unrecognized DB type: internal error"
raise InfoSystemExit()
if not os.path.exists(db_file) or not db_file.endswith(".db"):
print "DB file doesn't exist"
raise InfoSystemExit()
if not db_file.startswith(("/", "./")):
db_file = "./" + db_file # don't break if the bare db file is given
if db_type == "account":
broker = AccountBroker(db_file)
datadir = ABDATADIR
else:
broker = ContainerBroker(db_file)
datadir = CBDATADIR
try:
info = broker.get_info()
except sqlite3.OperationalError as err:
if "no such table" in str(err):
print 'Does not appear to be a DB of type "%s": %s' % (db_type, db_file)
raise InfoSystemExit()
raise
account = info["account"]
container = info["container"] if db_type == "container" else None
print_db_info_metadata(db_type, info, broker.metadata)
try:
ring = Ring(swift_dir, ring_name=db_type)
except Exception:
ring = None
else:
print_ring_locations(ring, datadir, account, container)
def test_get_info(self):
# Test AccountBroker.get_info
broker = AccountBroker(':memory:', account='test1')
broker.initialize(normalize_timestamp('1'))
info = broker.get_info()
self.assertEqual(info['account'], 'test1')
self.assertEqual(info['hash'], '00000000000000000000000000000000')
info = broker.get_info()
self.assertEqual(info['container_count'], 0)
broker.put_container('c1', normalize_timestamp(time()), 0, 0, 0)
info = broker.get_info()
self.assertEqual(info['container_count'], 1)
sleep(.00001)
broker.put_container('c2', normalize_timestamp(time()), 0, 0, 0)
info = broker.get_info()
self.assertEqual(info['container_count'], 2)
sleep(.00001)
broker.put_container('c2', normalize_timestamp(time()), 0, 0, 0)
info = broker.get_info()
self.assertEqual(info['container_count'], 2)
sleep(.00001)
broker.put_container('c1', 0, normalize_timestamp(time()), 0, 0)
info = broker.get_info()
self.assertEqual(info['container_count'], 1)
sleep(.00001)
broker.put_container('c2', 0, normalize_timestamp(time()), 0, 0)
info = broker.get_info()
self.assertEqual(info['container_count'], 0)
def account_audit(self, path):
"""
Audits the given account path
:param path: the path to an account db
"""
start_time = time.time()
try:
broker = AccountBroker(path)
if not broker.is_deleted():
broker.get_info()
self.logger.increment('passes')
self.account_passes += 1
self.logger.debug(_('Audit passed for %s') % broker)
except (Exception, Timeout):
self.logger.increment('failures')
self.account_failures += 1
self.logger.exception(_('ERROR Could not get account info %s'),
path)
self.logger.timing_since('timing', start_time)
def test_chexor(self):
broker = AccountBroker(':memory:', account='a')
broker.initialize(normalize_timestamp('1'))
broker.put_container('a', normalize_timestamp(1),
normalize_timestamp(0), 0, 0)
broker.put_container('b', normalize_timestamp(2),
normalize_timestamp(0), 0, 0)
hasha = hashlib.md5(
'%s-%s' % ('a', '0000000001.00000-0000000000.00000-0-0')).digest()
hashb = hashlib.md5(
'%s-%s' % ('b', '0000000002.00000-0000000000.00000-0-0')).digest()
hashc = \
''.join(('%02x' % (ord(a) ^ ord(b)) for a, b in zip(hasha, hashb)))
self.assertEqual(broker.get_info()['hash'], hashc)
broker.put_container('b', normalize_timestamp(3),
normalize_timestamp(0), 0, 0)
hashb = hashlib.md5(
'%s-%s' % ('b', '0000000003.00000-0000000000.00000-0-0')).digest()
hashc = \
''.join(('%02x' % (ord(a) ^ ord(b)) for a, b in zip(hasha, hashb)))
self.assertEqual(broker.get_info()['hash'], hashc)
def test_delete_db_status(self):
ts = (Timestamp(t).internal for t in itertools.count(int(time())))
start = ts.next()
broker = AccountBroker(':memory:', account='a')
broker.initialize(start)
info = broker.get_info()
self.assertEqual(info['put_timestamp'], Timestamp(start).internal)
self.assert_(Timestamp(info['created_at']) >= start)
self.assertEqual(info['delete_timestamp'], '0')
if self.__class__ == TestAccountBrokerBeforeMetadata:
self.assertEqual(info['status_changed_at'], '0')
else:
self.assertEqual(info['status_changed_at'],
Timestamp(start).internal)
# delete it
delete_timestamp = ts.next()
broker.delete_db(delete_timestamp)
info = broker.get_info()
self.assertEqual(info['put_timestamp'], Timestamp(start).internal)
self.assert_(Timestamp(info['created_at']) >= start)
self.assertEqual(info['delete_timestamp'], delete_timestamp)
self.assertEqual(info['status_changed_at'], delete_timestamp)
def test_delete_db_status(self):
ts = (Timestamp(t).internal for t in itertools.count(int(time())))
start = ts.next()
broker = AccountBroker(':memory:', account='a')
broker.initialize(start)
info = broker.get_info()
self.assertEqual(info['put_timestamp'], Timestamp(start).internal)
self.assert_(Timestamp(info['created_at']) >= start)
self.assertEqual(info['delete_timestamp'], '0')
if self.__class__ == TestAccountBrokerBeforeMetadata:
self.assertEqual(info['status_changed_at'], '0')
else:
self.assertEqual(info['status_changed_at'],
Timestamp(start).internal)
# delete it
delete_timestamp = ts.next()
broker.delete_db(delete_timestamp)
info = broker.get_info()
self.assertEqual(info['put_timestamp'], Timestamp(start).internal)
self.assert_(Timestamp(info['created_at']) >= start)
self.assertEqual(info['delete_timestamp'], delete_timestamp)
self.assertEqual(info['status_changed_at'], delete_timestamp)
def get_data(self, db_path):
"""
Data for generated csv has the following columns:
Account Hash, Container Count, Object Count, Bytes Used
:raises sqlite3.Error: does not catch errors connecting to db
"""
line_data = None
broker = AccountBroker(db_path)
if not broker.is_deleted():
info = broker.get_info()
line_data = '"%s",%d,%d,%d\n' % (
info['account'], info['container_count'], info['object_count'],
info['bytes_used'])
return line_data
def test_get_info(self):
# Test AccountBroker.get_info
broker = AccountBroker(':memory:', account='test1')
broker.initialize(Timestamp('1').internal)
info = broker.get_info()
self.assertEqual(info['account'], 'test1')
self.assertEqual(info['hash'], '00000000000000000000000000000000')
self.assertEqual(info['put_timestamp'], Timestamp(1).internal)
self.assertEqual(info['delete_timestamp'], '0')
if self.__class__ == TestAccountBrokerBeforeMetadata:
self.assertEqual(info['status_changed_at'], '0')
else:
self.assertEqual(info['status_changed_at'], Timestamp(1).internal)
info = broker.get_info()
self.assertEqual(info['container_count'], 0)
broker.put_container('c1',
Timestamp(time()).internal, 0, 0, 0,
POLICIES.default.idx)
info = broker.get_info()
self.assertEqual(info['container_count'], 1)
sleep(.00001)
broker.put_container('c2',
Timestamp(time()).internal, 0, 0, 0,
POLICIES.default.idx)
info = broker.get_info()
self.assertEqual(info['container_count'], 2)
sleep(.00001)
broker.put_container('c2',
Timestamp(time()).internal, 0, 0, 0,
POLICIES.default.idx)
info = broker.get_info()
self.assertEqual(info['container_count'], 2)
sleep(.00001)
broker.put_container('c1', 0,
Timestamp(time()).internal, 0, 0,
POLICIES.default.idx)
info = broker.get_info()
self.assertEqual(info['container_count'], 1)
sleep(.00001)
broker.put_container('c2', 0,
Timestamp(time()).internal, 0, 0,
POLICIES.default.idx)
info = broker.get_info()
self.assertEqual(info['container_count'], 0)
def test_chexor(self):
broker = AccountBroker(':memory:', account='a')
broker.initialize(normalize_timestamp('1'))
broker.put_container('a', normalize_timestamp(1),
normalize_timestamp(0), 0, 0)
broker.put_container('b', normalize_timestamp(2),
normalize_timestamp(0), 0, 0)
hasha = hashlib.md5(
'%s-%s' % ('a', '0000000001.00000-0000000000.00000-0-0')
).digest()
hashb = hashlib.md5(
'%s-%s' % ('b', '0000000002.00000-0000000000.00000-0-0')
).digest()
hashc = \
''.join(('%02x' % (ord(a) ^ ord(b)) for a, b in zip(hasha, hashb)))
self.assertEqual(broker.get_info()['hash'], hashc)
broker.put_container('b', normalize_timestamp(3),
normalize_timestamp(0), 0, 0)
hashb = hashlib.md5(
'%s-%s' % ('b', '0000000003.00000-0000000000.00000-0-0')
).digest()
hashc = \
''.join(('%02x' % (ord(a) ^ ord(b)) for a, b in zip(hasha, hashb)))
self.assertEqual(broker.get_info()['hash'], hashc)
def get_data(self, db_path):
"""
Data for generated csv has the following columns:
Account Hash, Container Count, Object Count, Bytes Used
:raises sqlite3.Error: does not catch errors connecting to db
"""
line_data = None
broker = AccountBroker(db_path)
if not broker.is_deleted():
info = broker.get_info()
line_data = '"%s",%d,%d,%d\n' % (info['account'],
info['container_count'],
info['object_count'],
info['bytes_used'])
return line_data
def print_info(db_type,
db_file,
swift_dir='/etc/swift',
stale_reads_ok=False,
drop_prefixes=False,
verbose=False):
if db_type not in ('account', 'container'):
print("Unrecognized DB type: internal error")
raise InfoSystemExit()
if not os.path.exists(db_file) or not db_file.endswith('.db'):
print("DB file doesn't exist")
raise InfoSystemExit()
if not db_file.startswith(('/', './')):
db_file = './' + db_file # don't break if the bare db file is given
if db_type == 'account':
broker = AccountBroker(db_file, stale_reads_ok=stale_reads_ok)
datadir = ABDATADIR
else:
broker = ContainerBroker(db_file, stale_reads_ok=stale_reads_ok)
datadir = CBDATADIR
try:
info = broker.get_info()
except sqlite3.OperationalError as err:
if 'no such table' in str(err):
print("Does not appear to be a DB of type \"%s\": %s" %
(db_type, db_file))
raise InfoSystemExit()
raise
account = info['account']
container = None
info['is_deleted'] = broker.is_deleted()
if db_type == 'container':
container = info['container']
info['is_root'] = broker.is_root_container()
sranges = broker.get_shard_ranges()
if sranges:
info['shard_ranges'] = sranges
print_db_info_metadata(db_type, info, broker.metadata, drop_prefixes,
verbose)
try:
ring = Ring(swift_dir, ring_name=db_type)
except Exception:
ring = None
else:
print_ring_locations(ring, datadir, account, container)
def test_get_info(self):
# Test AccountBroker.get_info
broker = AccountBroker(':memory:', account='test1')
broker.initialize(Timestamp('1').internal)
info = broker.get_info()
self.assertEqual(info['account'], 'test1')
self.assertEqual(info['hash'], '00000000000000000000000000000000')
self.assertEqual(info['put_timestamp'], Timestamp(1).internal)
self.assertEqual(info['delete_timestamp'], '0')
if self.__class__ == TestAccountBrokerBeforeMetadata:
self.assertEqual(info['status_changed_at'], '0')
else:
self.assertEqual(info['status_changed_at'], Timestamp(1).internal)
info = broker.get_info()
self.assertEqual(info['container_count'], 0)
broker.put_container('c1', Timestamp(time()).internal, 0, 0, 0,
POLICIES.default.idx)
info = broker.get_info()
self.assertEqual(info['container_count'], 1)
sleep(.00001)
broker.put_container('c2', Timestamp(time()).internal, 0, 0, 0,
POLICIES.default.idx)
info = broker.get_info()
self.assertEqual(info['container_count'], 2)
sleep(.00001)
broker.put_container('c2', Timestamp(time()).internal, 0, 0, 0,
POLICIES.default.idx)
info = broker.get_info()
self.assertEqual(info['container_count'], 2)
sleep(.00001)
broker.put_container('c1', 0, Timestamp(time()).internal, 0, 0,
POLICIES.default.idx)
info = broker.get_info()
self.assertEqual(info['container_count'], 1)
sleep(.00001)
broker.put_container('c2', 0, Timestamp(time()).internal, 0, 0,
POLICIES.default.idx)
info = broker.get_info()
self.assertEqual(info['container_count'], 0)
def test_merge_items(self):
broker1 = AccountBroker(':memory:', account='a')
broker1.initialize(normalize_timestamp('1'))
broker2 = AccountBroker(':memory:', account='a')
broker2.initialize(normalize_timestamp('1'))
broker1.put_container('a', normalize_timestamp(1), 0, 0, 0)
broker1.put_container('b', normalize_timestamp(2), 0, 0, 0)
id = broker1.get_info()['id']
broker2.merge_items(broker1.get_items_since(
broker2.get_sync(id), 1000), id)
items = broker2.get_items_since(-1, 1000)
self.assertEqual(len(items), 2)
self.assertEqual(['a', 'b'], sorted([rec['name'] for rec in items]))
broker1.put_container('c', normalize_timestamp(3), 0, 0, 0)
broker2.merge_items(broker1.get_items_since(
broker2.get_sync(id), 1000), id)
items = broker2.get_items_since(-1, 1000)
self.assertEqual(len(items), 3)
self.assertEqual(['a', 'b', 'c'],
sorted([rec['name'] for rec in items]))
def test_merge_items(self):
broker1 = AccountBroker(':memory:', account='a')
broker1.initialize(normalize_timestamp('1'))
broker2 = AccountBroker(':memory:', account='a')
broker2.initialize(normalize_timestamp('1'))
broker1.put_container('a', normalize_timestamp(1), 0, 0, 0)
broker1.put_container('b', normalize_timestamp(2), 0, 0, 0)
id = broker1.get_info()['id']
broker2.merge_items(
broker1.get_items_since(broker2.get_sync(id), 1000), id)
items = broker2.get_items_since(-1, 1000)
self.assertEquals(len(items), 2)
self.assertEquals(['a', 'b'], sorted([rec['name'] for rec in items]))
broker1.put_container('c', normalize_timestamp(3), 0, 0, 0)
broker2.merge_items(
broker1.get_items_since(broker2.get_sync(id), 1000), id)
items = broker2.get_items_since(-1, 1000)
self.assertEquals(len(items), 3)
self.assertEquals(['a', 'b', 'c'],
sorted([rec['name'] for rec in items]))
def account_crawl(self, path):
"""
Crawls the given account path
:param path: the path to an account db
"""
#start_time = time.time()
metaDict = {}
try:
broker = AccountBroker(path)
if not broker.is_deleted():
#reportedTime = broker.get_info()['put_timestamp']
#if normalize_timestamp(self.crawled_time) <
#reportedTime < normalize_timestamp(start_time):
metaDict = broker.get_info()
metaDict.update((key, value)
for key, (value, timestamp) in
broker.metadata.iteritems() if value != '')
except (Exception, Timeout):
self.logger.increment('failures')
return metaDict
def account_crawl(self, path):
"""
Crawls the given account path
:param path: the path to an account db
"""
#start_time = time.time()
metaDict = {}
try:
broker = AccountBroker(path)
if not broker.is_deleted():
#reportedTime = broker.get_info()['put_timestamp']
#if normalize_timestamp(self.crawled_time) <
#reportedTime < normalize_timestamp(start_time):
metaDict = broker.get_info()
metaDict.update(
(key, value)
for key, (value, timestamp) in broker.metadata.iteritems()
if value != '')
except (Exception, Timeout):
self.logger.increment('failures')
return metaDict
def print_info(db_type, db_file, swift_dir='/etc/swift', stale_reads_ok=False,
drop_prefixes=False):
if db_type not in ('account', 'container'):
print("Unrecognized DB type: internal error")
raise InfoSystemExit()
if not os.path.exists(db_file) or not db_file.endswith('.db'):
print("DB file doesn't exist")
raise InfoSystemExit()
if not db_file.startswith(('/', './')):
db_file = './' + db_file # don't break if the bare db file is given
if db_type == 'account':
broker = AccountBroker(db_file, stale_reads_ok=stale_reads_ok)
datadir = ABDATADIR
else:
broker = ContainerBroker(db_file, stale_reads_ok=stale_reads_ok)
datadir = CBDATADIR
try:
info = broker.get_info()
except sqlite3.OperationalError as err:
if 'no such table' in str(err):
print("Does not appear to be a DB of type \"%s\": %s"
% (db_type, db_file))
raise InfoSystemExit()
raise
account = info['account']
container = None
if db_type == 'container':
container = info['container']
info['is_root'] = broker.is_root_container()
sranges = broker.get_shard_ranges()
if sranges:
info['shard_ranges'] = sranges
print_db_info_metadata(db_type, info, broker.metadata, drop_prefixes)
try:
ring = Ring(swift_dir, ring_name=db_type)
except Exception:
ring = None
else:
print_ring_locations(ring, datadir, account, container)