def test_half_upgraded_database(self, tempdir):
db_path = os.path.join(tempdir, 'account.db')
ts = itertools.count()
ts = (Timestamp(t).internal for t in itertools.count(int(time())))
broker = AccountBroker(db_path, account='a')
broker.initialize(ts.next())
self.assertTrue(broker.empty())
# add a container (to pending file)
broker.put_container('c', ts.next(), 0, 0, 0, POLICIES.default.idx)
real_get = broker.get
called = []
@contextmanager
def mock_get():
with real_get() as conn:
def mock_executescript(script):
if called:
raise Exception('kaboom!')
called.append(script)
conn.executescript = mock_executescript
yield conn
broker.get = mock_get
try:
broker._commit_puts()
except Exception:
pass
else:
self.fail('mock exception was not raised')
self.assertEqual(len(called), 1)
self.assert_('CREATE TABLE policy_stat' in called[0])
# nothing was commited
broker = AccountBroker(db_path, account='a')
with broker.get() as conn:
try:
conn.execute('SELECT * FROM policy_stat')
except sqlite3.OperationalError as err:
self.assert_('no such table: policy_stat' in str(err))
else:
self.fail('half upgraded database!')
container_count = conn.execute(
'SELECT count(*) FROM container').fetchone()[0]
self.assertEqual(container_count, 0)
# try again to commit puts
self.assertFalse(broker.empty())
# full migration successful
with broker.get() as conn:
conn.execute('SELECT * FROM policy_stat')
conn.execute('SELECT storage_policy_index FROM container')
def test_merge_items(self):
broker1 = AccountBroker(':memory:', account='a')
broker1.initialize(Timestamp('1').internal)
broker2 = AccountBroker(':memory:', account='a')
broker2.initialize(Timestamp('1').internal)
broker1.put_container('a',
Timestamp(1).internal, 0, 0, 0,
POLICIES.default.idx)
broker1.put_container('b',
Timestamp(2).internal, 0, 0, 0,
POLICIES.default.idx)
id = broker1.get_info()['id']
broker2.merge_items(
broker1.get_items_since(broker2.get_sync(id), 1000), id)
items = broker2.get_items_since(-1, 1000)
self.assertEqual(len(items), 2)
self.assertEqual(['a', 'b'], sorted([rec['name'] for rec in items]))
broker1.put_container('c',
Timestamp(3).internal, 0, 0, 0,
POLICIES.default.idx)
broker2.merge_items(
broker1.get_items_since(broker2.get_sync(id), 1000), id)
items = broker2.get_items_since(-1, 1000)
self.assertEqual(len(items), 3)
self.assertEqual(['a', 'b', 'c'],
sorted([rec['name'] for rec in items]))
def test_is_status_deleted(self):
# Test AccountBroker.is_status_deleted
broker1 = AccountBroker(':memory:', account='a')
broker1.initialize(Timestamp(time()).internal)
self.assert_(not broker1.is_status_deleted())
broker1.delete_db(Timestamp(time()).internal)
self.assert_(broker1.is_status_deleted())
broker2 = AccountBroker(':memory:', account='a')
broker2.initialize(Timestamp(time()).internal)
# Set delete_timestamp greater than put_timestamp
broker2.merge_timestamps(time(),
Timestamp(time()).internal,
Timestamp(time() + 999).internal)
self.assert_(broker2.is_status_deleted())
def _get_account_broker(self, drive, part, account, **kwargs):
hsh = hash_path(account)
db_dir = storage_directory(DATADIR, part, hsh)
db_path = os.path.join(self.root, drive, db_dir, hsh + '.db')
kwargs.setdefault('account', account)
kwargs.setdefault('logger', self.logger)
return AccountBroker(db_path, **kwargs)
def tearDown(self):
AccountBroker.create_account_stat_table = \
self._imported_create_account_stat_table
broker = AccountBroker(':memory:', account='a')
broker.initialize(normalize_timestamp('1'))
with broker.get() as conn:
conn.execute('SELECT metadata FROM account_stat')
def account_audit(self, path):
"""
Audits the given account path
:param path: the path to an account db
"""
start_time = time.time()
try:
broker = AccountBroker(path)
if not broker.is_deleted():
self.validate_per_policy_counts(broker)
self.logger.increment('passes')
self.account_passes += 1
self.logger.debug('Audit passed for %s' % broker)
except InvalidAccountInfo as e:
self.logger.increment('failures')
self.account_failures += 1
self.logger.error(
_('Audit Failed for %(path)s: %(err)s'),
{'path': path, 'err': str(e)})
except (Exception, Timeout):
self.logger.increment('failures')
self.account_failures += 1
self.logger.exception(_('ERROR Could not get account info %s'),
path)
self.logger.timing_since('timing', start_time)
def main(self):
result = []
dir_list = self.findAccDir()
for _list in dir_list:
for x in os.listdir(_list):
y = self.retreive_dir(_list + '/' + x)
db_files = self.get_accountDB_path(y)
for db_file in db_files:
try:
account_stat = AccountBroker(db_file).get_info()
conv_row = {}
for key in account_stat:
if key == 'bytes_used':
conv_row[key] = "%i" % account_stat[key]
elif key == 'account':
conv_row['db_file'] = "%s" % db_file
else:
conv_row[key] = "%s" % account_stat[key]
result.append("\"%s\": %s" %
(account_stat['account'], conv_row))
except sqlite3.OperationalError as err:
if 'no such table' in str(err):
print "Does not appear to be a DB of type \"%s\": %s" % (
db_type, db_file)
raise
raise
result = "{" + (",".join(result)) + "}"
print result.replace("\'", "\"")
def test_delete_container(self):
# Test AccountBroker.delete_container
broker = AccountBroker(':memory:', account='a')
broker.initialize(Timestamp('1').internal)
broker.put_container('o',
Timestamp(time()).internal, 0, 0, 0,
POLICIES.default.idx)
with broker.get() as conn:
self.assertEqual(
conn.execute("SELECT count(*) FROM container "
"WHERE deleted = 0").fetchone()[0], 1)
self.assertEqual(
conn.execute("SELECT count(*) FROM container "
"WHERE deleted = 1").fetchone()[0], 0)
sleep(.00001)
broker.put_container('o', 0,
Timestamp(time()).internal, 0, 0,
POLICIES.default.idx)
with broker.get() as conn:
self.assertEqual(
conn.execute("SELECT count(*) FROM container "
"WHERE deleted = 0").fetchone()[0], 0)
self.assertEqual(
conn.execute("SELECT count(*) FROM container "
"WHERE deleted = 1").fetchone()[0], 1)
def test_policy_table_migration(self, tempdir):
db_path = os.path.join(tempdir, 'account.db')
# first init an acct DB without the policy_stat table present
broker = AccountBroker(db_path, account='a')
broker.initialize(Timestamp('1').internal)
with broker.get() as conn:
try:
conn.execute('''
SELECT * FROM policy_stat
''').fetchone()[0]
except sqlite3.OperationalError as err:
# confirm that the table really isn't there
self.assert_('no such table: policy_stat' in str(err))
else:
self.fail('broker did not raise sqlite3.OperationalError '
'trying to select from policy_stat table!')
# make sure we can HEAD this thing w/o the table
stats = broker.get_policy_stats()
self.assertEqual(len(stats), 0)
# now do a PUT to create the table
broker.put_container('o',
Timestamp(time()).internal, 0, 0, 0,
POLICIES.default.idx)
broker._commit_puts_stale_ok()
# now confirm that the table was created
with broker.get() as conn:
conn.execute('SELECT * FROM policy_stat')
stats = broker.get_policy_stats()
self.assertEqual(len(stats), 1)
def test_policy_stats_tracking(self):
ts = (Timestamp(t).internal for t in itertools.count(int(time())))
broker = AccountBroker(':memory:', account='a')
broker.initialize(ts.next())
# policy 0
broker.put_container('con1', ts.next(), 0, 12, 2798641, 0)
broker.put_container('con1', ts.next(), 0, 13, 8156441, 0)
# policy 1
broker.put_container('con2', ts.next(), 0, 7, 5751991, 1)
broker.put_container('con2', ts.next(), 0, 8, 6085379, 1)
stats = broker.get_policy_stats()
self.assertEqual(len(stats), 2)
self.assertEqual(stats[0]['object_count'], 13)
self.assertEqual(stats[0]['bytes_used'], 8156441)
self.assertEqual(stats[1]['object_count'], 8)
self.assertEqual(stats[1]['bytes_used'], 6085379)
# Break encapsulation here to make sure that there's only 2 rows in
# the stats table. It's possible that there could be 4 rows (one per
# put_container) but that they came out in the right order so that
# get_policy_stats() collapsed them down to the right number. To prove
# that's not so, we have to go peek at the broker's internals.
with broker.get() as conn:
nrows = conn.execute(
"SELECT COUNT(*) FROM policy_stat").fetchall()[0][0]
self.assertEqual(nrows, 2)
def print_info(db_type, db_file, swift_dir='/etc/swift'):
if db_type not in ('account', 'container'):
print "Unrecognized DB type: internal error"
raise InfoSystemExit()
if not os.path.exists(db_file) or not db_file.endswith('.db'):
print "DB file doesn't exist"
raise InfoSystemExit()
if not db_file.startswith(('/', './')):
db_file = './' + db_file # don't break if the bare db file is given
if db_type == 'account':
broker = AccountBroker(db_file)
datadir = ABDATADIR
else:
broker = ContainerBroker(db_file)
datadir = CBDATADIR
try:
info = broker.get_info()
except sqlite3.OperationalError as err:
if 'no such table' in str(err):
print "Does not appear to be a DB of type \"%s\": %s" % (
db_type, db_file)
raise InfoSystemExit()
raise
account = info['account']
container = info['container'] if db_type == 'container' else None
print_db_info_metadata(db_type, info, broker.metadata)
try:
ring = Ring(swift_dir, ring_name=db_type)
except Exception:
ring = None
else:
print_ring_locations(ring, datadir, account, container)
def test_load_old_pending_puts(self):
# pending puts from pre-storage-policy account brokers won't contain
# the storage policy index
tempdir = mkdtemp()
broker_path = os.path.join(tempdir, 'test-load-old.db')
try:
broker = AccountBroker(broker_path, account='real')
broker.initialize(Timestamp(1).internal)
with open(broker_path + '.pending', 'a+b') as pending:
pending.write(':')
pending.write(
pickle.dumps(
# name, put_timestamp, delete_timestamp, object_count,
# bytes_used, deleted
('oldcon', Timestamp(200).internal,
Timestamp(0).internal, 896, 9216695, 0
)).encode('base64'))
broker._commit_puts()
with broker.get() as conn:
results = list(
conn.execute('''
SELECT name, storage_policy_index FROM container
'''))
self.assertEqual(len(results), 1)
self.assertEqual(dict(results[0]), {
'name': 'oldcon',
'storage_policy_index': 0
})
finally:
rmtree(tempdir)
def test_get_info(self):
# Test AccountBroker.get_info
broker = AccountBroker(':memory:', account='test1')
broker.initialize(normalize_timestamp('1'))
info = broker.get_info()
self.assertEqual(info['account'], 'test1')
self.assertEqual(info['hash'], '00000000000000000000000000000000')
info = broker.get_info()
self.assertEqual(info['container_count'], 0)
broker.put_container('c1', normalize_timestamp(time()), 0, 0, 0)
info = broker.get_info()
self.assertEqual(info['container_count'], 1)
sleep(.00001)
broker.put_container('c2', normalize_timestamp(time()), 0, 0, 0)
info = broker.get_info()
self.assertEqual(info['container_count'], 2)
sleep(.00001)
broker.put_container('c2', normalize_timestamp(time()), 0, 0, 0)
info = broker.get_info()
self.assertEqual(info['container_count'], 2)
sleep(.00001)
broker.put_container('c1', 0, normalize_timestamp(time()), 0, 0)
info = broker.get_info()
self.assertEqual(info['container_count'], 1)
sleep(.00001)
broker.put_container('c2', 0, normalize_timestamp(time()), 0, 0)
info = broker.get_info()
self.assertEqual(info['container_count'], 0)
def test_chexor(self):
broker = AccountBroker(':memory:', account='a')
broker.initialize(Timestamp('1').internal)
broker.put_container('a',
Timestamp(1).internal,
Timestamp(0).internal, 0, 0, POLICIES.default.idx)
broker.put_container('b',
Timestamp(2).internal,
Timestamp(0).internal, 0, 0, POLICIES.default.idx)
hasha = hashlib.md5(
'%s-%s' %
('a', "%s-%s-%s-%s" %
(Timestamp(1).internal, Timestamp(0).internal, 0, 0))).digest()
hashb = hashlib.md5(
'%s-%s' %
('b', "%s-%s-%s-%s" %
(Timestamp(2).internal, Timestamp(0).internal, 0, 0))).digest()
hashc = \
''.join(('%02x' % (ord(a) ^ ord(b)) for a, b in zip(hasha, hashb)))
self.assertEqual(broker.get_info()['hash'], hashc)
broker.put_container('b',
Timestamp(3).internal,
Timestamp(0).internal, 0, 0, POLICIES.default.idx)
hashb = hashlib.md5(
'%s-%s' %
('b', "%s-%s-%s-%s" %
(Timestamp(3).internal, Timestamp(0).internal, 0, 0))).digest()
hashc = \
''.join(('%02x' % (ord(a) ^ ord(b)) for a, b in zip(hasha, hashb)))
self.assertEqual(broker.get_info()['hash'], hashc)
def test_double_check_trailing_delimiter(self):
# Test AccountBroker.list_containers_iter for an
# account that has an odd container with a trailing delimiter
broker = AccountBroker(':memory:', account='a')
broker.initialize(normalize_timestamp('1'))
broker.put_container('a', normalize_timestamp(time()), 0, 0, 0)
broker.put_container('a-', normalize_timestamp(time()), 0, 0, 0)
broker.put_container('a-a', normalize_timestamp(time()), 0, 0, 0)
broker.put_container('a-a-a', normalize_timestamp(time()), 0, 0, 0)
broker.put_container('a-a-b', normalize_timestamp(time()), 0, 0, 0)
broker.put_container('a-b', normalize_timestamp(time()), 0, 0, 0)
broker.put_container('b', normalize_timestamp(time()), 0, 0, 0)
broker.put_container('b-a', normalize_timestamp(time()), 0, 0, 0)
broker.put_container('b-b', normalize_timestamp(time()), 0, 0, 0)
broker.put_container('c', normalize_timestamp(time()), 0, 0, 0)
listing = broker.list_containers_iter(15, None, None, None, None)
self.assertEqual(len(listing), 10)
self.assertEqual([row[0] for row in listing],
['a', 'a-', 'a-a', 'a-a-a', 'a-a-b', 'a-b', 'b',
'b-a', 'b-b', 'c'])
listing = broker.list_containers_iter(15, None, None, '', '-')
self.assertEqual(len(listing), 5)
self.assertEqual([row[0] for row in listing],
['a', 'a-', 'b', 'b-', 'c'])
listing = broker.list_containers_iter(15, None, None, 'a-', '-')
self.assertEqual(len(listing), 4)
self.assertEqual([row[0] for row in listing],
['a-', 'a-a', 'a-a-', 'a-b'])
listing = broker.list_containers_iter(15, None, None, 'b-', '-')
self.assertEqual(len(listing), 2)
self.assertEqual([row[0] for row in listing], ['b-a', 'b-b'])
def tearDown(self):
AccountBroker.create_container_table = \
self._imported_create_container_table
AccountBroker._initialize = self._imported_initialize
broker = AccountBroker(':memory:', account='a')
broker.initialize(Timestamp('1').internal)
with broker.get() as conn:
conn.execute('SELECT storage_policy_index FROM container')
def test_reclaim(self):
broker = AccountBroker(':memory:', account='test_account')
broker.initialize(normalize_timestamp('1'))
broker.put_container('c', normalize_timestamp(time()), 0, 0, 0)
with broker.get() as conn:
self.assertEqual(conn.execute(
"SELECT count(*) FROM container "
"WHERE deleted = 0").fetchone()[0], 1)
self.assertEqual(conn.execute(
"SELECT count(*) FROM container "
"WHERE deleted = 1").fetchone()[0], 0)
broker.reclaim(normalize_timestamp(time() - 999), time())
with broker.get() as conn:
self.assertEqual(conn.execute(
"SELECT count(*) FROM container "
"WHERE deleted = 0").fetchone()[0], 1)
self.assertEqual(conn.execute(
"SELECT count(*) FROM container "
"WHERE deleted = 1").fetchone()[0], 0)
sleep(.00001)
broker.put_container('c', 0, normalize_timestamp(time()), 0, 0)
with broker.get() as conn:
self.assertEqual(conn.execute(
"SELECT count(*) FROM container "
"WHERE deleted = 0").fetchone()[0], 0)
self.assertEqual(conn.execute(
"SELECT count(*) FROM container "
"WHERE deleted = 1").fetchone()[0], 1)
broker.reclaim(normalize_timestamp(time() - 999), time())
with broker.get() as conn:
self.assertEqual(conn.execute(
"SELECT count(*) FROM container "
"WHERE deleted = 0").fetchone()[0], 0)
self.assertEqual(conn.execute(
"SELECT count(*) FROM container "
"WHERE deleted = 1").fetchone()[0], 1)
sleep(.00001)
broker.reclaim(normalize_timestamp(time()), time())
with broker.get() as conn:
self.assertEqual(conn.execute(
"SELECT count(*) FROM container "
"WHERE deleted = 0").fetchone()[0], 0)
self.assertEqual(conn.execute(
"SELECT count(*) FROM container "
"WHERE deleted = 1").fetchone()[0], 0)
# Test reclaim after deletion. Create 3 test containers
broker.put_container('x', 0, 0, 0, 0)
broker.put_container('y', 0, 0, 0, 0)
broker.put_container('z', 0, 0, 0, 0)
broker.reclaim(normalize_timestamp(time()), time())
# self.assertEqual(len(res), 2)
# self.assert_(isinstance(res, tuple))
# containers, account_name = res
# self.assert_(containers is None)
# self.assert_(account_name is None)
# Now delete the account
broker.delete_db(normalize_timestamp(time()))
broker.reclaim(normalize_timestamp(time()), time())
def test_account_stat_get_data(self):
stat = db_stats_collector.AccountStatsCollector(self.conf)
account_db = AccountBroker("%s/acc.db" % self.accounts,
account='test_acc')
account_db.initialize()
account_db.put_container('test_container', time.time(), None, 10, 1000,
1)
info = stat.get_data("%s/acc.db" % self.accounts)
self.assertEquals('''"test_acc",1,10,1000\n''', info)
def reap_device(self, device):
"""
Called once per pass for each device on the server. This will scan the
accounts directory for the device, looking for partitions this device
is the primary for, then looking for account databases that are marked
status=DELETED and still have containers and calling
:func:`reap_account`. Account databases marked status=DELETED that no
longer have containers will eventually be permanently removed by the
reclaim process within the account replicator (see
:mod:`swift.db_replicator`).
:param device: The device to look for accounts to be deleted.
"""
datadir = os.path.join(self.devices, device, DATADIR)
if not os.path.exists(datadir):
return
for partition in os.listdir(datadir):
partition_path = os.path.join(datadir, partition)
if not partition.isdigit():
continue
nodes = self.get_account_ring().get_part_nodes(int(partition))
if not os.path.isdir(partition_path):
continue
container_shard = None
for container_shard, node in enumerate(nodes):
if is_local_device(self.myips, None, node['ip'], None) and \
(not self.bind_port or
self.bind_port == node['port']) and \
(device == node['device']):
break
else:
continue
for suffix in os.listdir(partition_path):
suffix_path = os.path.join(partition_path, suffix)
if not os.path.isdir(suffix_path):
continue
for hsh in os.listdir(suffix_path):
hsh_path = os.path.join(suffix_path, hsh)
if not os.path.isdir(hsh_path):
continue
for fname in sorted(os.listdir(hsh_path), reverse=True):
if fname.endswith('.ts'):
break
elif fname.endswith('.db'):
self.start_time = time()
broker = \
AccountBroker(os.path.join(hsh_path, fname),
logger=self.logger)
if broker.is_status_deleted() and \
not broker.empty():
self.reap_account(
broker,
partition,
nodes,
container_shard=container_shard)
def test_empty(self):
# Test AccountBroker.empty
broker = AccountBroker(':memory:', account='a')
broker.initialize(normalize_timestamp('1'))
self.assert_(broker.empty())
broker.put_container('o', normalize_timestamp(time()), 0, 0, 0)
self.assert_(not broker.empty())
sleep(.00001)
broker.put_container('o', 0, normalize_timestamp(time()), 0, 0)
self.assert_(broker.empty())
def test_merge_items(self):
broker1 = AccountBroker(':memory:', account='a')
broker1.initialize(normalize_timestamp('1'))
broker2 = AccountBroker(':memory:', account='a')
broker2.initialize(normalize_timestamp('1'))
broker1.put_container('a', normalize_timestamp(1), 0, 0, 0)
broker1.put_container('b', normalize_timestamp(2), 0, 0, 0)
id = broker1.get_info()['id']
broker2.merge_items(
broker1.get_items_since(broker2.get_sync(id), 1000), id)
items = broker2.get_items_since(-1, 1000)
self.assertEquals(len(items), 2)
self.assertEquals(['a', 'b'], sorted([rec['name'] for rec in items]))
broker1.put_container('c', normalize_timestamp(3), 0, 0, 0)
broker2.merge_items(
broker1.get_items_since(broker2.get_sync(id), 1000), id)
items = broker2.get_items_since(-1, 1000)
self.assertEquals(len(items), 3)
self.assertEquals(['a', 'b', 'c'],
sorted([rec['name'] for rec in items]))
def test_get_policy_stats(self):
ts = (Timestamp(t).internal for t in itertools.count(int(time())))
broker = AccountBroker(':memory:', account='a')
broker.initialize(ts.next())
# check empty policy_stats
self.assertTrue(broker.empty())
policy_stats = broker.get_policy_stats()
self.assertEqual(policy_stats, {})
# add some empty containers
for policy in POLICIES:
container_name = 'c-%s' % policy.name
put_timestamp = ts.next()
broker.put_container(container_name, put_timestamp, 0, 0, 0,
policy.idx)
policy_stats = broker.get_policy_stats()
stats = policy_stats[policy.idx]
self.assertEqual(stats['object_count'], 0)
self.assertEqual(stats['bytes_used'], 0)
# update the containers object & byte count
for policy in POLICIES:
container_name = 'c-%s' % policy.name
put_timestamp = ts.next()
count = policy.idx * 100 # good as any integer
broker.put_container(container_name, put_timestamp, 0, count,
count, policy.idx)
policy_stats = broker.get_policy_stats()
stats = policy_stats[policy.idx]
self.assertEqual(stats['object_count'], count)
self.assertEqual(stats['bytes_used'], count)
# check all the policy_stats at once
for policy_index, stats in policy_stats.items():
policy = POLICIES[policy_index]
count = policy.idx * 100 # coupled with policy for test
self.assertEqual(stats['object_count'], count)
self.assertEqual(stats['bytes_used'], count)
# now delete the containers one by one
for policy in POLICIES:
container_name = 'c-%s' % policy.name
delete_timestamp = ts.next()
broker.put_container(container_name, 0, delete_timestamp, 0, 0,
policy.idx)
policy_stats = broker.get_policy_stats()
stats = policy_stats[policy.idx]
self.assertEqual(stats['object_count'], 0)
self.assertEqual(stats['bytes_used'], 0)
def test_get_info(self):
# Test AccountBroker.get_info
broker = AccountBroker(':memory:', account='test1')
broker.initialize(Timestamp('1').internal)
info = broker.get_info()
self.assertEqual(info['account'], 'test1')
self.assertEqual(info['hash'], '00000000000000000000000000000000')
self.assertEqual(info['put_timestamp'], Timestamp(1).internal)
self.assertEqual(info['delete_timestamp'], '0')
if self.__class__ == TestAccountBrokerBeforeMetadata:
self.assertEqual(info['status_changed_at'], '0')
else:
self.assertEqual(info['status_changed_at'], Timestamp(1).internal)
info = broker.get_info()
self.assertEqual(info['container_count'], 0)
broker.put_container('c1',
Timestamp(time()).internal, 0, 0, 0,
POLICIES.default.idx)
info = broker.get_info()
self.assertEqual(info['container_count'], 1)
sleep(.00001)
broker.put_container('c2',
Timestamp(time()).internal, 0, 0, 0,
POLICIES.default.idx)
info = broker.get_info()
self.assertEqual(info['container_count'], 2)
sleep(.00001)
broker.put_container('c2',
Timestamp(time()).internal, 0, 0, 0,
POLICIES.default.idx)
info = broker.get_info()
self.assertEqual(info['container_count'], 2)
sleep(.00001)
broker.put_container('c1', 0,
Timestamp(time()).internal, 0, 0,
POLICIES.default.idx)
info = broker.get_info()
self.assertEqual(info['container_count'], 1)
sleep(.00001)
broker.put_container('c2', 0,
Timestamp(time()).internal, 0, 0,
POLICIES.default.idx)
info = broker.get_info()
self.assertEqual(info['container_count'], 0)
def setUp(self):
self._imported_create_account_stat_table = \
AccountBroker.create_account_stat_table
AccountBroker.create_account_stat_table = \
premetadata_create_account_stat_table
broker = AccountBroker(':memory:', account='a')
broker.initialize(normalize_timestamp('1'))
exc = None
with broker.get() as conn:
try:
conn.execute('SELECT metadata FROM account_stat')
except BaseException as err:
exc = err
self.assert_('no such column: metadata' in str(exc))
def test_exception(self):
# Test AccountBroker throwing a conn away after exception
first_conn = None
broker = AccountBroker(':memory:', account='a')
broker.initialize(normalize_timestamp('1'))
with broker.get() as conn:
first_conn = conn
try:
with broker.get() as conn:
self.assertEqual(first_conn, conn)
raise Exception('OMG')
except Exception:
pass
self.assert_(broker.conn is None)
def test_empty(self):
# Test AccountBroker.empty
broker = AccountBroker(':memory:', account='a')
broker.initialize(Timestamp('1').internal)
self.assert_(broker.empty())
broker.put_container('o',
Timestamp(time()).internal, 0, 0, 0,
POLICIES.default.idx)
self.assert_(not broker.empty())
sleep(.00001)
broker.put_container('o', 0,
Timestamp(time()).internal, 0, 0,
POLICIES.default.idx)
self.assert_(broker.empty())
def get_data(self, db_path):
"""
Data for generated csv has the following columns:
Account Hash, Container Count, Object Count, Bytes Used
:raises sqlite3.Error: does not catch errors connecting to db
"""
line_data = None
broker = AccountBroker(db_path)
if not broker.is_deleted():
info = broker.get_info()
line_data = '"%s",%d,%d,%d\n' % (
info['account'], info['container_count'], info['object_count'],
info['bytes_used'])
return line_data
def _gen_account_stat(self):
stat = db_stats_collector.AccountStatsCollector(self.conf)
output_data = set()
for i in range(10):
account_db = AccountBroker("%s/stats-201001010%s-%s.db" %
(self.accounts, i, uuid.uuid4().hex),
account='test_acc_%s' % i)
account_db.initialize()
account_db.put_container('test_container', time.time(), None, 10,
1000, 1)
# this will "commit" the data
account_db.get_info()
output_data.add('''"test_acc_%s",1,10,1000''' % i),
self.assertEqual(len(output_data), 10)
return stat, output_data
def test_creation(self):
# Test AccountBroker.__init__
broker = AccountBroker(':memory:', account='a')
self.assertEqual(broker.db_file, ':memory:')
got_exc = False
try:
with broker.get() as conn:
pass
except Exception:
got_exc = True
self.assert_(got_exc)
broker.initialize(normalize_timestamp('1'))
with broker.get() as conn:
curs = conn.cursor()
curs.execute('SELECT 1')
self.assertEqual(curs.fetchall()[0][0], 1)
def print_info(db_type,
db_file,
swift_dir='/etc/swift',
stale_reads_ok=False,
drop_prefixes=False,
verbose=False):
if db_type not in ('account', 'container'):
print("Unrecognized DB type: internal error")
raise InfoSystemExit()
if not os.path.exists(db_file) or not db_file.endswith('.db'):
print("DB file doesn't exist")
raise InfoSystemExit()
if not db_file.startswith(('/', './')):
db_file = './' + db_file # don't break if the bare db file is given
if db_type == 'account':
broker = AccountBroker(db_file, stale_reads_ok=stale_reads_ok)
datadir = ABDATADIR
else:
broker = ContainerBroker(db_file, stale_reads_ok=stale_reads_ok)
datadir = CBDATADIR
try:
info = broker.get_info()
except sqlite3.OperationalError as err:
if 'no such table' in str(err):
print("Does not appear to be a DB of type \"%s\": %s" %
(db_type, db_file))
raise InfoSystemExit()
raise
account = info['account']
container = None
info['is_deleted'] = broker.is_deleted()
if db_type == 'container':
container = info['container']
info['is_root'] = broker.is_root_container()
sranges = broker.get_shard_ranges()
if sranges:
info['shard_ranges'] = sranges
print_db_info_metadata(db_type, info, broker.metadata, drop_prefixes,
verbose)
try:
ring = Ring(swift_dir, ring_name=db_type)
except Exception:
ring = None
else:
print_ring_locations(ring, datadir, account, container)
