def test_double_check_trailing_delimiter(self): # Test AccountBroker.list_containers_iter for an # account that has an odd container with a trailing delimiter broker = AccountBroker(':memory:', account='a') broker.initialize(normalize_timestamp('1')) broker.put_container('a', normalize_timestamp(time()), 0, 0, 0) broker.put_container('a-', normalize_timestamp(time()), 0, 0, 0) broker.put_container('a-a', normalize_timestamp(time()), 0, 0, 0) broker.put_container('a-a-a', normalize_timestamp(time()), 0, 0, 0) broker.put_container('a-a-b', normalize_timestamp(time()), 0, 0, 0) broker.put_container('a-b', normalize_timestamp(time()), 0, 0, 0) broker.put_container('b', normalize_timestamp(time()), 0, 0, 0) broker.put_container('b-a', normalize_timestamp(time()), 0, 0, 0) broker.put_container('b-b', normalize_timestamp(time()), 0, 0, 0) broker.put_container('c', normalize_timestamp(time()), 0, 0, 0) listing = broker.list_containers_iter(15, None, None, None, None) self.assertEqual(len(listing), 10) self.assertEqual([row[0] for row in listing], ['a', 'a-', 'a-a', 'a-a-a', 'a-a-b', 'a-b', 'b', 'b-a', 'b-b', 'c']) listing = broker.list_containers_iter(15, None, None, '', '-') self.assertEqual(len(listing), 5) self.assertEqual([row[0] for row in listing], ['a', 'a-', 'b', 'b-', 'c']) listing = broker.list_containers_iter(15, None, None, 'a-', '-') self.assertEqual(len(listing), 4) self.assertEqual([row[0] for row in listing], ['a-', 'a-a', 'a-a-', 'a-b']) listing = broker.list_containers_iter(15, None, None, 'b-', '-') self.assertEqual(len(listing), 2) self.assertEqual([row[0] for row in listing], ['b-a', 'b-b'])
def test_container_table_migration(self, tempdir): db_path = os.path.join(tempdir, 'account.db') # first init an acct DB without the policy_stat table present broker = AccountBroker(db_path, account='a') broker.initialize(Timestamp('1').internal) with broker.get() as conn: try: conn.execute(''' SELECT storage_policy_index FROM container ''').fetchone()[0] except sqlite3.OperationalError as err: # confirm that the table doesn't have this column self.assert_( 'no such column: storage_policy_index' in str(err)) else: self.fail('broker did not raise sqlite3.OperationalError ' 'trying to select from storage_policy_index ' 'from container table!') # manually insert an existing row to avoid migration with broker.get() as conn: conn.execute( ''' INSERT INTO container (name, put_timestamp, delete_timestamp, object_count, bytes_used, deleted) VALUES (?, ?, ?, ?, ?, ?) ''', ('test_name', Timestamp(time()).internal, 0, 1, 2, 0)) conn.commit() # make sure we can iter containers without the migration for c in broker.list_containers_iter(1, None, None, None, None): self.assertEqual(c, ('test_name', 1, 2, 0)) # stats table is mysteriously empty... stats = broker.get_policy_stats() self.assertEqual(len(stats), 0) # now do a PUT with a different value for storage_policy_index # which will update the DB schema as well as update policy_stats # for legacy containers in the DB (those without an SPI) other_policy = [p for p in POLICIES if p.idx != 0][0] broker.put_container('test_second', Timestamp(time()).internal, 0, 3, 4, other_policy.idx) broker._commit_puts_stale_ok() with broker.get() as conn: rows = conn.execute(''' SELECT name, storage_policy_index FROM container ''').fetchall() for row in rows: if row[0] == 'test_name': self.assertEqual(row[1], 0) else: self.assertEqual(row[1], other_policy.idx) # we should have stats for both containers stats = broker.get_policy_stats() self.assertEqual(len(stats), 2) self.assertEqual(stats[0]['object_count'], 1) self.assertEqual(stats[0]['bytes_used'], 2) self.assertEqual(stats[1]['object_count'], 3) self.assertEqual(stats[1]['bytes_used'], 4) # now lets delete a container and make sure policy_stats is OK with broker.get() as conn: conn.execute( ''' DELETE FROM container WHERE name = ? ''', ('test_name', )) conn.commit() stats = broker.get_policy_stats() self.assertEqual(len(stats), 2) self.assertEqual(stats[0]['object_count'], 0) self.assertEqual(stats[0]['bytes_used'], 0) self.assertEqual(stats[1]['object_count'], 3) self.assertEqual(stats[1]['bytes_used'], 4)
def test_list_containers_iter(self): # Test AccountBroker.list_containers_iter broker = AccountBroker(':memory:', account='a') broker.initialize(normalize_timestamp('1')) for cont1 in xrange(4): for cont2 in xrange(125): broker.put_container('%d-%04d' % (cont1, cont2), normalize_timestamp(time()), 0, 0, 0) for cont in xrange(125): broker.put_container('2-0051-%04d' % cont, normalize_timestamp(time()), 0, 0, 0) for cont in xrange(125): broker.put_container('3-%04d-0049' % cont, normalize_timestamp(time()), 0, 0, 0) listing = broker.list_containers_iter(100, '', None, None, '') self.assertEqual(len(listing), 100) self.assertEqual(listing[0][0], '0-0000') self.assertEqual(listing[-1][0], '0-0099') listing = broker.list_containers_iter(100, '', '0-0050', None, '') self.assertEqual(len(listing), 50) self.assertEqual(listing[0][0], '0-0000') self.assertEqual(listing[-1][0], '0-0049') listing = broker.list_containers_iter(100, '0-0099', None, None, '') self.assertEqual(len(listing), 100) self.assertEqual(listing[0][0], '0-0100') self.assertEqual(listing[-1][0], '1-0074') listing = broker.list_containers_iter(55, '1-0074', None, None, '') self.assertEqual(len(listing), 55) self.assertEqual(listing[0][0], '1-0075') self.assertEqual(listing[-1][0], '2-0004') listing = broker.list_containers_iter(10, '', None, '0-01', '') self.assertEqual(len(listing), 10) self.assertEqual(listing[0][0], '0-0100') self.assertEqual(listing[-1][0], '0-0109') listing = broker.list_containers_iter(10, '', None, '0-01', '-') self.assertEqual(len(listing), 10) self.assertEqual(listing[0][0], '0-0100') self.assertEqual(listing[-1][0], '0-0109') listing = broker.list_containers_iter(10, '', None, '0-', '-') self.assertEqual(len(listing), 10) self.assertEqual(listing[0][0], '0-0000') self.assertEqual(listing[-1][0], '0-0009') listing = broker.list_containers_iter(10, '', None, '', '-') self.assertEqual(len(listing), 4) self.assertEqual([row[0] for row in listing], ['0-', '1-', '2-', '3-']) listing = broker.list_containers_iter(10, '2-', None, None, '-') self.assertEqual(len(listing), 1) self.assertEqual([row[0] for row in listing], ['3-']) listing = broker.list_containers_iter(10, '', None, '2', '-') self.assertEqual(len(listing), 1) self.assertEqual([row[0] for row in listing], ['2-']) listing = broker.list_containers_iter(10, '2-0050', None, '2-', '-') self.assertEqual(len(listing), 10) self.assertEqual(listing[0][0], '2-0051') self.assertEqual(listing[1][0], '2-0051-') self.assertEqual(listing[2][0], '2-0052') self.assertEqual(listing[-1][0], '2-0059') listing = broker.list_containers_iter(10, '3-0045', None, '3-', '-') self.assertEqual(len(listing), 10) self.assertEqual([row[0] for row in listing], ['3-0045-', '3-0046', '3-0046-', '3-0047', '3-0047-', '3-0048', '3-0048-', '3-0049', '3-0049-', '3-0050']) broker.put_container('3-0049-', normalize_timestamp(time()), 0, 0, 0) listing = broker.list_containers_iter(10, '3-0048', None, None, None) self.assertEqual(len(listing), 10) self.assertEqual([row[0] for row in listing], ['3-0048-0049', '3-0049', '3-0049-', '3-0049-0049', '3-0050', '3-0050-0049', '3-0051', '3-0051-0049', '3-0052', '3-0052-0049']) listing = broker.list_containers_iter(10, '3-0048', None, '3-', '-') self.assertEqual(len(listing), 10) self.assertEqual([row[0] for row in listing], ['3-0048-', '3-0049', '3-0049-', '3-0050', '3-0050-', '3-0051', '3-0051-', '3-0052', '3-0052-', '3-0053']) listing = broker.list_containers_iter(10, None, None, '3-0049-', '-') self.assertEqual(len(listing), 2) self.assertEqual([row[0] for row in listing], ['3-0049-', '3-0049-0049'])
def test_container_table_migration(self, tempdir): db_path = os.path.join(tempdir, 'account.db') # first init an acct DB without the policy_stat table present broker = AccountBroker(db_path, account='a') broker.initialize(Timestamp('1').internal) with broker.get() as conn: try: conn.execute(''' SELECT storage_policy_index FROM container ''').fetchone()[0] except sqlite3.OperationalError as err: # confirm that the table doesn't have this column self.assert_('no such column: storage_policy_index' in str(err)) else: self.fail('broker did not raise sqlite3.OperationalError ' 'trying to select from storage_policy_index ' 'from container table!') # manually insert an existing row to avoid migration with broker.get() as conn: conn.execute(''' INSERT INTO container (name, put_timestamp, delete_timestamp, object_count, bytes_used, deleted) VALUES (?, ?, ?, ?, ?, ?) ''', ('test_name', Timestamp(time()).internal, 0, 1, 2, 0)) conn.commit() # make sure we can iter containers without the migration for c in broker.list_containers_iter(1, None, None, None, None): self.assertEqual(c, ('test_name', 1, 2, 0)) # stats table is mysteriously empty... stats = broker.get_policy_stats() self.assertEqual(len(stats), 0) # now do a PUT with a different value for storage_policy_index # which will update the DB schema as well as update policy_stats # for legacy containers in the DB (those without an SPI) other_policy = [p for p in POLICIES if p.idx != 0][0] broker.put_container('test_second', Timestamp(time()).internal, 0, 3, 4, other_policy.idx) broker._commit_puts_stale_ok() with broker.get() as conn: rows = conn.execute(''' SELECT name, storage_policy_index FROM container ''').fetchall() for row in rows: if row[0] == 'test_name': self.assertEqual(row[1], 0) else: self.assertEqual(row[1], other_policy.idx) # we should have stats for both containers stats = broker.get_policy_stats() self.assertEqual(len(stats), 2) self.assertEqual(stats[0]['object_count'], 1) self.assertEqual(stats[0]['bytes_used'], 2) self.assertEqual(stats[1]['object_count'], 3) self.assertEqual(stats[1]['bytes_used'], 4) # now lets delete a container and make sure policy_stats is OK with broker.get() as conn: conn.execute(''' DELETE FROM container WHERE name = ? ''', ('test_name',)) conn.commit() stats = broker.get_policy_stats() self.assertEqual(len(stats), 2) self.assertEqual(stats[0]['object_count'], 0) self.assertEqual(stats[0]['bytes_used'], 0) self.assertEqual(stats[1]['object_count'], 3) self.assertEqual(stats[1]['bytes_used'], 4)