def test_policy_table_migration(self, tempdir): db_path = os.path.join(tempdir, 'account.db') # first init an acct DB without the policy_stat table present broker = AccountBroker(db_path, account='a') broker.initialize(Timestamp('1').internal) with broker.get() as conn: try: conn.execute(''' SELECT * FROM policy_stat ''').fetchone()[0] except sqlite3.OperationalError as err: # confirm that the table really isn't there self.assert_('no such table: policy_stat' in str(err)) else: self.fail('broker did not raise sqlite3.OperationalError ' 'trying to select from policy_stat table!') # make sure we can HEAD this thing w/o the table stats = broker.get_policy_stats() self.assertEqual(len(stats), 0) # now do a PUT to create the table broker.put_container('o', Timestamp(time()).internal, 0, 0, 0, POLICIES.default.idx) broker._commit_puts_stale_ok() # now confirm that the table was created with broker.get() as conn: conn.execute('SELECT * FROM policy_stat') stats = broker.get_policy_stats() self.assertEqual(len(stats), 1)
def test_chexor(self): broker = AccountBroker(':memory:', account='a') broker.initialize(Timestamp('1').internal) broker.put_container('a', Timestamp(1).internal, Timestamp(0).internal, 0, 0, POLICIES.default.idx) broker.put_container('b', Timestamp(2).internal, Timestamp(0).internal, 0, 0, POLICIES.default.idx) hasha = hashlib.md5( '%s-%s' % ('a', "%s-%s-%s-%s" % ( Timestamp(1).internal, Timestamp(0).internal, 0, 0)) ).digest() hashb = hashlib.md5( '%s-%s' % ('b', "%s-%s-%s-%s" % ( Timestamp(2).internal, Timestamp(0).internal, 0, 0)) ).digest() hashc = \ ''.join(('%02x' % (ord(a) ^ ord(b)) for a, b in zip(hasha, hashb))) self.assertEqual(broker.get_info()['hash'], hashc) broker.put_container('b', Timestamp(3).internal, Timestamp(0).internal, 0, 0, POLICIES.default.idx) hashb = hashlib.md5( '%s-%s' % ('b', "%s-%s-%s-%s" % ( Timestamp(3).internal, Timestamp(0).internal, 0, 0)) ).digest() hashc = \ ''.join(('%02x' % (ord(a) ^ ord(b)) for a, b in zip(hasha, hashb))) self.assertEqual(broker.get_info()['hash'], hashc)
def test_double_check_trailing_delimiter(self): # Test AccountBroker.list_containers_iter for an # account that has an odd container with a trailing delimiter broker = AccountBroker(':memory:', account='a') broker.initialize(normalize_timestamp('1')) broker.put_container('a', normalize_timestamp(time()), 0, 0, 0) broker.put_container('a-', normalize_timestamp(time()), 0, 0, 0) broker.put_container('a-a', normalize_timestamp(time()), 0, 0, 0) broker.put_container('a-a-a', normalize_timestamp(time()), 0, 0, 0) broker.put_container('a-a-b', normalize_timestamp(time()), 0, 0, 0) broker.put_container('a-b', normalize_timestamp(time()), 0, 0, 0) broker.put_container('b', normalize_timestamp(time()), 0, 0, 0) broker.put_container('b-a', normalize_timestamp(time()), 0, 0, 0) broker.put_container('b-b', normalize_timestamp(time()), 0, 0, 0) broker.put_container('c', normalize_timestamp(time()), 0, 0, 0) listing = broker.list_containers_iter(15, None, None, None, None) self.assertEqual(len(listing), 10) self.assertEqual([row[0] for row in listing], ['a', 'a-', 'a-a', 'a-a-a', 'a-a-b', 'a-b', 'b', 'b-a', 'b-b', 'c']) listing = broker.list_containers_iter(15, None, None, '', '-') self.assertEqual(len(listing), 5) self.assertEqual([row[0] for row in listing], ['a', 'a-', 'b', 'b-', 'c']) listing = broker.list_containers_iter(15, None, None, 'a-', '-') self.assertEqual(len(listing), 4) self.assertEqual([row[0] for row in listing], ['a-', 'a-a', 'a-a-', 'a-b']) listing = broker.list_containers_iter(15, None, None, 'b-', '-') self.assertEqual(len(listing), 2) self.assertEqual([row[0] for row in listing], ['b-a', 'b-b'])
def test_get_info(self): # Test AccountBroker.get_info broker = AccountBroker(':memory:', account='test1') broker.initialize(normalize_timestamp('1')) info = broker.get_info() self.assertEqual(info['account'], 'test1') self.assertEqual(info['hash'], '00000000000000000000000000000000') info = broker.get_info() self.assertEqual(info['container_count'], 0) broker.put_container('c1', normalize_timestamp(time()), 0, 0, 0) info = broker.get_info() self.assertEqual(info['container_count'], 1) sleep(.00001) broker.put_container('c2', normalize_timestamp(time()), 0, 0, 0) info = broker.get_info() self.assertEqual(info['container_count'], 2) sleep(.00001) broker.put_container('c2', normalize_timestamp(time()), 0, 0, 0) info = broker.get_info() self.assertEqual(info['container_count'], 2) sleep(.00001) broker.put_container('c1', 0, normalize_timestamp(time()), 0, 0) info = broker.get_info() self.assertEqual(info['container_count'], 1) sleep(.00001) broker.put_container('c2', 0, normalize_timestamp(time()), 0, 0) info = broker.get_info() self.assertEqual(info['container_count'], 0)
def test_reclaim(self): broker = AccountBroker(':memory:', account='test_account') broker.initialize(Timestamp('1').internal) broker.put_container('c', Timestamp(time()).internal, 0, 0, 0, POLICIES.default.idx) with broker.get() as conn: self.assertEqual(conn.execute( "SELECT count(*) FROM container " "WHERE deleted = 0").fetchone()[0], 1) self.assertEqual(conn.execute( "SELECT count(*) FROM container " "WHERE deleted = 1").fetchone()[0], 0) broker.reclaim(Timestamp(time() - 999).internal, time()) with broker.get() as conn: self.assertEqual(conn.execute( "SELECT count(*) FROM container " "WHERE deleted = 0").fetchone()[0], 1) self.assertEqual(conn.execute( "SELECT count(*) FROM container " "WHERE deleted = 1").fetchone()[0], 0) sleep(.00001) broker.put_container('c', 0, Timestamp(time()).internal, 0, 0, POLICIES.default.idx) with broker.get() as conn: self.assertEqual(conn.execute( "SELECT count(*) FROM container " "WHERE deleted = 0").fetchone()[0], 0) self.assertEqual(conn.execute( "SELECT count(*) FROM container " "WHERE deleted = 1").fetchone()[0], 1) broker.reclaim(Timestamp(time() - 999).internal, time()) with broker.get() as conn: self.assertEqual(conn.execute( "SELECT count(*) FROM container " "WHERE deleted = 0").fetchone()[0], 0) self.assertEqual(conn.execute( "SELECT count(*) FROM container " "WHERE deleted = 1").fetchone()[0], 1) sleep(.00001) broker.reclaim(Timestamp(time()).internal, time()) with broker.get() as conn: self.assertEqual(conn.execute( "SELECT count(*) FROM container " "WHERE deleted = 0").fetchone()[0], 0) self.assertEqual(conn.execute( "SELECT count(*) FROM container " "WHERE deleted = 1").fetchone()[0], 0) # Test reclaim after deletion. Create 3 test containers broker.put_container('x', 0, 0, 0, 0, POLICIES.default.idx) broker.put_container('y', 0, 0, 0, 0, POLICIES.default.idx) broker.put_container('z', 0, 0, 0, 0, POLICIES.default.idx) broker.reclaim(Timestamp(time()).internal, time()) # self.assertEqual(len(res), 2) # self.assert_(isinstance(res, tuple)) # containers, account_name = res # self.assert_(containers is None) # self.assert_(account_name is None) # Now delete the account broker.delete_db(Timestamp(time()).internal) broker.reclaim(Timestamp(time()).internal, time())
def tearDown(self): AccountBroker.create_account_stat_table = \ self._imported_create_account_stat_table broker = AccountBroker(':memory:', account='a') broker.initialize(normalize_timestamp('1')) with broker.get() as conn: conn.execute('SELECT metadata FROM account_stat')
def test_half_upgraded_database(self, tempdir): db_path = os.path.join(tempdir, 'account.db') ts = itertools.count() ts = (Timestamp(t).internal for t in itertools.count(int(time()))) broker = AccountBroker(db_path, account='a') broker.initialize(ts.next()) self.assertTrue(broker.empty()) # add a container (to pending file) broker.put_container('c', ts.next(), 0, 0, 0, POLICIES.default.idx) real_get = broker.get called = [] @contextmanager def mock_get(): with real_get() as conn: def mock_executescript(script): if called: raise Exception('kaboom!') called.append(script) conn.executescript = mock_executescript yield conn broker.get = mock_get try: broker._commit_puts() except Exception: pass else: self.fail('mock exception was not raised') self.assertEqual(len(called), 1) self.assert_('CREATE TABLE policy_stat' in called[0]) # nothing was commited broker = AccountBroker(db_path, account='a') with broker.get() as conn: try: conn.execute('SELECT * FROM policy_stat') except sqlite3.OperationalError as err: self.assert_('no such table: policy_stat' in str(err)) else: self.fail('half upgraded database!') container_count = conn.execute( 'SELECT count(*) FROM container').fetchone()[0] self.assertEqual(container_count, 0) # try again to commit puts self.assertFalse(broker.empty()) # full migration successful with broker.get() as conn: conn.execute('SELECT * FROM policy_stat') conn.execute('SELECT storage_policy_index FROM container')
def test_load_old_pending_puts(self): # pending puts from pre-storage-policy account brokers won't contain # the storage policy index tempdir = mkdtemp() broker_path = os.path.join(tempdir, 'test-load-old.db') try: broker = AccountBroker(broker_path, account='real') broker.initialize(Timestamp(1).internal) with open(broker_path + '.pending', 'a+b') as pending: pending.write(':') pending.write( pickle.dumps( # name, put_timestamp, delete_timestamp, object_count, # bytes_used, deleted ('oldcon', Timestamp(200).internal, Timestamp(0).internal, 896, 9216695, 0 )).encode('base64')) broker._commit_puts() with broker.get() as conn: results = list( conn.execute(''' SELECT name, storage_policy_index FROM container ''')) self.assertEqual(len(results), 1) self.assertEqual(dict(results[0]), { 'name': 'oldcon', 'storage_policy_index': 0 }) finally: rmtree(tempdir)
def test_load_old_pending_puts(self): # pending puts from pre-storage-policy account brokers won't contain # the storage policy index tempdir = mkdtemp() broker_path = os.path.join(tempdir, 'test-load-old.db') try: broker = AccountBroker(broker_path, account='real') broker.initialize(Timestamp(1).internal) with open(broker_path + '.pending', 'a+b') as pending: pending.write(':') pending.write(pickle.dumps( # name, put_timestamp, delete_timestamp, object_count, # bytes_used, deleted ('oldcon', Timestamp(200).internal, Timestamp(0).internal, 896, 9216695, 0)).encode('base64')) broker._commit_puts() with broker.get() as conn: results = list(conn.execute(''' SELECT name, storage_policy_index FROM container ''')) self.assertEqual(len(results), 1) self.assertEqual(dict(results[0]), {'name': 'oldcon', 'storage_policy_index': 0}) finally: rmtree(tempdir)
def test_merge_items(self): broker1 = AccountBroker(':memory:', account='a') broker1.initialize(Timestamp('1').internal) broker2 = AccountBroker(':memory:', account='a') broker2.initialize(Timestamp('1').internal) broker1.put_container('a', Timestamp(1).internal, 0, 0, 0, POLICIES.default.idx) broker1.put_container('b', Timestamp(2).internal, 0, 0, 0, POLICIES.default.idx) id = broker1.get_info()['id'] broker2.merge_items( broker1.get_items_since(broker2.get_sync(id), 1000), id) items = broker2.get_items_since(-1, 1000) self.assertEqual(len(items), 2) self.assertEqual(['a', 'b'], sorted([rec['name'] for rec in items])) broker1.put_container('c', Timestamp(3).internal, 0, 0, 0, POLICIES.default.idx) broker2.merge_items( broker1.get_items_since(broker2.get_sync(id), 1000), id) items = broker2.get_items_since(-1, 1000) self.assertEqual(len(items), 3) self.assertEqual(['a', 'b', 'c'], sorted([rec['name'] for rec in items]))
def test_chexor(self): broker = AccountBroker(':memory:', account='a') broker.initialize(Timestamp('1').internal) broker.put_container('a', Timestamp(1).internal, Timestamp(0).internal, 0, 0, POLICIES.default.idx) broker.put_container('b', Timestamp(2).internal, Timestamp(0).internal, 0, 0, POLICIES.default.idx) hasha = hashlib.md5( '%s-%s' % ('a', "%s-%s-%s-%s" % (Timestamp(1).internal, Timestamp(0).internal, 0, 0))).digest() hashb = hashlib.md5( '%s-%s' % ('b', "%s-%s-%s-%s" % (Timestamp(2).internal, Timestamp(0).internal, 0, 0))).digest() hashc = \ ''.join(('%02x' % (ord(a) ^ ord(b)) for a, b in zip(hasha, hashb))) self.assertEqual(broker.get_info()['hash'], hashc) broker.put_container('b', Timestamp(3).internal, Timestamp(0).internal, 0, 0, POLICIES.default.idx) hashb = hashlib.md5( '%s-%s' % ('b', "%s-%s-%s-%s" % (Timestamp(3).internal, Timestamp(0).internal, 0, 0))).digest() hashc = \ ''.join(('%02x' % (ord(a) ^ ord(b)) for a, b in zip(hasha, hashb))) self.assertEqual(broker.get_info()['hash'], hashc)
def test_policy_table_migration(self, tempdir): db_path = os.path.join(tempdir, 'account.db') # first init an acct DB without the policy_stat table present broker = AccountBroker(db_path, account='a') broker.initialize(Timestamp('1').internal) with broker.get() as conn: try: conn.execute(''' SELECT * FROM policy_stat ''').fetchone()[0] except sqlite3.OperationalError as err: # confirm that the table really isn't there self.assert_('no such table: policy_stat' in str(err)) else: self.fail('broker did not raise sqlite3.OperationalError ' 'trying to select from policy_stat table!') # make sure we can HEAD this thing w/o the table stats = broker.get_policy_stats() self.assertEqual(len(stats), 0) # now do a PUT to create the table broker.put_container('o', Timestamp(time()).internal, 0, 0, 0, POLICIES.default.idx) broker._commit_puts_stale_ok() # now confirm that the table was created with broker.get() as conn: conn.execute('SELECT * FROM policy_stat') stats = broker.get_policy_stats() self.assertEqual(len(stats), 1)
def test_policy_stats_tracking(self): ts = (Timestamp(t).internal for t in itertools.count(int(time()))) broker = AccountBroker(':memory:', account='a') broker.initialize(ts.next()) # policy 0 broker.put_container('con1', ts.next(), 0, 12, 2798641, 0) broker.put_container('con1', ts.next(), 0, 13, 8156441, 0) # policy 1 broker.put_container('con2', ts.next(), 0, 7, 5751991, 1) broker.put_container('con2', ts.next(), 0, 8, 6085379, 1) stats = broker.get_policy_stats() self.assertEqual(len(stats), 2) self.assertEqual(stats[0]['object_count'], 13) self.assertEqual(stats[0]['bytes_used'], 8156441) self.assertEqual(stats[1]['object_count'], 8) self.assertEqual(stats[1]['bytes_used'], 6085379) # Break encapsulation here to make sure that there's only 2 rows in # the stats table. It's possible that there could be 4 rows (one per # put_container) but that they came out in the right order so that # get_policy_stats() collapsed them down to the right number. To prove # that's not so, we have to go peek at the broker's internals. with broker.get() as conn: nrows = conn.execute( "SELECT COUNT(*) FROM policy_stat").fetchall()[0][0] self.assertEqual(nrows, 2)
def test_delete_container(self): # Test AccountBroker.delete_container broker = AccountBroker(':memory:', account='a') broker.initialize(Timestamp('1').internal) broker.put_container('o', Timestamp(time()).internal, 0, 0, 0, POLICIES.default.idx) with broker.get() as conn: self.assertEqual( conn.execute("SELECT count(*) FROM container " "WHERE deleted = 0").fetchone()[0], 1) self.assertEqual( conn.execute("SELECT count(*) FROM container " "WHERE deleted = 1").fetchone()[0], 0) sleep(.00001) broker.put_container('o', 0, Timestamp(time()).internal, 0, 0, POLICIES.default.idx) with broker.get() as conn: self.assertEqual( conn.execute("SELECT count(*) FROM container " "WHERE deleted = 0").fetchone()[0], 0) self.assertEqual( conn.execute("SELECT count(*) FROM container " "WHERE deleted = 1").fetchone()[0], 1)
def test_policy_stats_tracking(self): ts = (Timestamp(t).internal for t in itertools.count(int(time()))) broker = AccountBroker(':memory:', account='a') broker.initialize(ts.next()) # policy 0 broker.put_container('con1', ts.next(), 0, 12, 2798641, 0) broker.put_container('con1', ts.next(), 0, 13, 8156441, 0) # policy 1 broker.put_container('con2', ts.next(), 0, 7, 5751991, 1) broker.put_container('con2', ts.next(), 0, 8, 6085379, 1) stats = broker.get_policy_stats() self.assertEqual(len(stats), 2) self.assertEqual(stats[0]['object_count'], 13) self.assertEqual(stats[0]['bytes_used'], 8156441) self.assertEqual(stats[1]['object_count'], 8) self.assertEqual(stats[1]['bytes_used'], 6085379) # Break encapsulation here to make sure that there's only 2 rows in # the stats table. It's possible that there could be 4 rows (one per # put_container) but that they came out in the right order so that # get_policy_stats() collapsed them down to the right number. To prove # that's not so, we have to go peek at the broker's internals. with broker.get() as conn: nrows = conn.execute( "SELECT COUNT(*) FROM policy_stat").fetchall()[0][0] self.assertEqual(nrows, 2)
def tearDown(self): AccountBroker.create_container_table = \ self._imported_create_container_table AccountBroker._initialize = self._imported_initialize broker = AccountBroker(':memory:', account='a') broker.initialize(Timestamp('1').internal) with broker.get() as conn: conn.execute('SELECT storage_policy_index FROM container')
def tearDown(self): AccountBroker.create_container_table = \ self._imported_create_container_table AccountBroker._initialize = self._imported_initialize broker = AccountBroker(':memory:', account='a') broker.initialize(Timestamp('1').internal) with broker.get() as conn: conn.execute('SELECT storage_policy_index FROM container')
def test_get_policy_stats(self): ts = (Timestamp(t).internal for t in itertools.count(int(time()))) broker = AccountBroker(':memory:', account='a') broker.initialize(ts.next()) # check empty policy_stats self.assertTrue(broker.empty()) policy_stats = broker.get_policy_stats() self.assertEqual(policy_stats, {}) # add some empty containers for policy in POLICIES: container_name = 'c-%s' % policy.name put_timestamp = ts.next() broker.put_container(container_name, put_timestamp, 0, 0, 0, policy.idx) policy_stats = broker.get_policy_stats() stats = policy_stats[policy.idx] self.assertEqual(stats['object_count'], 0) self.assertEqual(stats['bytes_used'], 0) # update the containers object & byte count for policy in POLICIES: container_name = 'c-%s' % policy.name put_timestamp = ts.next() count = policy.idx * 100 # good as any integer broker.put_container(container_name, put_timestamp, 0, count, count, policy.idx) policy_stats = broker.get_policy_stats() stats = policy_stats[policy.idx] self.assertEqual(stats['object_count'], count) self.assertEqual(stats['bytes_used'], count) # check all the policy_stats at once for policy_index, stats in policy_stats.items(): policy = POLICIES[policy_index] count = policy.idx * 100 # coupled with policy for test self.assertEqual(stats['object_count'], count) self.assertEqual(stats['bytes_used'], count) # now delete the containers one by one for policy in POLICIES: container_name = 'c-%s' % policy.name delete_timestamp = ts.next() broker.put_container(container_name, 0, delete_timestamp, 0, 0, policy.idx) policy_stats = broker.get_policy_stats() stats = policy_stats[policy.idx] self.assertEqual(stats['object_count'], 0) self.assertEqual(stats['bytes_used'], 0)
def test_reclaim(self): broker = AccountBroker(':memory:', account='test_account') broker.initialize(normalize_timestamp('1')) broker.put_container('c', normalize_timestamp(time()), 0, 0, 0) with broker.get() as conn: self.assertEqual(conn.execute( "SELECT count(*) FROM container " "WHERE deleted = 0").fetchone()[0], 1) self.assertEqual(conn.execute( "SELECT count(*) FROM container " "WHERE deleted = 1").fetchone()[0], 0) broker.reclaim(normalize_timestamp(time() - 999), time()) with broker.get() as conn: self.assertEqual(conn.execute( "SELECT count(*) FROM container " "WHERE deleted = 0").fetchone()[0], 1) self.assertEqual(conn.execute( "SELECT count(*) FROM container " "WHERE deleted = 1").fetchone()[0], 0) sleep(.00001) broker.put_container('c', 0, normalize_timestamp(time()), 0, 0) with broker.get() as conn: self.assertEqual(conn.execute( "SELECT count(*) FROM container " "WHERE deleted = 0").fetchone()[0], 0) self.assertEqual(conn.execute( "SELECT count(*) FROM container " "WHERE deleted = 1").fetchone()[0], 1) broker.reclaim(normalize_timestamp(time() - 999), time()) with broker.get() as conn: self.assertEqual(conn.execute( "SELECT count(*) FROM container " "WHERE deleted = 0").fetchone()[0], 0) self.assertEqual(conn.execute( "SELECT count(*) FROM container " "WHERE deleted = 1").fetchone()[0], 1) sleep(.00001) broker.reclaim(normalize_timestamp(time()), time()) with broker.get() as conn: self.assertEqual(conn.execute( "SELECT count(*) FROM container " "WHERE deleted = 0").fetchone()[0], 0) self.assertEqual(conn.execute( "SELECT count(*) FROM container " "WHERE deleted = 1").fetchone()[0], 0) # Test reclaim after deletion. Create 3 test containers broker.put_container('x', 0, 0, 0, 0) broker.put_container('y', 0, 0, 0, 0) broker.put_container('z', 0, 0, 0, 0) broker.reclaim(normalize_timestamp(time()), time()) # self.assertEqual(len(res), 2) # self.assert_(isinstance(res, tuple)) # containers, account_name = res # self.assert_(containers is None) # self.assert_(account_name is None) # Now delete the account broker.delete_db(normalize_timestamp(time())) broker.reclaim(normalize_timestamp(time()), time())
def test_account_stat_get_data(self): stat = db_stats_collector.AccountStatsCollector(self.conf) account_db = AccountBroker("%s/acc.db" % self.accounts, account='test_acc') account_db.initialize() account_db.put_container('test_container', time.time(), None, 10, 1000) info = stat.get_data("%s/acc.db" % self.accounts) self.assertEquals('''"test_acc",1,10,1000\n''', info)
def test_account_stat_get_data(self): stat = db_stats_collector.AccountStatsCollector(self.conf) account_db = AccountBroker("%s/acc.db" % self.accounts, account='test_acc') account_db.initialize() account_db.put_container('test_container', time.time(), None, 10, 1000, 1) info = stat.get_data("%s/acc.db" % self.accounts) self.assertEquals('''"test_acc",1,10,1000\n''', info)
def test_empty(self): # Test AccountBroker.empty broker = AccountBroker(':memory:', account='a') broker.initialize(normalize_timestamp('1')) self.assert_(broker.empty()) broker.put_container('o', normalize_timestamp(time()), 0, 0, 0) self.assert_(not broker.empty()) sleep(.00001) broker.put_container('o', 0, normalize_timestamp(time()), 0, 0) self.assert_(broker.empty())
def test_empty(self): # Test AccountBroker.empty broker = AccountBroker(':memory:', account='a') broker.initialize(Timestamp('1').internal) self.assert_(broker.empty()) broker.put_container('o', Timestamp(time()).internal, 0, 0, 0, POLICIES.default.idx) self.assert_(not broker.empty()) sleep(.00001) broker.put_container('o', 0, Timestamp(time()).internal, 0, 0, POLICIES.default.idx) self.assert_(broker.empty())
def test_get_policy_stats(self): ts = (Timestamp(t).internal for t in itertools.count(int(time()))) broker = AccountBroker(':memory:', account='a') broker.initialize(ts.next()) # check empty policy_stats self.assertTrue(broker.empty()) policy_stats = broker.get_policy_stats() self.assertEqual(policy_stats, {}) # add some empty containers for policy in POLICIES: container_name = 'c-%s' % policy.name put_timestamp = ts.next() broker.put_container(container_name, put_timestamp, 0, 0, 0, policy.idx) policy_stats = broker.get_policy_stats() stats = policy_stats[policy.idx] self.assertEqual(stats['object_count'], 0) self.assertEqual(stats['bytes_used'], 0) # update the containers object & byte count for policy in POLICIES: container_name = 'c-%s' % policy.name put_timestamp = ts.next() count = policy.idx * 100 # good as any integer broker.put_container(container_name, put_timestamp, 0, count, count, policy.idx) policy_stats = broker.get_policy_stats() stats = policy_stats[policy.idx] self.assertEqual(stats['object_count'], count) self.assertEqual(stats['bytes_used'], count) # check all the policy_stats at once for policy_index, stats in policy_stats.items(): policy = POLICIES[policy_index] count = policy.idx * 100 # coupled with policy for test self.assertEqual(stats['object_count'], count) self.assertEqual(stats['bytes_used'], count) # now delete the containers one by one for policy in POLICIES: container_name = 'c-%s' % policy.name delete_timestamp = ts.next() broker.put_container(container_name, 0, delete_timestamp, 0, 0, policy.idx) policy_stats = broker.get_policy_stats() stats = policy_stats[policy.idx] self.assertEqual(stats['object_count'], 0) self.assertEqual(stats['bytes_used'], 0)
def test_get_info(self): # Test AccountBroker.get_info broker = AccountBroker(':memory:', account='test1') broker.initialize(Timestamp('1').internal) info = broker.get_info() self.assertEqual(info['account'], 'test1') self.assertEqual(info['hash'], '00000000000000000000000000000000') self.assertEqual(info['put_timestamp'], Timestamp(1).internal) self.assertEqual(info['delete_timestamp'], '0') if self.__class__ == TestAccountBrokerBeforeMetadata: self.assertEqual(info['status_changed_at'], '0') else: self.assertEqual(info['status_changed_at'], Timestamp(1).internal) info = broker.get_info() self.assertEqual(info['container_count'], 0) broker.put_container('c1', Timestamp(time()).internal, 0, 0, 0, POLICIES.default.idx) info = broker.get_info() self.assertEqual(info['container_count'], 1) sleep(.00001) broker.put_container('c2', Timestamp(time()).internal, 0, 0, 0, POLICIES.default.idx) info = broker.get_info() self.assertEqual(info['container_count'], 2) sleep(.00001) broker.put_container('c2', Timestamp(time()).internal, 0, 0, 0, POLICIES.default.idx) info = broker.get_info() self.assertEqual(info['container_count'], 2) sleep(.00001) broker.put_container('c1', 0, Timestamp(time()).internal, 0, 0, POLICIES.default.idx) info = broker.get_info() self.assertEqual(info['container_count'], 1) sleep(.00001) broker.put_container('c2', 0, Timestamp(time()).internal, 0, 0, POLICIES.default.idx) info = broker.get_info() self.assertEqual(info['container_count'], 0)
def test_is_status_deleted(self): # Test AccountBroker.is_status_deleted broker1 = AccountBroker(':memory:', account='a') broker1.initialize(Timestamp(time()).internal) self.assert_(not broker1.is_status_deleted()) broker1.delete_db(Timestamp(time()).internal) self.assert_(broker1.is_status_deleted()) broker2 = AccountBroker(':memory:', account='a') broker2.initialize(Timestamp(time()).internal) # Set delete_timestamp greater than put_timestamp broker2.merge_timestamps(time(), Timestamp(time()).internal, Timestamp(time() + 999).internal) self.assert_(broker2.is_status_deleted())
def test_empty(self): # Test AccountBroker.empty broker = AccountBroker(':memory:', account='a') broker.initialize(Timestamp('1').internal) self.assert_(broker.empty()) broker.put_container('o', Timestamp(time()).internal, 0, 0, 0, POLICIES.default.idx) self.assert_(not broker.empty()) sleep(.00001) broker.put_container('o', 0, Timestamp(time()).internal, 0, 0, POLICIES.default.idx) self.assert_(broker.empty())
def test_exception(self): # Test AccountBroker throwing a conn away after exception first_conn = None broker = AccountBroker(':memory:', account='a') broker.initialize(normalize_timestamp('1')) with broker.get() as conn: first_conn = conn try: with broker.get() as conn: self.assertEqual(first_conn, conn) raise Exception('OMG') except Exception: pass self.assert_(broker.conn is None)
def test_is_status_deleted(self): # Test AccountBroker.is_status_deleted broker1 = AccountBroker(':memory:', account='a') broker1.initialize(Timestamp(time()).internal) self.assert_(not broker1.is_status_deleted()) broker1.delete_db(Timestamp(time()).internal) self.assert_(broker1.is_status_deleted()) broker2 = AccountBroker(':memory:', account='a') broker2.initialize(Timestamp(time()).internal) # Set delete_timestamp greater than put_timestamp broker2.merge_timestamps( time(), Timestamp(time()).internal, Timestamp(time() + 999).internal) self.assert_(broker2.is_status_deleted())
def setUp(self): self._imported_create_account_stat_table = \ AccountBroker.create_account_stat_table AccountBroker.create_account_stat_table = \ premetadata_create_account_stat_table broker = AccountBroker(':memory:', account='a') broker.initialize(normalize_timestamp('1')) exc = None with broker.get() as conn: try: conn.execute('SELECT metadata FROM account_stat') except BaseException as err: exc = err self.assert_('no such column: metadata' in str(exc))
def test_creation(self): # Test AccountBroker.__init__ broker = AccountBroker(':memory:', account='a') self.assertEqual(broker.db_file, ':memory:') got_exc = False try: with broker.get() as conn: pass except Exception: got_exc = True self.assert_(got_exc) broker.initialize(normalize_timestamp('1')) with broker.get() as conn: curs = conn.cursor() curs.execute('SELECT 1') self.assertEqual(curs.fetchall()[0][0], 1)
def _gen_account_stat(self): stat = db_stats_collector.AccountStatsCollector(self.conf) output_data = set() for i in range(10): account_db = AccountBroker("%s/stats-201001010%s-%s.db" % (self.accounts, i, uuid.uuid4().hex), account='test_acc_%s' % i) account_db.initialize() account_db.put_container('test_container', time.time(), None, 10, 1000, 1) # this will "commit" the data account_db.get_info() output_data.add('''"test_acc_%s",1,10,1000''' % i), self.assertEqual(len(output_data), 10) return stat, output_data
def _gen_account_stat(self): stat = db_stats_collector.AccountStatsCollector(self.conf) output_data = set() for i in range(10): account_db = AccountBroker("%s/stats-201001010%s-%s.db" % (self.accounts, i, uuid.uuid4().hex), account='test_acc_%s' % i) account_db.initialize() account_db.put_container('test_container', time.time(), None, 10, 1000) # this will "commit" the data account_db.get_info() output_data.add('''"test_acc_%s",1,10,1000''' % i), self.assertEqual(len(output_data), 10) return stat, output_data
def test_get_info(self): # Test AccountBroker.get_info broker = AccountBroker(':memory:', account='test1') broker.initialize(Timestamp('1').internal) info = broker.get_info() self.assertEqual(info['account'], 'test1') self.assertEqual(info['hash'], '00000000000000000000000000000000') self.assertEqual(info['put_timestamp'], Timestamp(1).internal) self.assertEqual(info['delete_timestamp'], '0') if self.__class__ == TestAccountBrokerBeforeMetadata: self.assertEqual(info['status_changed_at'], '0') else: self.assertEqual(info['status_changed_at'], Timestamp(1).internal) info = broker.get_info() self.assertEqual(info['container_count'], 0) broker.put_container('c1', Timestamp(time()).internal, 0, 0, 0, POLICIES.default.idx) info = broker.get_info() self.assertEqual(info['container_count'], 1) sleep(.00001) broker.put_container('c2', Timestamp(time()).internal, 0, 0, 0, POLICIES.default.idx) info = broker.get_info() self.assertEqual(info['container_count'], 2) sleep(.00001) broker.put_container('c2', Timestamp(time()).internal, 0, 0, 0, POLICIES.default.idx) info = broker.get_info() self.assertEqual(info['container_count'], 2) sleep(.00001) broker.put_container('c1', 0, Timestamp(time()).internal, 0, 0, POLICIES.default.idx) info = broker.get_info() self.assertEqual(info['container_count'], 1) sleep(.00001) broker.put_container('c2', 0, Timestamp(time()).internal, 0, 0, POLICIES.default.idx) info = broker.get_info() self.assertEqual(info['container_count'], 0)
def test_merge_items(self): broker1 = AccountBroker(':memory:', account='a') broker1.initialize(normalize_timestamp('1')) broker2 = AccountBroker(':memory:', account='a') broker2.initialize(normalize_timestamp('1')) broker1.put_container('a', normalize_timestamp(1), 0, 0, 0) broker1.put_container('b', normalize_timestamp(2), 0, 0, 0) id = broker1.get_info()['id'] broker2.merge_items(broker1.get_items_since( broker2.get_sync(id), 1000), id) items = broker2.get_items_since(-1, 1000) self.assertEqual(len(items), 2) self.assertEqual(['a', 'b'], sorted([rec['name'] for rec in items])) broker1.put_container('c', normalize_timestamp(3), 0, 0, 0) broker2.merge_items(broker1.get_items_since( broker2.get_sync(id), 1000), id) items = broker2.get_items_since(-1, 1000) self.assertEqual(len(items), 3) self.assertEqual(['a', 'b', 'c'], sorted([rec['name'] for rec in items]))
def test_merge_items(self): broker1 = AccountBroker(':memory:', account='a') broker1.initialize(normalize_timestamp('1')) broker2 = AccountBroker(':memory:', account='a') broker2.initialize(normalize_timestamp('1')) broker1.put_container('a', normalize_timestamp(1), 0, 0, 0) broker1.put_container('b', normalize_timestamp(2), 0, 0, 0) id = broker1.get_info()['id'] broker2.merge_items( broker1.get_items_since(broker2.get_sync(id), 1000), id) items = broker2.get_items_since(-1, 1000) self.assertEquals(len(items), 2) self.assertEquals(['a', 'b'], sorted([rec['name'] for rec in items])) broker1.put_container('c', normalize_timestamp(3), 0, 0, 0) broker2.merge_items( broker1.get_items_since(broker2.get_sync(id), 1000), id) items = broker2.get_items_since(-1, 1000) self.assertEquals(len(items), 3) self.assertEquals(['a', 'b', 'c'], sorted([rec['name'] for rec in items]))
def test_creation(self): # Test AccountBroker.__init__ broker = AccountBroker(':memory:', account='a') self.assertEqual(broker.db_file, ':memory:') try: with broker.get() as conn: pass except DatabaseConnectionError as e: self.assertTrue(hasattr(e, 'path')) self.assertEquals(e.path, ':memory:') self.assertTrue(hasattr(e, 'msg')) self.assertEquals(e.msg, "DB doesn't exist") except Exception as e: self.fail("Unexpected exception raised: %r" % e) else: self.fail("Expected a DatabaseConnectionError exception") broker.initialize(Timestamp('1').internal) with broker.get() as conn: curs = conn.cursor() curs.execute('SELECT 1') self.assertEqual(curs.fetchall()[0][0], 1)
def test_creation(self): # Test AccountBroker.__init__ broker = AccountBroker(':memory:', account='a') self.assertEqual(broker.db_file, ':memory:') try: with broker.get() as conn: pass except DatabaseConnectionError as e: self.assertTrue(hasattr(e, 'path')) self.assertEquals(e.path, ':memory:') self.assertTrue(hasattr(e, 'msg')) self.assertEquals(e.msg, "DB doesn't exist") except Exception as e: self.fail("Unexpected exception raised: %r" % e) else: self.fail("Expected a DatabaseConnectionError exception") broker.initialize(Timestamp('1').internal) with broker.get() as conn: curs = conn.cursor() curs.execute('SELECT 1') self.assertEqual(curs.fetchall()[0][0], 1)
def test_delete_container(self): # Test AccountBroker.delete_container broker = AccountBroker(':memory:', account='a') broker.initialize(normalize_timestamp('1')) broker.put_container('o', normalize_timestamp(time()), 0, 0, 0) with broker.get() as conn: self.assertEquals( conn.execute("SELECT count(*) FROM container " "WHERE deleted = 0").fetchone()[0], 1) self.assertEquals( conn.execute("SELECT count(*) FROM container " "WHERE deleted = 1").fetchone()[0], 0) sleep(.00001) broker.put_container('o', 0, normalize_timestamp(time()), 0, 0) with broker.get() as conn: self.assertEquals( conn.execute("SELECT count(*) FROM container " "WHERE deleted = 0").fetchone()[0], 0) self.assertEquals( conn.execute("SELECT count(*) FROM container " "WHERE deleted = 1").fetchone()[0], 1)
def test_chexor(self): broker = AccountBroker(':memory:', account='a') broker.initialize(normalize_timestamp('1')) broker.put_container('a', normalize_timestamp(1), normalize_timestamp(0), 0, 0) broker.put_container('b', normalize_timestamp(2), normalize_timestamp(0), 0, 0) hasha = hashlib.md5( '%s-%s' % ('a', '0000000001.00000-0000000000.00000-0-0')).digest() hashb = hashlib.md5( '%s-%s' % ('b', '0000000002.00000-0000000000.00000-0-0')).digest() hashc = \ ''.join(('%02x' % (ord(a) ^ ord(b)) for a, b in zip(hasha, hashb))) self.assertEqual(broker.get_info()['hash'], hashc) broker.put_container('b', normalize_timestamp(3), normalize_timestamp(0), 0, 0) hashb = hashlib.md5( '%s-%s' % ('b', '0000000003.00000-0000000000.00000-0-0')).digest() hashc = \ ''.join(('%02x' % (ord(a) ^ ord(b)) for a, b in zip(hasha, hashb))) self.assertEqual(broker.get_info()['hash'], hashc)
def test_delete_container(self): # Test AccountBroker.delete_container broker = AccountBroker(':memory:', account='a') broker.initialize(normalize_timestamp('1')) broker.put_container('o', normalize_timestamp(time()), 0, 0, 0) with broker.get() as conn: self.assertEqual(conn.execute( "SELECT count(*) FROM container " "WHERE deleted = 0").fetchone()[0], 1) self.assertEqual(conn.execute( "SELECT count(*) FROM container " "WHERE deleted = 1").fetchone()[0], 0) sleep(.00001) broker.put_container('o', 0, normalize_timestamp(time()), 0, 0) with broker.get() as conn: self.assertEqual(conn.execute( "SELECT count(*) FROM container " "WHERE deleted = 0").fetchone()[0], 0) self.assertEqual(conn.execute( "SELECT count(*) FROM container " "WHERE deleted = 1").fetchone()[0], 1)
def test_delete_db_status(self): ts = (Timestamp(t).internal for t in itertools.count(int(time()))) start = ts.next() broker = AccountBroker(':memory:', account='a') broker.initialize(start) info = broker.get_info() self.assertEqual(info['put_timestamp'], Timestamp(start).internal) self.assert_(Timestamp(info['created_at']) >= start) self.assertEqual(info['delete_timestamp'], '0') if self.__class__ == TestAccountBrokerBeforeMetadata: self.assertEqual(info['status_changed_at'], '0') else: self.assertEqual(info['status_changed_at'], Timestamp(start).internal) # delete it delete_timestamp = ts.next() broker.delete_db(delete_timestamp) info = broker.get_info() self.assertEqual(info['put_timestamp'], Timestamp(start).internal) self.assert_(Timestamp(info['created_at']) >= start) self.assertEqual(info['delete_timestamp'], delete_timestamp) self.assertEqual(info['status_changed_at'], delete_timestamp)
def setUp(self): self._imported_create_container_table = \ AccountBroker.create_container_table AccountBroker.create_container_table = \ prespi_create_container_table self._imported_initialize = AccountBroker._initialize AccountBroker._initialize = prespi_AccountBroker_initialize broker = AccountBroker(':memory:', account='a') broker.initialize(Timestamp('1').internal) exc = None with broker.get() as conn: try: conn.execute('SELECT storage_policy_index FROM container') except BaseException as err: exc = err self.assert_('no such column: storage_policy_index' in str(exc)) with broker.get() as conn: try: conn.execute('SELECT * FROM policy_stat') except sqlite3.OperationalError as err: self.assert_('no such table: policy_stat' in str(err)) else: self.fail('database created with policy_stat table')
def test_delete_db_status(self): ts = (Timestamp(t).internal for t in itertools.count(int(time()))) start = ts.next() broker = AccountBroker(':memory:', account='a') broker.initialize(start) info = broker.get_info() self.assertEqual(info['put_timestamp'], Timestamp(start).internal) self.assert_(Timestamp(info['created_at']) >= start) self.assertEqual(info['delete_timestamp'], '0') if self.__class__ == TestAccountBrokerBeforeMetadata: self.assertEqual(info['status_changed_at'], '0') else: self.assertEqual(info['status_changed_at'], Timestamp(start).internal) # delete it delete_timestamp = ts.next() broker.delete_db(delete_timestamp) info = broker.get_info() self.assertEqual(info['put_timestamp'], Timestamp(start).internal) self.assert_(Timestamp(info['created_at']) >= start) self.assertEqual(info['delete_timestamp'], delete_timestamp) self.assertEqual(info['status_changed_at'], delete_timestamp)
def setUp(self): self._imported_create_container_table = \ AccountBroker.create_container_table AccountBroker.create_container_table = \ prespi_create_container_table self._imported_initialize = AccountBroker._initialize AccountBroker._initialize = prespi_AccountBroker_initialize broker = AccountBroker(':memory:', account='a') broker.initialize(Timestamp('1').internal) exc = None with broker.get() as conn: try: conn.execute('SELECT storage_policy_index FROM container') except BaseException as err: exc = err self.assert_('no such column: storage_policy_index' in str(exc)) with broker.get() as conn: try: conn.execute('SELECT * FROM policy_stat') except sqlite3.OperationalError as err: self.assert_('no such table: policy_stat' in str(err)) else: self.fail('database created with policy_stat table')
def test_chexor(self): broker = AccountBroker(':memory:', account='a') broker.initialize(normalize_timestamp('1')) broker.put_container('a', normalize_timestamp(1), normalize_timestamp(0), 0, 0) broker.put_container('b', normalize_timestamp(2), normalize_timestamp(0), 0, 0) hasha = hashlib.md5( '%s-%s' % ('a', '0000000001.00000-0000000000.00000-0-0') ).digest() hashb = hashlib.md5( '%s-%s' % ('b', '0000000002.00000-0000000000.00000-0-0') ).digest() hashc = \ ''.join(('%02x' % (ord(a) ^ ord(b)) for a, b in zip(hasha, hashb))) self.assertEqual(broker.get_info()['hash'], hashc) broker.put_container('b', normalize_timestamp(3), normalize_timestamp(0), 0, 0) hashb = hashlib.md5( '%s-%s' % ('b', '0000000003.00000-0000000000.00000-0-0') ).digest() hashc = \ ''.join(('%02x' % (ord(a) ^ ord(b)) for a, b in zip(hasha, hashb))) self.assertEqual(broker.get_info()['hash'], hashc)
def test_half_upgraded_database(self, tempdir): db_path = os.path.join(tempdir, 'account.db') ts = itertools.count() ts = (Timestamp(t).internal for t in itertools.count(int(time()))) broker = AccountBroker(db_path, account='a') broker.initialize(ts.next()) self.assertTrue(broker.empty()) # add a container (to pending file) broker.put_container('c', ts.next(), 0, 0, 0, POLICIES.default.idx) real_get = broker.get called = [] @contextmanager def mock_get(): with real_get() as conn: def mock_executescript(script): if called: raise Exception('kaboom!') called.append(script) conn.executescript = mock_executescript yield conn broker.get = mock_get try: broker._commit_puts() except Exception: pass else: self.fail('mock exception was not raised') self.assertEqual(len(called), 1) self.assert_('CREATE TABLE policy_stat' in called[0]) # nothing was committed broker = AccountBroker(db_path, account='a') with broker.get() as conn: try: conn.execute('SELECT * FROM policy_stat') except sqlite3.OperationalError as err: self.assert_('no such table: policy_stat' in str(err)) else: self.fail('half upgraded database!') container_count = conn.execute( 'SELECT count(*) FROM container').fetchone()[0] self.assertEqual(container_count, 0) # try again to commit puts self.assertFalse(broker.empty()) # full migration successful with broker.get() as conn: conn.execute('SELECT * FROM policy_stat') conn.execute('SELECT storage_policy_index FROM container')
def test_container_table_migration(self, tempdir): db_path = os.path.join(tempdir, 'account.db') # first init an acct DB without the policy_stat table present broker = AccountBroker(db_path, account='a') broker.initialize(Timestamp('1').internal) with broker.get() as conn: try: conn.execute(''' SELECT storage_policy_index FROM container ''').fetchone()[0] except sqlite3.OperationalError as err: # confirm that the table doesn't have this column self.assert_('no such column: storage_policy_index' in str(err)) else: self.fail('broker did not raise sqlite3.OperationalError ' 'trying to select from storage_policy_index ' 'from container table!') # manually insert an existing row to avoid migration with broker.get() as conn: conn.execute(''' INSERT INTO container (name, put_timestamp, delete_timestamp, object_count, bytes_used, deleted) VALUES (?, ?, ?, ?, ?, ?) ''', ('test_name', Timestamp(time()).internal, 0, 1, 2, 0)) conn.commit() # make sure we can iter containers without the migration for c in broker.list_containers_iter(1, None, None, None, None): self.assertEqual(c, ('test_name', 1, 2, 0)) # stats table is mysteriously empty... stats = broker.get_policy_stats() self.assertEqual(len(stats), 0) # now do a PUT with a different value for storage_policy_index # which will update the DB schema as well as update policy_stats # for legacy containers in the DB (those without an SPI) other_policy = [p for p in POLICIES if p.idx != 0][0] broker.put_container('test_second', Timestamp(time()).internal, 0, 3, 4, other_policy.idx) broker._commit_puts_stale_ok() with broker.get() as conn: rows = conn.execute(''' SELECT name, storage_policy_index FROM container ''').fetchall() for row in rows: if row[0] == 'test_name': self.assertEqual(row[1], 0) else: self.assertEqual(row[1], other_policy.idx) # we should have stats for both containers stats = broker.get_policy_stats() self.assertEqual(len(stats), 2) self.assertEqual(stats[0]['object_count'], 1) self.assertEqual(stats[0]['bytes_used'], 2) self.assertEqual(stats[1]['object_count'], 3) self.assertEqual(stats[1]['bytes_used'], 4) # now lets delete a container and make sure policy_stats is OK with broker.get() as conn: conn.execute(''' DELETE FROM container WHERE name = ? ''', ('test_name',)) conn.commit() stats = broker.get_policy_stats() self.assertEqual(len(stats), 2) self.assertEqual(stats[0]['object_count'], 0) self.assertEqual(stats[0]['bytes_used'], 0) self.assertEqual(stats[1]['object_count'], 3) self.assertEqual(stats[1]['bytes_used'], 4)
def test_list_containers_iter(self): # Test AccountBroker.list_containers_iter broker = AccountBroker(':memory:', account='a') broker.initialize(normalize_timestamp('1')) for cont1 in xrange(4): for cont2 in xrange(125): broker.put_container('%d-%04d' % (cont1, cont2), normalize_timestamp(time()), 0, 0, 0) for cont in xrange(125): broker.put_container('2-0051-%04d' % cont, normalize_timestamp(time()), 0, 0, 0) for cont in xrange(125): broker.put_container('3-%04d-0049' % cont, normalize_timestamp(time()), 0, 0, 0) listing = broker.list_containers_iter(100, '', None, None, '') self.assertEqual(len(listing), 100) self.assertEqual(listing[0][0], '0-0000') self.assertEqual(listing[-1][0], '0-0099') listing = broker.list_containers_iter(100, '', '0-0050', None, '') self.assertEqual(len(listing), 50) self.assertEqual(listing[0][0], '0-0000') self.assertEqual(listing[-1][0], '0-0049') listing = broker.list_containers_iter(100, '0-0099', None, None, '') self.assertEqual(len(listing), 100) self.assertEqual(listing[0][0], '0-0100') self.assertEqual(listing[-1][0], '1-0074') listing = broker.list_containers_iter(55, '1-0074', None, None, '') self.assertEqual(len(listing), 55) self.assertEqual(listing[0][0], '1-0075') self.assertEqual(listing[-1][0], '2-0004') listing = broker.list_containers_iter(10, '', None, '0-01', '') self.assertEqual(len(listing), 10) self.assertEqual(listing[0][0], '0-0100') self.assertEqual(listing[-1][0], '0-0109') listing = broker.list_containers_iter(10, '', None, '0-01', '-') self.assertEqual(len(listing), 10) self.assertEqual(listing[0][0], '0-0100') self.assertEqual(listing[-1][0], '0-0109') listing = broker.list_containers_iter(10, '', None, '0-', '-') self.assertEqual(len(listing), 10) self.assertEqual(listing[0][0], '0-0000') self.assertEqual(listing[-1][0], '0-0009') listing = broker.list_containers_iter(10, '', None, '', '-') self.assertEqual(len(listing), 4) self.assertEqual([row[0] for row in listing], ['0-', '1-', '2-', '3-']) listing = broker.list_containers_iter(10, '2-', None, None, '-') self.assertEqual(len(listing), 1) self.assertEqual([row[0] for row in listing], ['3-']) listing = broker.list_containers_iter(10, '', None, '2', '-') self.assertEqual(len(listing), 1) self.assertEqual([row[0] for row in listing], ['2-']) listing = broker.list_containers_iter(10, '2-0050', None, '2-', '-') self.assertEqual(len(listing), 10) self.assertEqual(listing[0][0], '2-0051') self.assertEqual(listing[1][0], '2-0051-') self.assertEqual(listing[2][0], '2-0052') self.assertEqual(listing[-1][0], '2-0059') listing = broker.list_containers_iter(10, '3-0045', None, '3-', '-') self.assertEqual(len(listing), 10) self.assertEqual([row[0] for row in listing], ['3-0045-', '3-0046', '3-0046-', '3-0047', '3-0047-', '3-0048', '3-0048-', '3-0049', '3-0049-', '3-0050']) broker.put_container('3-0049-', normalize_timestamp(time()), 0, 0, 0) listing = broker.list_containers_iter(10, '3-0048', None, None, None) self.assertEqual(len(listing), 10) self.assertEqual([row[0] for row in listing], ['3-0048-0049', '3-0049', '3-0049-', '3-0049-0049', '3-0050', '3-0050-0049', '3-0051', '3-0051-0049', '3-0052', '3-0052-0049']) listing = broker.list_containers_iter(10, '3-0048', None, '3-', '-') self.assertEqual(len(listing), 10) self.assertEqual([row[0] for row in listing], ['3-0048-', '3-0049', '3-0049-', '3-0050', '3-0050-', '3-0051', '3-0051-', '3-0052', '3-0052-', '3-0053']) listing = broker.list_containers_iter(10, None, None, '3-0049-', '-') self.assertEqual(len(listing), 2) self.assertEqual([row[0] for row in listing], ['3-0049-', '3-0049-0049'])
def test_put_container(self): # Test AccountBroker.put_container broker = AccountBroker(':memory:', account='a') broker.initialize(normalize_timestamp('1')) # Create initial container timestamp = normalize_timestamp(time()) broker.put_container('"{<container \'&\' name>}"', timestamp, 0, 0, 0) with broker.get() as conn: self.assertEqual(conn.execute( "SELECT name FROM container").fetchone()[0], '"{<container \'&\' name>}"') self.assertEqual(conn.execute( "SELECT put_timestamp FROM container").fetchone()[0], timestamp) self.assertEqual(conn.execute( "SELECT deleted FROM container").fetchone()[0], 0) # Reput same event broker.put_container('"{<container \'&\' name>}"', timestamp, 0, 0, 0) with broker.get() as conn: self.assertEqual(conn.execute( "SELECT name FROM container").fetchone()[0], '"{<container \'&\' name>}"') self.assertEqual(conn.execute( "SELECT put_timestamp FROM container").fetchone()[0], timestamp) self.assertEqual(conn.execute( "SELECT deleted FROM container").fetchone()[0], 0) # Put new event sleep(.00001) timestamp = normalize_timestamp(time()) broker.put_container('"{<container \'&\' name>}"', timestamp, 0, 0, 0) with broker.get() as conn: self.assertEqual(conn.execute( "SELECT name FROM container").fetchone()[0], '"{<container \'&\' name>}"') self.assertEqual(conn.execute( "SELECT put_timestamp FROM container").fetchone()[0], timestamp) self.assertEqual(conn.execute( "SELECT deleted FROM container").fetchone()[0], 0) # Put old event otimestamp = normalize_timestamp(float(timestamp) - 1) broker.put_container('"{<container \'&\' name>}"', otimestamp, 0, 0, 0) with broker.get() as conn: self.assertEqual(conn.execute( "SELECT name FROM container").fetchone()[0], '"{<container \'&\' name>}"') self.assertEqual(conn.execute( "SELECT put_timestamp FROM container").fetchone()[0], timestamp) self.assertEqual(conn.execute( "SELECT deleted FROM container").fetchone()[0], 0) # Put old delete event dtimestamp = normalize_timestamp(float(timestamp) - 1) broker.put_container('"{<container \'&\' name>}"', 0, dtimestamp, 0, 0) with broker.get() as conn: self.assertEqual(conn.execute( "SELECT name FROM container").fetchone()[0], '"{<container \'&\' name>}"') self.assertEqual(conn.execute( "SELECT put_timestamp FROM container").fetchone()[0], timestamp) self.assertEqual(conn.execute( "SELECT delete_timestamp FROM container").fetchone()[0], dtimestamp) self.assertEqual(conn.execute( "SELECT deleted FROM container").fetchone()[0], 0) # Put new delete event sleep(.00001) timestamp = normalize_timestamp(time()) broker.put_container('"{<container \'&\' name>}"', 0, timestamp, 0, 0) with broker.get() as conn: self.assertEqual(conn.execute( "SELECT name FROM container").fetchone()[0], '"{<container \'&\' name>}"') self.assertEqual(conn.execute( "SELECT delete_timestamp FROM container").fetchone()[0], timestamp) self.assertEqual(conn.execute( "SELECT deleted FROM container").fetchone()[0], 1) # Put new event sleep(.00001) timestamp = normalize_timestamp(time()) broker.put_container('"{<container \'&\' name>}"', timestamp, 0, 0, 0) with broker.get() as conn: self.assertEqual(conn.execute( "SELECT name FROM container").fetchone()[0], '"{<container \'&\' name>}"') self.assertEqual(conn.execute( "SELECT put_timestamp FROM container").fetchone()[0], timestamp) self.assertEqual(conn.execute( "SELECT deleted FROM container").fetchone()[0], 0)