def test_delete_container(self): # Test AccountBroker.delete_container broker = AccountBroker(':memory:', account='a') broker.initialize(Timestamp('1').internal) broker.put_container('o', Timestamp(time()).internal, 0, 0, 0, POLICIES.default.idx) with broker.get() as conn: self.assertEqual( conn.execute("SELECT count(*) FROM container " "WHERE deleted = 0").fetchone()[0], 1) self.assertEqual( conn.execute("SELECT count(*) FROM container " "WHERE deleted = 1").fetchone()[0], 0) sleep(.00001) broker.put_container('o', 0, Timestamp(time()).internal, 0, 0, POLICIES.default.idx) with broker.get() as conn: self.assertEqual( conn.execute("SELECT count(*) FROM container " "WHERE deleted = 0").fetchone()[0], 0) self.assertEqual( conn.execute("SELECT count(*) FROM container " "WHERE deleted = 1").fetchone()[0], 1)
def test_policy_table_migration(self, tempdir): db_path = os.path.join(tempdir, 'account.db') # first init an acct DB without the policy_stat table present broker = AccountBroker(db_path, account='a') broker.initialize(Timestamp('1').internal) with broker.get() as conn: try: conn.execute(''' SELECT * FROM policy_stat ''').fetchone()[0] except sqlite3.OperationalError as err: # confirm that the table really isn't there self.assert_('no such table: policy_stat' in str(err)) else: self.fail('broker did not raise sqlite3.OperationalError ' 'trying to select from policy_stat table!') # make sure we can HEAD this thing w/o the table stats = broker.get_policy_stats() self.assertEqual(len(stats), 0) # now do a PUT to create the table broker.put_container('o', Timestamp(time()).internal, 0, 0, 0, POLICIES.default.idx) broker._commit_puts_stale_ok() # now confirm that the table was created with broker.get() as conn: conn.execute('SELECT * FROM policy_stat') stats = broker.get_policy_stats() self.assertEqual(len(stats), 1)
def test_reclaim(self): broker = AccountBroker(':memory:', account='test_account') broker.initialize(Timestamp('1').internal) broker.put_container('c', Timestamp(time()).internal, 0, 0, 0, POLICIES.default.idx) with broker.get() as conn: self.assertEqual(conn.execute( "SELECT count(*) FROM container " "WHERE deleted = 0").fetchone()[0], 1) self.assertEqual(conn.execute( "SELECT count(*) FROM container " "WHERE deleted = 1").fetchone()[0], 0) broker.reclaim(Timestamp(time() - 999).internal, time()) with broker.get() as conn: self.assertEqual(conn.execute( "SELECT count(*) FROM container " "WHERE deleted = 0").fetchone()[0], 1) self.assertEqual(conn.execute( "SELECT count(*) FROM container " "WHERE deleted = 1").fetchone()[0], 0) sleep(.00001) broker.put_container('c', 0, Timestamp(time()).internal, 0, 0, POLICIES.default.idx) with broker.get() as conn: self.assertEqual(conn.execute( "SELECT count(*) FROM container " "WHERE deleted = 0").fetchone()[0], 0) self.assertEqual(conn.execute( "SELECT count(*) FROM container " "WHERE deleted = 1").fetchone()[0], 1) broker.reclaim(Timestamp(time() - 999).internal, time()) with broker.get() as conn: self.assertEqual(conn.execute( "SELECT count(*) FROM container " "WHERE deleted = 0").fetchone()[0], 0) self.assertEqual(conn.execute( "SELECT count(*) FROM container " "WHERE deleted = 1").fetchone()[0], 1) sleep(.00001) broker.reclaim(Timestamp(time()).internal, time()) with broker.get() as conn: self.assertEqual(conn.execute( "SELECT count(*) FROM container " "WHERE deleted = 0").fetchone()[0], 0) self.assertEqual(conn.execute( "SELECT count(*) FROM container " "WHERE deleted = 1").fetchone()[0], 0) # Test reclaim after deletion. Create 3 test containers broker.put_container('x', 0, 0, 0, 0, POLICIES.default.idx) broker.put_container('y', 0, 0, 0, 0, POLICIES.default.idx) broker.put_container('z', 0, 0, 0, 0, POLICIES.default.idx) broker.reclaim(Timestamp(time()).internal, time()) # self.assertEqual(len(res), 2) # self.assert_(isinstance(res, tuple)) # containers, account_name = res # self.assert_(containers is None) # self.assert_(account_name is None) # Now delete the account broker.delete_db(Timestamp(time()).internal) broker.reclaim(Timestamp(time()).internal, time())
def test_policy_table_migration(self, tempdir): db_path = os.path.join(tempdir, 'account.db') # first init an acct DB without the policy_stat table present broker = AccountBroker(db_path, account='a') broker.initialize(Timestamp('1').internal) with broker.get() as conn: try: conn.execute(''' SELECT * FROM policy_stat ''').fetchone()[0] except sqlite3.OperationalError as err: # confirm that the table really isn't there self.assert_('no such table: policy_stat' in str(err)) else: self.fail('broker did not raise sqlite3.OperationalError ' 'trying to select from policy_stat table!') # make sure we can HEAD this thing w/o the table stats = broker.get_policy_stats() self.assertEqual(len(stats), 0) # now do a PUT to create the table broker.put_container('o', Timestamp(time()).internal, 0, 0, 0, POLICIES.default.idx) broker._commit_puts_stale_ok() # now confirm that the table was created with broker.get() as conn: conn.execute('SELECT * FROM policy_stat') stats = broker.get_policy_stats() self.assertEqual(len(stats), 1)
def test_half_upgraded_database(self, tempdir): db_path = os.path.join(tempdir, 'account.db') ts = itertools.count() ts = (Timestamp(t).internal for t in itertools.count(int(time()))) broker = AccountBroker(db_path, account='a') broker.initialize(ts.next()) self.assertTrue(broker.empty()) # add a container (to pending file) broker.put_container('c', ts.next(), 0, 0, 0, POLICIES.default.idx) real_get = broker.get called = [] @contextmanager def mock_get(): with real_get() as conn: def mock_executescript(script): if called: raise Exception('kaboom!') called.append(script) conn.executescript = mock_executescript yield conn broker.get = mock_get try: broker._commit_puts() except Exception: pass else: self.fail('mock exception was not raised') self.assertEqual(len(called), 1) self.assert_('CREATE TABLE policy_stat' in called[0]) # nothing was commited broker = AccountBroker(db_path, account='a') with broker.get() as conn: try: conn.execute('SELECT * FROM policy_stat') except sqlite3.OperationalError as err: self.assert_('no such table: policy_stat' in str(err)) else: self.fail('half upgraded database!') container_count = conn.execute( 'SELECT count(*) FROM container').fetchone()[0] self.assertEqual(container_count, 0) # try again to commit puts self.assertFalse(broker.empty()) # full migration successful with broker.get() as conn: conn.execute('SELECT * FROM policy_stat') conn.execute('SELECT storage_policy_index FROM container')
def test_reclaim(self): broker = AccountBroker(':memory:', account='test_account') broker.initialize(normalize_timestamp('1')) broker.put_container('c', normalize_timestamp(time()), 0, 0, 0) with broker.get() as conn: self.assertEqual(conn.execute( "SELECT count(*) FROM container " "WHERE deleted = 0").fetchone()[0], 1) self.assertEqual(conn.execute( "SELECT count(*) FROM container " "WHERE deleted = 1").fetchone()[0], 0) broker.reclaim(normalize_timestamp(time() - 999), time()) with broker.get() as conn: self.assertEqual(conn.execute( "SELECT count(*) FROM container " "WHERE deleted = 0").fetchone()[0], 1) self.assertEqual(conn.execute( "SELECT count(*) FROM container " "WHERE deleted = 1").fetchone()[0], 0) sleep(.00001) broker.put_container('c', 0, normalize_timestamp(time()), 0, 0) with broker.get() as conn: self.assertEqual(conn.execute( "SELECT count(*) FROM container " "WHERE deleted = 0").fetchone()[0], 0) self.assertEqual(conn.execute( "SELECT count(*) FROM container " "WHERE deleted = 1").fetchone()[0], 1) broker.reclaim(normalize_timestamp(time() - 999), time()) with broker.get() as conn: self.assertEqual(conn.execute( "SELECT count(*) FROM container " "WHERE deleted = 0").fetchone()[0], 0) self.assertEqual(conn.execute( "SELECT count(*) FROM container " "WHERE deleted = 1").fetchone()[0], 1) sleep(.00001) broker.reclaim(normalize_timestamp(time()), time()) with broker.get() as conn: self.assertEqual(conn.execute( "SELECT count(*) FROM container " "WHERE deleted = 0").fetchone()[0], 0) self.assertEqual(conn.execute( "SELECT count(*) FROM container " "WHERE deleted = 1").fetchone()[0], 0) # Test reclaim after deletion. Create 3 test containers broker.put_container('x', 0, 0, 0, 0) broker.put_container('y', 0, 0, 0, 0) broker.put_container('z', 0, 0, 0, 0) broker.reclaim(normalize_timestamp(time()), time()) # self.assertEqual(len(res), 2) # self.assert_(isinstance(res, tuple)) # containers, account_name = res # self.assert_(containers is None) # self.assert_(account_name is None) # Now delete the account broker.delete_db(normalize_timestamp(time())) broker.reclaim(normalize_timestamp(time()), time())
def test_exception(self): # Test AccountBroker throwing a conn away after exception first_conn = None broker = AccountBroker(':memory:', account='a') broker.initialize(normalize_timestamp('1')) with broker.get() as conn: first_conn = conn try: with broker.get() as conn: self.assertEqual(first_conn, conn) raise Exception('OMG') except Exception: pass self.assert_(broker.conn is None)
def test_policy_stats_tracking(self): ts = (Timestamp(t).internal for t in itertools.count(int(time()))) broker = AccountBroker(':memory:', account='a') broker.initialize(ts.next()) # policy 0 broker.put_container('con1', ts.next(), 0, 12, 2798641, 0) broker.put_container('con1', ts.next(), 0, 13, 8156441, 0) # policy 1 broker.put_container('con2', ts.next(), 0, 7, 5751991, 1) broker.put_container('con2', ts.next(), 0, 8, 6085379, 1) stats = broker.get_policy_stats() self.assertEqual(len(stats), 2) self.assertEqual(stats[0]['object_count'], 13) self.assertEqual(stats[0]['bytes_used'], 8156441) self.assertEqual(stats[1]['object_count'], 8) self.assertEqual(stats[1]['bytes_used'], 6085379) # Break encapsulation here to make sure that there's only 2 rows in # the stats table. It's possible that there could be 4 rows (one per # put_container) but that they came out in the right order so that # get_policy_stats() collapsed them down to the right number. To prove # that's not so, we have to go peek at the broker's internals. with broker.get() as conn: nrows = conn.execute( "SELECT COUNT(*) FROM policy_stat").fetchall()[0][0] self.assertEqual(nrows, 2)
def test_load_old_pending_puts(self): # pending puts from pre-storage-policy account brokers won't contain # the storage policy index tempdir = mkdtemp() broker_path = os.path.join(tempdir, 'test-load-old.db') try: broker = AccountBroker(broker_path, account='real') broker.initialize(Timestamp(1).internal) with open(broker_path + '.pending', 'a+b') as pending: pending.write(':') pending.write(pickle.dumps( # name, put_timestamp, delete_timestamp, object_count, # bytes_used, deleted ('oldcon', Timestamp(200).internal, Timestamp(0).internal, 896, 9216695, 0)).encode('base64')) broker._commit_puts() with broker.get() as conn: results = list(conn.execute(''' SELECT name, storage_policy_index FROM container ''')) self.assertEqual(len(results), 1) self.assertEqual(dict(results[0]), {'name': 'oldcon', 'storage_policy_index': 0}) finally: rmtree(tempdir)
def test_load_old_pending_puts(self): # pending puts from pre-storage-policy account brokers won't contain # the storage policy index tempdir = mkdtemp() broker_path = os.path.join(tempdir, 'test-load-old.db') try: broker = AccountBroker(broker_path, account='real') broker.initialize(Timestamp(1).internal) with open(broker_path + '.pending', 'a+b') as pending: pending.write(':') pending.write( pickle.dumps( # name, put_timestamp, delete_timestamp, object_count, # bytes_used, deleted ('oldcon', Timestamp(200).internal, Timestamp(0).internal, 896, 9216695, 0 )).encode('base64')) broker._commit_puts() with broker.get() as conn: results = list( conn.execute(''' SELECT name, storage_policy_index FROM container ''')) self.assertEqual(len(results), 1) self.assertEqual(dict(results[0]), { 'name': 'oldcon', 'storage_policy_index': 0 }) finally: rmtree(tempdir)
def test_creation(self): # Test AccountBroker.__init__ broker = AccountBroker(':memory:', account='a') self.assertEqual(broker.db_file, ':memory:') got_exc = False try: with broker.get() as conn: pass except Exception: got_exc = True self.assert_(got_exc) broker.initialize(normalize_timestamp('1')) with broker.get() as conn: curs = conn.cursor() curs.execute('SELECT 1') self.assertEqual(curs.fetchall()[0][0], 1)
def tearDown(self): AccountBroker.create_account_stat_table = \ self._imported_create_account_stat_table broker = AccountBroker(':memory:', account='a') broker.initialize(normalize_timestamp('1')) with broker.get() as conn: conn.execute('SELECT metadata FROM account_stat')
def test_policy_stats_tracking(self): ts = (Timestamp(t).internal for t in itertools.count(int(time()))) broker = AccountBroker(':memory:', account='a') broker.initialize(ts.next()) # policy 0 broker.put_container('con1', ts.next(), 0, 12, 2798641, 0) broker.put_container('con1', ts.next(), 0, 13, 8156441, 0) # policy 1 broker.put_container('con2', ts.next(), 0, 7, 5751991, 1) broker.put_container('con2', ts.next(), 0, 8, 6085379, 1) stats = broker.get_policy_stats() self.assertEqual(len(stats), 2) self.assertEqual(stats[0]['object_count'], 13) self.assertEqual(stats[0]['bytes_used'], 8156441) self.assertEqual(stats[1]['object_count'], 8) self.assertEqual(stats[1]['bytes_used'], 6085379) # Break encapsulation here to make sure that there's only 2 rows in # the stats table. It's possible that there could be 4 rows (one per # put_container) but that they came out in the right order so that # get_policy_stats() collapsed them down to the right number. To prove # that's not so, we have to go peek at the broker's internals. with broker.get() as conn: nrows = conn.execute( "SELECT COUNT(*) FROM policy_stat").fetchall()[0][0] self.assertEqual(nrows, 2)
def tearDown(self): AccountBroker.create_container_table = \ self._imported_create_container_table AccountBroker._initialize = self._imported_initialize broker = AccountBroker(':memory:', account='a') broker.initialize(Timestamp('1').internal) with broker.get() as conn: conn.execute('SELECT storage_policy_index FROM container')
def tearDown(self): AccountBroker.create_container_table = \ self._imported_create_container_table AccountBroker._initialize = self._imported_initialize broker = AccountBroker(':memory:', account='a') broker.initialize(Timestamp('1').internal) with broker.get() as conn: conn.execute('SELECT storage_policy_index FROM container')
def test_delete_container(self): # Test AccountBroker.delete_container broker = AccountBroker(':memory:', account='a') broker.initialize(normalize_timestamp('1')) broker.put_container('o', normalize_timestamp(time()), 0, 0, 0) with broker.get() as conn: self.assertEquals( conn.execute("SELECT count(*) FROM container " "WHERE deleted = 0").fetchone()[0], 1) self.assertEquals( conn.execute("SELECT count(*) FROM container " "WHERE deleted = 1").fetchone()[0], 0) sleep(.00001) broker.put_container('o', 0, normalize_timestamp(time()), 0, 0) with broker.get() as conn: self.assertEquals( conn.execute("SELECT count(*) FROM container " "WHERE deleted = 0").fetchone()[0], 0) self.assertEquals( conn.execute("SELECT count(*) FROM container " "WHERE deleted = 1").fetchone()[0], 1)
def test_creation(self): # Test AccountBroker.__init__ broker = AccountBroker(':memory:', account='a') self.assertEqual(broker.db_file, ':memory:') try: with broker.get() as conn: pass except DatabaseConnectionError as e: self.assertTrue(hasattr(e, 'path')) self.assertEquals(e.path, ':memory:') self.assertTrue(hasattr(e, 'msg')) self.assertEquals(e.msg, "DB doesn't exist") except Exception as e: self.fail("Unexpected exception raised: %r" % e) else: self.fail("Expected a DatabaseConnectionError exception") broker.initialize(Timestamp('1').internal) with broker.get() as conn: curs = conn.cursor() curs.execute('SELECT 1') self.assertEqual(curs.fetchall()[0][0], 1)
def test_delete_container(self): # Test AccountBroker.delete_container broker = AccountBroker(':memory:', account='a') broker.initialize(normalize_timestamp('1')) broker.put_container('o', normalize_timestamp(time()), 0, 0, 0) with broker.get() as conn: self.assertEqual(conn.execute( "SELECT count(*) FROM container " "WHERE deleted = 0").fetchone()[0], 1) self.assertEqual(conn.execute( "SELECT count(*) FROM container " "WHERE deleted = 1").fetchone()[0], 0) sleep(.00001) broker.put_container('o', 0, normalize_timestamp(time()), 0, 0) with broker.get() as conn: self.assertEqual(conn.execute( "SELECT count(*) FROM container " "WHERE deleted = 0").fetchone()[0], 0) self.assertEqual(conn.execute( "SELECT count(*) FROM container " "WHERE deleted = 1").fetchone()[0], 1)
def test_creation(self): # Test AccountBroker.__init__ broker = AccountBroker(':memory:', account='a') self.assertEqual(broker.db_file, ':memory:') try: with broker.get() as conn: pass except DatabaseConnectionError as e: self.assertTrue(hasattr(e, 'path')) self.assertEquals(e.path, ':memory:') self.assertTrue(hasattr(e, 'msg')) self.assertEquals(e.msg, "DB doesn't exist") except Exception as e: self.fail("Unexpected exception raised: %r" % e) else: self.fail("Expected a DatabaseConnectionError exception") broker.initialize(Timestamp('1').internal) with broker.get() as conn: curs = conn.cursor() curs.execute('SELECT 1') self.assertEqual(curs.fetchall()[0][0], 1)
def setUp(self): self._imported_create_account_stat_table = \ AccountBroker.create_account_stat_table AccountBroker.create_account_stat_table = \ premetadata_create_account_stat_table broker = AccountBroker(':memory:', account='a') broker.initialize(normalize_timestamp('1')) exc = None with broker.get() as conn: try: conn.execute('SELECT metadata FROM account_stat') except BaseException as err: exc = err self.assert_('no such column: metadata' in str(exc))
def setUp(self): self._imported_create_container_table = \ AccountBroker.create_container_table AccountBroker.create_container_table = \ prespi_create_container_table self._imported_initialize = AccountBroker._initialize AccountBroker._initialize = prespi_AccountBroker_initialize broker = AccountBroker(':memory:', account='a') broker.initialize(Timestamp('1').internal) exc = None with broker.get() as conn: try: conn.execute('SELECT storage_policy_index FROM container') except BaseException as err: exc = err self.assert_('no such column: storage_policy_index' in str(exc)) with broker.get() as conn: try: conn.execute('SELECT * FROM policy_stat') except sqlite3.OperationalError as err: self.assert_('no such table: policy_stat' in str(err)) else: self.fail('database created with policy_stat table')
def setUp(self): self._imported_create_container_table = \ AccountBroker.create_container_table AccountBroker.create_container_table = \ prespi_create_container_table self._imported_initialize = AccountBroker._initialize AccountBroker._initialize = prespi_AccountBroker_initialize broker = AccountBroker(':memory:', account='a') broker.initialize(Timestamp('1').internal) exc = None with broker.get() as conn: try: conn.execute('SELECT storage_policy_index FROM container') except BaseException as err: exc = err self.assert_('no such column: storage_policy_index' in str(exc)) with broker.get() as conn: try: conn.execute('SELECT * FROM policy_stat') except sqlite3.OperationalError as err: self.assert_('no such table: policy_stat' in str(err)) else: self.fail('database created with policy_stat table')
def test_put_container(self): # Test AccountBroker.put_container broker = AccountBroker(':memory:', account='a') broker.initialize(normalize_timestamp('1')) # Create initial container timestamp = normalize_timestamp(time()) broker.put_container('"{<container \'&\' name>}"', timestamp, 0, 0, 0) with broker.get() as conn: self.assertEqual(conn.execute( "SELECT name FROM container").fetchone()[0], '"{<container \'&\' name>}"') self.assertEqual(conn.execute( "SELECT put_timestamp FROM container").fetchone()[0], timestamp) self.assertEqual(conn.execute( "SELECT deleted FROM container").fetchone()[0], 0) # Reput same event broker.put_container('"{<container \'&\' name>}"', timestamp, 0, 0, 0) with broker.get() as conn: self.assertEqual(conn.execute( "SELECT name FROM container").fetchone()[0], '"{<container \'&\' name>}"') self.assertEqual(conn.execute( "SELECT put_timestamp FROM container").fetchone()[0], timestamp) self.assertEqual(conn.execute( "SELECT deleted FROM container").fetchone()[0], 0) # Put new event sleep(.00001) timestamp = normalize_timestamp(time()) broker.put_container('"{<container \'&\' name>}"', timestamp, 0, 0, 0) with broker.get() as conn: self.assertEqual(conn.execute( "SELECT name FROM container").fetchone()[0], '"{<container \'&\' name>}"') self.assertEqual(conn.execute( "SELECT put_timestamp FROM container").fetchone()[0], timestamp) self.assertEqual(conn.execute( "SELECT deleted FROM container").fetchone()[0], 0) # Put old event otimestamp = normalize_timestamp(float(timestamp) - 1) broker.put_container('"{<container \'&\' name>}"', otimestamp, 0, 0, 0) with broker.get() as conn: self.assertEqual(conn.execute( "SELECT name FROM container").fetchone()[0], '"{<container \'&\' name>}"') self.assertEqual(conn.execute( "SELECT put_timestamp FROM container").fetchone()[0], timestamp) self.assertEqual(conn.execute( "SELECT deleted FROM container").fetchone()[0], 0) # Put old delete event dtimestamp = normalize_timestamp(float(timestamp) - 1) broker.put_container('"{<container \'&\' name>}"', 0, dtimestamp, 0, 0) with broker.get() as conn: self.assertEqual(conn.execute( "SELECT name FROM container").fetchone()[0], '"{<container \'&\' name>}"') self.assertEqual(conn.execute( "SELECT put_timestamp FROM container").fetchone()[0], timestamp) self.assertEqual(conn.execute( "SELECT delete_timestamp FROM container").fetchone()[0], dtimestamp) self.assertEqual(conn.execute( "SELECT deleted FROM container").fetchone()[0], 0) # Put new delete event sleep(.00001) timestamp = normalize_timestamp(time()) broker.put_container('"{<container \'&\' name>}"', 0, timestamp, 0, 0) with broker.get() as conn: self.assertEqual(conn.execute( "SELECT name FROM container").fetchone()[0], '"{<container \'&\' name>}"') self.assertEqual(conn.execute( "SELECT delete_timestamp FROM container").fetchone()[0], timestamp) self.assertEqual(conn.execute( "SELECT deleted FROM container").fetchone()[0], 1) # Put new event sleep(.00001) timestamp = normalize_timestamp(time()) broker.put_container('"{<container \'&\' name>}"', timestamp, 0, 0, 0) with broker.get() as conn: self.assertEqual(conn.execute( "SELECT name FROM container").fetchone()[0], '"{<container \'&\' name>}"') self.assertEqual(conn.execute( "SELECT put_timestamp FROM container").fetchone()[0], timestamp) self.assertEqual(conn.execute( "SELECT deleted FROM container").fetchone()[0], 0)
def test_container_table_migration(self, tempdir): db_path = os.path.join(tempdir, 'account.db') # first init an acct DB without the policy_stat table present broker = AccountBroker(db_path, account='a') broker.initialize(Timestamp('1').internal) with broker.get() as conn: try: conn.execute(''' SELECT storage_policy_index FROM container ''').fetchone()[0] except sqlite3.OperationalError as err: # confirm that the table doesn't have this column self.assert_( 'no such column: storage_policy_index' in str(err)) else: self.fail('broker did not raise sqlite3.OperationalError ' 'trying to select from storage_policy_index ' 'from container table!') # manually insert an existing row to avoid migration with broker.get() as conn: conn.execute( ''' INSERT INTO container (name, put_timestamp, delete_timestamp, object_count, bytes_used, deleted) VALUES (?, ?, ?, ?, ?, ?) ''', ('test_name', Timestamp(time()).internal, 0, 1, 2, 0)) conn.commit() # make sure we can iter containers without the migration for c in broker.list_containers_iter(1, None, None, None, None): self.assertEqual(c, ('test_name', 1, 2, 0)) # stats table is mysteriously empty... stats = broker.get_policy_stats() self.assertEqual(len(stats), 0) # now do a PUT with a different value for storage_policy_index # which will update the DB schema as well as update policy_stats # for legacy containers in the DB (those without an SPI) other_policy = [p for p in POLICIES if p.idx != 0][0] broker.put_container('test_second', Timestamp(time()).internal, 0, 3, 4, other_policy.idx) broker._commit_puts_stale_ok() with broker.get() as conn: rows = conn.execute(''' SELECT name, storage_policy_index FROM container ''').fetchall() for row in rows: if row[0] == 'test_name': self.assertEqual(row[1], 0) else: self.assertEqual(row[1], other_policy.idx) # we should have stats for both containers stats = broker.get_policy_stats() self.assertEqual(len(stats), 2) self.assertEqual(stats[0]['object_count'], 1) self.assertEqual(stats[0]['bytes_used'], 2) self.assertEqual(stats[1]['object_count'], 3) self.assertEqual(stats[1]['bytes_used'], 4) # now lets delete a container and make sure policy_stats is OK with broker.get() as conn: conn.execute( ''' DELETE FROM container WHERE name = ? ''', ('test_name', )) conn.commit() stats = broker.get_policy_stats() self.assertEqual(len(stats), 2) self.assertEqual(stats[0]['object_count'], 0) self.assertEqual(stats[0]['bytes_used'], 0) self.assertEqual(stats[1]['object_count'], 3) self.assertEqual(stats[1]['bytes_used'], 4)
def test_half_upgraded_database(self, tempdir): db_path = os.path.join(tempdir, 'account.db') ts = itertools.count() ts = (Timestamp(t).internal for t in itertools.count(int(time()))) broker = AccountBroker(db_path, account='a') broker.initialize(ts.next()) self.assertTrue(broker.empty()) # add a container (to pending file) broker.put_container('c', ts.next(), 0, 0, 0, POLICIES.default.idx) real_get = broker.get called = [] @contextmanager def mock_get(): with real_get() as conn: def mock_executescript(script): if called: raise Exception('kaboom!') called.append(script) conn.executescript = mock_executescript yield conn broker.get = mock_get try: broker._commit_puts() except Exception: pass else: self.fail('mock exception was not raised') self.assertEqual(len(called), 1) self.assert_('CREATE TABLE policy_stat' in called[0]) # nothing was committed broker = AccountBroker(db_path, account='a') with broker.get() as conn: try: conn.execute('SELECT * FROM policy_stat') except sqlite3.OperationalError as err: self.assert_('no such table: policy_stat' in str(err)) else: self.fail('half upgraded database!') container_count = conn.execute( 'SELECT count(*) FROM container').fetchone()[0] self.assertEqual(container_count, 0) # try again to commit puts self.assertFalse(broker.empty()) # full migration successful with broker.get() as conn: conn.execute('SELECT * FROM policy_stat') conn.execute('SELECT storage_policy_index FROM container')
def test_container_table_migration(self, tempdir): db_path = os.path.join(tempdir, 'account.db') # first init an acct DB without the policy_stat table present broker = AccountBroker(db_path, account='a') broker.initialize(Timestamp('1').internal) with broker.get() as conn: try: conn.execute(''' SELECT storage_policy_index FROM container ''').fetchone()[0] except sqlite3.OperationalError as err: # confirm that the table doesn't have this column self.assert_('no such column: storage_policy_index' in str(err)) else: self.fail('broker did not raise sqlite3.OperationalError ' 'trying to select from storage_policy_index ' 'from container table!') # manually insert an existing row to avoid migration with broker.get() as conn: conn.execute(''' INSERT INTO container (name, put_timestamp, delete_timestamp, object_count, bytes_used, deleted) VALUES (?, ?, ?, ?, ?, ?) ''', ('test_name', Timestamp(time()).internal, 0, 1, 2, 0)) conn.commit() # make sure we can iter containers without the migration for c in broker.list_containers_iter(1, None, None, None, None): self.assertEqual(c, ('test_name', 1, 2, 0)) # stats table is mysteriously empty... stats = broker.get_policy_stats() self.assertEqual(len(stats), 0) # now do a PUT with a different value for storage_policy_index # which will update the DB schema as well as update policy_stats # for legacy containers in the DB (those without an SPI) other_policy = [p for p in POLICIES if p.idx != 0][0] broker.put_container('test_second', Timestamp(time()).internal, 0, 3, 4, other_policy.idx) broker._commit_puts_stale_ok() with broker.get() as conn: rows = conn.execute(''' SELECT name, storage_policy_index FROM container ''').fetchall() for row in rows: if row[0] == 'test_name': self.assertEqual(row[1], 0) else: self.assertEqual(row[1], other_policy.idx) # we should have stats for both containers stats = broker.get_policy_stats() self.assertEqual(len(stats), 2) self.assertEqual(stats[0]['object_count'], 1) self.assertEqual(stats[0]['bytes_used'], 2) self.assertEqual(stats[1]['object_count'], 3) self.assertEqual(stats[1]['bytes_used'], 4) # now lets delete a container and make sure policy_stats is OK with broker.get() as conn: conn.execute(''' DELETE FROM container WHERE name = ? ''', ('test_name',)) conn.commit() stats = broker.get_policy_stats() self.assertEqual(len(stats), 2) self.assertEqual(stats[0]['object_count'], 0) self.assertEqual(stats[0]['bytes_used'], 0) self.assertEqual(stats[1]['object_count'], 3) self.assertEqual(stats[1]['bytes_used'], 4)