def test_process_statistics(self): zookeeper = flexmock() flexmock(datastore_server.DatastoreDistributed)\ .should_receive("get_entity_kind").and_return("kind") dsg = groomer.DatastoreGroomer(zookeeper, "cassandra", "localhost:8888") dsg = flexmock(dsg) dsg.stats['app_id'] = {'kind': {'size': 0, 'number': 0}} # This one gets ignored dsg.should_receive("initialize_kind") self.assertEquals(True, dsg.process_statistics("key", FakeEntity(), 1)) self.assertEquals(dsg.stats, {'app_id': { 'kind': { 'size': 1, 'number': 1 } }}) self.assertEquals(True, dsg.process_statistics("key", FakeEntity(), 1)) self.assertEquals(dsg.stats, {'app_id': { 'kind': { 'size': 2, 'number': 2 } }})
def test_initialize_kind(self): zookeeper = flexmock() flexmock(entity_pb).should_receive('EntityProto').and_return(FakeEntity()) dsg = groomer.DatastoreGroomer(zookeeper, "cassandra", "localhost:8888") dsg = flexmock(dsg) dsg.initialize_kind('app_id', 'kind') self.assertEquals(dsg.stats, {'app_id': {'kind': {'size': 0, 'number': 0}}})
def test_create_global_stat_entry(self): zookeeper = flexmock() stats = flexmock(db.stats) stats.should_receive("KindStat").and_return(FakeEntity()) dsg = groomer.DatastoreGroomer(zookeeper, "cassandra", "localhost:8888") self.assertRaises(Exception, dsg.create_kind_stat_entry, 0, 0, 0)
def test_hard_delete_row(self): zookeeper = flexmock() dsg = groomer.DatastoreGroomer(zookeeper, "cassandra", "localhost:8888") dsg = flexmock(dsg) dsg.db_access = FakeDatastore() self.assertEquals(False, dsg.hard_delete_row("some_key"))
def test_reset_statistics(self): zookeeper = flexmock() flexmock(entity_pb).should_receive('EntityProto').and_return( FakeEntity()) dsg = groomer.DatastoreGroomer(zookeeper, "cassandra", "localhost:8888") dsg.reset_statistics() self.assertEquals(dsg.stats, {})
def test_remove_old_statistics(self): zookeeper = flexmock() dsg = groomer.DatastoreGroomer(zookeeper, "cassandra", "localhost:8888") dsg = flexmock(dsg) dsg.should_receive("register_db_accessor").and_return(FakeDistributedDB()) dsg.stats['app_id'] = {'kind': {'size': 0, 'number': 0}} dsg.stats['app_id1'] = {'kind': {'size': 0, 'number': 0}} self.assertRaises(Exception, dsg.remove_old_statistics)
def test_register_db_accessor(self): zookeeper = flexmock() fake_ds = FakeDatastore() flexmock(datastore_distributed).should_receive('DatastoreDistributed').\ and_return(fake_ds) flexmock(apiproxy_stub_map.apiproxy).should_receive('RegisterStub') dsg = groomer.DatastoreGroomer(zookeeper, "cassandra", "localhost:8888") self.assertEquals(fake_ds, dsg.register_db_accessor("app_id"))
def test_remove_old_dashboard_data(self): zookeeper = flexmock() dsg = groomer.DatastoreGroomer(zookeeper, "cassandra", "localhost:8888") dsg = flexmock(dsg) dsg.should_receive("register_db_accessor").and_return( FakeDistributedDB()) dsg.DASHBOARD_DATA_MODELS = [FakeEntity] self.assertRaises(Exception, dsg.remove_old_dashboard_data)
def test_process_entity(self): zookeeper = flexmock() flexmock(entity_pb).should_receive('EntityProto').and_return(FakeEntity()) dsg = groomer.DatastoreGroomer(zookeeper, "cassandra", "localhost:8888") dsg = flexmock(dsg) dsg.should_receive('process_statistics') self.assertEquals(True, dsg.process_entity({'key':{dbconstants.APP_ENTITY_SCHEMA[0]:'ent', dbconstants.APP_ENTITY_SCHEMA[1]:'version'}}))
def test_run_groomer(self): zookeeper = flexmock() dsg = groomer.DatastoreGroomer(zookeeper, "cassandra", "localhost:8888") dsg = flexmock(dsg) dsg.should_receive("get_entity_batch").and_return([]) dsg.should_receive("process_entity") dsg.should_receive("update_statistics").and_raise(Exception) ds_factory = flexmock(appscale_datastore_batch.DatastoreFactory) ds_factory.should_receive("getDatastore").and_return(FakeDatastore()) self.assertRaises(Exception, dsg.run_groomer)
def test_update_statistics(self): zookeeper = flexmock() dsg = groomer.DatastoreGroomer(zookeeper, "cassandra", "localhost:8888") dsg = flexmock(dsg) dsg.should_receive("register_db_accessor").and_return(FakeDistributedDB()) dsg.should_receive("create_global_stat_entry").and_return(True) dsg.should_receive("create_kind_stat_entry").and_return(True) dsg.stats['app_id'] = {'kind': {'size': 0, 'number': 0}} dsg.stats['app_id1'] = {'kind': {'size': 0, 'number': 0}} # Should loop twice and on the second raise an exception. self.assertEquals(True, dsg.update_statistics()) dsg.should_receive("create_kind_stat_entry").and_return(False) self.assertEquals(False, dsg.update_statistics())
def main(): """ This main function allows you to run the groomer manually. """ zk_connection_locations = appscale_info.get_zk_locations_string() zookeeper = zk.ZKTransaction(host=zk_connection_locations) db_info = appscale_info.get_db_info() table = db_info[':table'] master = appscale_info.get_db_master_ip() datastore_path = "{0}:8888".format(master) ds_groomer = groomer.DatastoreGroomer(zookeeper, table, datastore_path) try: ds_groomer.remove_old_logs(None) finally: zookeeper.close()
def test_process_tombstone(self): zookeeper = flexmock() zookeeper.should_receive("get_transaction_id").and_return(1) zookeeper.should_receive("acquire_lock").and_return(True) zookeeper.should_receive("release_lock").and_return(True) zookeeper.should_receive("is_blacklisted").and_return(False) zookeeper.should_receive("notify_failed_transaction").and_return(True) flexmock(time) time.should_receive('sleep').and_return() flexmock(FakeDatastore) FakeDatastore.should_receive("batch_delete") dsg = groomer.DatastoreGroomer(zookeeper, "cassandra", "localhost:8888") dsg = flexmock(dsg) dsg.should_receive("hard_delete_row").and_return(True) flexmock(entity_utils) entity_utils.should_receive("get_root_key_from_entity_key").and_return( "key") entity_utils.should_receive("get_prefix_from_entity_key").and_return( "app/ns") dsg.db_access = FakeDatastore() # Successful operation. self.assertEquals(True, dsg.process_tombstone("key", "entity", "1")) # Failure on release lock but delete was successful. zookeeper.should_receive("release_lock").\ and_raise(ZKTransactionException('zk')) self.assertEquals(True, dsg.process_tombstone("key", "entity", "1")) # Hard delete failed. dsg.should_receive("hard_delete_row").and_return(False) self.assertEquals(False, dsg.process_tombstone("key", "entity", "1")) # Failed to acquire lock. zookeeper.should_receive("acquire_lock").and_return(False) self.assertEquals(False, dsg.process_tombstone("key", "entity", "1")) # Failed to acquire lock with an exception. zookeeper.should_receive("acquire_lock").\ and_raise(ZKTransactionException('zk')) self.assertEquals(False, dsg.process_tombstone("key", "entity", "1"))
def test_init(self): zookeeper = flexmock() dsg = groomer.DatastoreGroomer(zookeeper, "cassandra", "localhost:8888")
def test_init(self): zookeeper = flexmock() dsg = groomer.DatastoreGroomer(zookeeper, "hypertable", "localhost:8888")
""" Provides a service which periodically runs the groomer. """ import logging import os import sys import groomer from zkappscale import zktransaction as zk sys.path.append(os.path.join(os.path.dirname(__file__), "../lib/")) import appscale_info # Location to find the datastore service. LOCAL_DATASTORE = "localhost:8888" if __name__ == "__main__": logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) zookeeper_locations = appscale_info.get_zk_locations_string() gc_zookeeper = zk.ZKTransaction(host=zookeeper_locations) logger.info("Using ZK locations {0}".format(zookeeper_locations)) ds_groomer = groomer.DatastoreGroomer(gc_zookeeper, "cassandra", LOCAL_DATASTORE) logger.info("Starting service.") try: ds_groomer.start() except Exception, exception: logger.warning("An exception slipped through:") logger.exception(exception) logger.warning("Exiting service.")
def test_get_groomer_lock(self): zookeeper = flexmock() zookeeper.should_receive("get_datastore_groomer_lock").and_return(True) dsg = groomer.DatastoreGroomer(zookeeper, "cassandra", "localhost:8888") self.assertEquals(True, dsg.get_groomer_lock())