def test_get_default_worker_with_os_env(mock_kafka_consumer, mock_docker_client): try: os.environ['CODEPACK_CONFIG_DIR'] = 'config' os.environ['CODEPACK_CONFIG_PATH'] = 'sample.ini' worker = Default.get_employee('worker') assert isinstance(worker, Worker) arg_list = mock_kafka_consumer.call_args_list assert len(arg_list) == 1 args, kwargs = arg_list[0] assert kwargs.get('bootstrap_servers', '') == '?:9092,?:9092,?:9092' assert isinstance(kwargs.get('value_deserializer', ''), Callable) assert len(args) == 1 and args[0] == 'codepack' assert hasattr(worker.messenger, 'consumer') assert getattr(worker.messenger, 'consumer').session is mock_kafka_consumer() mock_docker_client.assert_called_once_with( base_url='unix://var/run/docker.sock') assert worker.docker_manager.docker.session == mock_docker_client() assert isinstance(worker.callback_service, CallbackService) assert isinstance(worker.logger, logging.Logger) assert worker.logger.name == 'worker-logger' assert hasattr(worker, 'script') and worker.script == 'run_snapshot.py' default_dir = Default.get_config_instance().get_default_config_dir() script_dir = os.path.join(default_dir, 'scripts') assert hasattr(worker, 'script_dir') and worker.script_dir == script_dir assert hasattr(worker, 'script_path') and worker.script_path == os.path.join( default_dir, 'scripts/run_snapshot.py') finally: os.environ.pop('CODEPACK_CONFIG_DIR', None) os.environ.pop('CODEPACK_CONFIG_PATH', None)
def test_get_default_docker_manager(mock_client): docker_manager = Default.get_docker_manager() mock_client.assert_called_once_with(base_url='unix://var/run/docker.sock') assert isinstance(docker_manager, DockerManager) default_dir = Default.get_config_instance().get_default_config_dir() assert docker_manager.path == os.path.join(default_dir, 'scripts') assert docker_manager.run_opt == {'dns': ['8.8.8.8'], 'auto_remove': True}
async def run_by_id_pair(params: IDPair): codepack_storage_service = Default.get_service('codepack', 'storage_service') codepack = codepack_storage_service.load(params.codepack_id) argpack_storage_service = Default.get_service('argpack', 'storage_service') argpack = argpack_storage_service.load(params.argpack_id) common.supervisor.run_codepack(codepack=codepack, argpack=argpack) return {'serial_number': codepack.serial_number}
def test_get_default_logger(): logger = Default.get_logger() assert isinstance(logger, logging.Logger) assert logger.name == 'default-logger' assert len(logger.handlers) == 1 assert logger.handlers[0].get_name() == 'console' logger2 = Default.get_logger('worker-logger') assert isinstance(logger2, logging.Logger) assert logger2.name == 'worker-logger' assert len(logger2.handlers) == 2
async def register_by_id_pair(params: IDPairJob): codepack_storage_service = Default.get_service('codepack', 'storage_service') codepack = codepack_storage_service.load(params.codepack_id) argpack_storage_service = Default.get_service('argpack', 'storage_service') argpack = argpack_storage_service.load(params.argpack_id) common.scheduler.add_codepack(codepack=codepack, argpack=argpack, job_id=params.job_id, trigger=params.trigger, **params.trigger_config) return {'serial_number': codepack.serial_number}
async def startup(): Default.init() config = Default.config.get_config(section='apiserver') _scheduler = config.get('scheduler', 'self') common.add(key='supervisor', value=Default.get_employee('supervisor'), destroy=destroy_supervisor) if isinstance(_scheduler, Scheduler): common.add(key='scheduler', value=_scheduler, destroy=destroy_scheduler) common.scheduler.start() elif isinstance(_scheduler, str): if _scheduler == 'self': common.add(key='scheduler', value=Default.get_scheduler(), destroy=destroy_scheduler) common.scheduler.start() else: common.add(key='scheduler', value=_scheduler)
def test_default_init_with_nothing(): Default() assert len(Default.instances) == 0 assert Default.config is not None assert Default.config.config_path is None assert Default.alias is not None assert Default.alias.aliases is None
def test_default_init_with_config_path(): Default(config_path='config/test.ini') assert len(Default.instances) == 0 assert Default.config is not None assert Default.config.config_path == 'config/test.ini' assert Default.alias is not None assert Default.alias.aliases is None
def test_memory_worker_run_snapshot_with_image_and_callback( mock_docker_client): worker = Default.get_employee('worker') worker.callback = dummy_callback_function assert isinstance(worker.messenger, MemoryMessenger) code = Code(add2, image='dummy') assert code.get_state() == 'UNKNOWN' sn = worker.run_snapshot(code.to_snapshot(kwargs={'a': 3, 'b': 5})) assert sn == code.serial_number default_config_dir = Config.get_default_config_dir() script_dir = os.path.join(default_config_dir, 'scripts') mock_docker_client.return_value.containers.run.assert_called_once_with( auto_remove=True, command=[ 'python', 'run_snapshot.py', '%s.json' % sn, '-p', '.', '-l', 'worker-logger', '-c', 'dummy_callback_function' ], dns=['8.8.8.8'], environment=['CODEPACK_LOGGER_LOG_DIR=/usr/logs'], image='dummy', name=id(worker.docker_manager), volumes=[ '%s:/usr/src/codepack' % script_dir, '%s:/usr/logs' % os.path.abspath(Config.get_log_dir()) ], working_dir='/usr/src/codepack') worker.stop()
def test_get_default_scheduler_with_some_os_env(mock_client): try: os.environ['CODEPACK_SCHEDULER_SOURCE'] = 'mongodb' os.environ['CODEPACK_SCHEDULER_DB'] = 'test_db' os.environ['CODEPACK_SCHEDULER_COLLECTION'] = 'test_collection' os.environ['CODEPACK_SCHEDULER_SUPERVISOR'] = 'dummy_supervisor' os.environ['CODEPACK_MONGODB_REPLICASET'] = 'TEST' scheduler = Default.get_scheduler() assert isinstance(scheduler, Scheduler) assert 'codepack' in scheduler.jobstores jobstore = scheduler.jobstores['codepack'] assert hasattr(jobstore, 'storage') assert isinstance(jobstore.storage, MongoStorage) assert jobstore.storage.db == 'test_db' assert jobstore.storage.collection == 'test_collection' assert jobstore.storage.item_type == StorableJob assert jobstore.storage.key == 'id' assert scheduler.supervisor == 'dummy_supervisor' mock_client.assert_called_once_with(host='localhost', port=27017, replicaset='TEST') finally: os.environ.pop('CODEPACK_SCHEDULER_SOURCE', None) os.environ.pop('CODEPACK_SCHEDULER_DB', None) os.environ.pop('CODEPACK_SCHEDULER_COLLECTION', None) os.environ.pop('CODEPACK_SCHEDULER_SUPERVISOR', None) os.environ.pop('CODEPACK_MONGODB_REPLICASET', None)
def test_memory_supervisor_organize(): supervisor = Default.get_employee('supervisor') assert isinstance(supervisor.messenger, MemoryMessenger) assert supervisor.messenger.topic == 'codepack' assert supervisor.messenger.queues['codepack'].empty() code1 = Code(add2) code2 = Code(mul2) code1 >> code2 code2(3, b=5) assert code1.get_state() == 'UNKNOWN' assert code2.get_state() == 'WAITING' supervisor.organize() assert code1.get_state() == 'UNKNOWN' assert code2.get_state() == 'WAITING' assert supervisor.messenger.queues['codepack'].empty() code1(a=2, b=4) assert code1.get_state() == 'TERMINATED' assert code2.get_state() == 'WAITING' assert supervisor.messenger.queues['codepack'].empty() supervisor.organize() assert code1.get_state() == 'TERMINATED' assert code2.get_state() == 'READY' assert not supervisor.messenger.queues['codepack'].empty( ) and supervisor.messenger.queues['codepack'].qsize() == 1 item = supervisor.messenger.queues['codepack'].get(block=False) item.pop('source', None) expected_item = code2.to_snapshot(args=(3, ), kwargs={ 'b': 5 }, timestamp=item['timestamp']).to_dict() expected_item.pop('source', None) assert item == expected_item supervisor.close()
async def result(serial_number: str): delivery_service = Default.get_service('delivery', 'delivery_service') tmp = delivery_service.check(serial_number=serial_number) if tmp: _result = delivery_service.receive(serial_number=serial_number) else: _result = None return {'serial_number': serial_number, 'result': _result}
def test_default_get_storage_config(): Default(config_path='config/test.ini') assert len(Default.instances) == 0 config = Default.get_config_instance() storage_config = config.get_storage_config('worker') assert storage_config == { 'source': 'kafka', 'topic': 'test', 'kafka': { 'bootstrap_servers': 'localhost:9092' }, 'group_id': 'codepack_worker_test', 'interval': '5', 'supervisor': 'http://localhost:8000', 'script_path': 'scripts/run_snapshot.py', 'logger': 'worker-logger' }
def test_default_init_with_alias_path(): Default(alias_path='codepack/utils/config/default/alias.ini') assert len(Default.instances) == 0 assert Default.config is not None assert Default.config.config_path is None assert Default.alias is not None assert Default.alias.aliases is not None and isinstance( Default.alias.aliases, dict)
def default_os_env(): os.environ['CODEPACK_CONFIG_DIR'] = 'config' os.environ['CODEPACK_CONFIG_PATH'] = 'test.ini' Default.get_service('delivery', 'delivery_service').storage.init() Default.get_service('code_snapshot', 'snapshot_service').storage.init() Default.get_service('code', 'storage_service').storage.init() Default.get_service('codepack', 'storage_service').storage.init() yield os.environ.pop('CODEPACK_CONFIG_DIR', None) os.environ.pop('CODEPACK_CONFIG_PATH', None)
def test_get_default_scheduler(): scheduler = Default.get_scheduler() assert isinstance(scheduler, Scheduler) assert 'codepack' in scheduler.jobstores jobstore = scheduler.jobstores['codepack'] assert hasattr(jobstore, 'storage') assert isinstance(jobstore.storage, MemoryStorage) assert jobstore.storage.item_type == StorableJob assert jobstore.storage.key == 'id' assert scheduler.supervisor is None
def test_memory_worker_run_snapshot_with_nothing(): worker = Default.get_employee('worker') assert isinstance(worker.messenger, MemoryMessenger) code = Code(add2) assert code.get_state() == 'UNKNOWN' sn = worker.run_snapshot(code.to_snapshot(kwargs={'a': 3, 'b': 5})) assert sn == code.serial_number assert code.get_state() == 'TERMINATED' assert code.get_result() == 8 worker.stop()
async def result(serial_number: str): snapshot_service = Default.get_service('codepack_snapshot', 'snapshot_service') snapshot = snapshot_service.load(serial_number=serial_number) _result = None if snapshot and snapshot['subscribe']: codepack_snapshot = CodePackSnapshot.from_dict(snapshot) codepack = CodePack.from_snapshot(codepack_snapshot) _result = codepack.get_result() return {'serial_number': serial_number, 'result': _result}
async def register_by_id(params: CodePackIDJob): storage_service = Default.get_service('codepack', 'storage_service') codepack = storage_service.load(params.id) argpack = ArgPack.from_json(params.argpack) common.scheduler.add_codepack(codepack=codepack, argpack=argpack, job_id=params.job_id, trigger=params.trigger, **params.trigger_config) return {'serial_number': codepack.serial_number}
def test_default_memory_delivery_service_with_os_env(): config = Config() storage_config = config.get_storage_config('delivery') assert storage_config == {'source': 'memory'} env_source = 'CODEPACK_DELIVERY_SOURCE' try: os.environ[env_source] = 'memory' mds = Default.get_service('delivery', 'delivery_service') assert isinstance(mds.storage, MemoryStorage) finally: for env in [env_source]: os.environ.pop(env, None)
def test_default_init_with_config_path_and_alias_path_os_env(): try: os.environ[ 'CODEPACK_ALIAS_PATH'] = 'codepack/utils/config/default/alias.ini' Default(config_path='config/test.ini') assert len(Default.instances) == 0 assert Default.config is not None assert Default.config.config_path == 'config/test.ini' assert Default.alias is not None assert Default.alias.aliases is None finally: os.environ.pop('CODEPACK_ALIAS_PATH', None)
async def state(serial_number: str): snapshot_service = Default.get_service('code_snapshot', 'snapshot_service') snapshot = snapshot_service.load(serial_number=serial_number, projection={'state', 'message'}) ret = {'serial_number': serial_number} if snapshot: _state = snapshot['state'] if snapshot['message']: ret['message'] = snapshot['message'] else: _state = 'UNKNOWN' ret['state'] = _state return ret
def test_default_init_with_os_env(): try: os.environ['CODEPACK_CONFIG_DIR'] = 'config' os.environ['CODEPACK_CONFIG_PATH'] = 'test.ini' Default() assert len(Default.instances) == 0 assert Default.config is not None assert Default.config.config_path is None assert Default.alias is not None assert Default.alias.aliases is None finally: os.environ.pop('CODEPACK_CONFIG_DIR', None) os.environ.pop('CODEPACK_CONFIG_PATH', None)
def test_get_default_worker_without_os_env(mock_docker_client): worker = Default.get_employee('worker') assert isinstance(worker, Worker) assert isinstance(worker.messenger, MemoryMessenger) assert worker.messenger.topic == 'codepack' assert worker.supervisor is None mock_docker_client.assert_called_once_with( base_url='unix://var/run/docker.sock') assert worker.docker_manager.docker.session == mock_docker_client() assert isinstance(worker.callback_service, CallbackService) assert isinstance(worker.callback_service.storage, FileStorage) default_dir = Default.get_config_instance().get_default_config_dir() assert worker.callback_service.storage.path == os.path.join( default_dir, 'scripts') assert isinstance(worker.logger, logging.Logger) assert worker.logger.name == 'worker-logger' assert hasattr(worker, 'script') and worker.script == 'run_snapshot.py' assert hasattr(worker, 'script_dir') and worker.script_dir == os.path.join( default_dir, 'scripts') assert hasattr(worker, 'script_path') and worker.script_path == os.path.join( default_dir, 'scripts/run_snapshot.py')
def test_default_memory_code_storage_service_with_os_env(): config = Config() storage_config = config.get_storage_config('code') assert storage_config == {'source': 'memory'} env_source = 'CODEPACK_CODE_SOURCE' try: os.environ[env_source] = 'memory' mss = Default.get_service('code', 'storage_service') assert hasattr(mss.storage, 'memory') assert mss.storage.item_type == Code finally: if env_source in os.environ: os.environ.pop(env_source, None)
def test_get_default_supervisor(): try: os.environ['CODEPACK_CODESNAPSHOT_SOURCE'] = 'mongodb' os.environ['CODEPACK_CODESNAPSHOT_DB'] = 'test_db' os.environ['CODEPACK_CODESNAPSHOT_COLLECTION'] = 'test_collection' supervisor = Default.get_employee('supervisor') assert isinstance(supervisor, Supervisor) assert isinstance(supervisor.messenger, MemoryMessenger) assert supervisor.messenger.topic == 'codepack' assert isinstance(supervisor.snapshot_service, SnapshotService) assert isinstance(supervisor.snapshot_service.storage, MongoStorage) finally: os.environ.pop('CODEPACK_CODESNAPSHOT_SOURCE', None) os.environ.pop('CODEPACK_CODESNAPSHOT_DB', None) os.environ.pop('CODEPACK_CODESNAPSHOT_COLLECTION', None)
async def state(serial_number: str): snapshot_service = Default.get_service('codepack_snapshot', 'snapshot_service') tmp = snapshot_service.load(serial_number=serial_number) ret = {'serial_number': serial_number} if tmp: codepack_snapshot = CodePackSnapshot.from_dict(tmp) codepack = CodePack.from_snapshot(codepack_snapshot) _state = codepack.get_state() if _state == 'ERROR': ret['message'] = codepack.get_message() else: _state = 'UNKNOWN' ret['state'] = _state return ret
def test_default_get_alias_from_source(): assert Default.get_alias_from_source(source='memory') == 'memory' assert Default.get_alias_from_source(source='memory', prefix='front') == 'front_memory' assert Default.get_alias_from_source(source='memory', suffix='back') == 'memory_back' assert Default.get_alias_from_source(source='memory', prefix='front', suffix='back') == 'front_memory_back' assert Default.get_alias_from_source(source='mongodb') == 'mongo' assert Default.get_alias_from_source(source='mongodb', prefix='front') == 'front_mongo' assert Default.get_alias_from_source(source='mongodb', suffix='back') == 'mongo_back' assert Default.get_alias_from_source(source='mongodb', prefix='front', suffix='back') == 'front_mongo_back'
def test_default_file_code_snapshot_service_with_os_env(): config = Config() storage_config = config.get_storage_config('code_snapshot') assert storage_config == {'source': 'memory'} env_source = 'CODEPACK_CODESNAPSHOT_SOURCE' env_path = 'CODEPACK_CODESNAPSHOT_PATH' try: os.environ[env_source] = 'file' os.environ[env_path] = 'tmp/' fss = Default.get_service('code_snapshot', 'snapshot_service') assert hasattr(fss.storage, 'path') assert fss.storage.path == 'tmp/' finally: for env in [env_source, env_path]: os.environ.pop(env, None)
def test_default_file_delivery_service_with_os_env(): config = Config() storage_config = config.get_storage_config('delivery') assert storage_config == {'source': 'memory'} env_source = 'CODEPACK_DELIVERY_SOURCE' env_path = 'CODEPACK_DELIVERY_PATH' try: os.environ[env_source] = 'file' os.environ[env_path] = 'tmp/' fds = Default.get_service('delivery', 'delivery_service') assert hasattr(fds.storage, 'path') assert fds.storage.path == 'tmp/' finally: for env in [env_source, env_path]: os.environ.pop(env, None)