def test_get_default_worker_with_os_env(mock_kafka_consumer,
mock_docker_client):
try:
os.environ['CODEPACK_CONFIG_DIR'] = 'config'
os.environ['CODEPACK_CONFIG_PATH'] = 'sample.ini'
worker = Default.get_employee('worker')
assert isinstance(worker, Worker)
arg_list = mock_kafka_consumer.call_args_list
assert len(arg_list) == 1
args, kwargs = arg_list[0]
assert kwargs.get('bootstrap_servers', '') == '?:9092,?:9092,?:9092'
assert isinstance(kwargs.get('value_deserializer', ''), Callable)
assert len(args) == 1 and args[0] == 'codepack'
assert hasattr(worker.messenger, 'consumer')
assert getattr(worker.messenger,
'consumer').session is mock_kafka_consumer()
mock_docker_client.assert_called_once_with(
base_url='unix://var/run/docker.sock')
assert worker.docker_manager.docker.session == mock_docker_client()
assert isinstance(worker.callback_service, CallbackService)
assert isinstance(worker.logger, logging.Logger)
assert worker.logger.name == 'worker-logger'
assert hasattr(worker, 'script') and worker.script == 'run_snapshot.py'
default_dir = Default.get_config_instance().get_default_config_dir()
script_dir = os.path.join(default_dir, 'scripts')
assert hasattr(worker,
'script_dir') and worker.script_dir == script_dir
assert hasattr(worker,
'script_path') and worker.script_path == os.path.join(
default_dir, 'scripts/run_snapshot.py')
finally:
os.environ.pop('CODEPACK_CONFIG_DIR', None)
os.environ.pop('CODEPACK_CONFIG_PATH', None)
def test_get_default_docker_manager(mock_client):
docker_manager = Default.get_docker_manager()
mock_client.assert_called_once_with(base_url='unix://var/run/docker.sock')
assert isinstance(docker_manager, DockerManager)
default_dir = Default.get_config_instance().get_default_config_dir()
assert docker_manager.path == os.path.join(default_dir, 'scripts')
assert docker_manager.run_opt == {'dns': ['8.8.8.8'], 'auto_remove': True}
async def run_by_id_pair(params: IDPair):
codepack_storage_service = Default.get_service('codepack',
'storage_service')
codepack = codepack_storage_service.load(params.codepack_id)
argpack_storage_service = Default.get_service('argpack', 'storage_service')
argpack = argpack_storage_service.load(params.argpack_id)
common.supervisor.run_codepack(codepack=codepack, argpack=argpack)
return {'serial_number': codepack.serial_number}
def test_get_default_logger():
logger = Default.get_logger()
assert isinstance(logger, logging.Logger)
assert logger.name == 'default-logger'
assert len(logger.handlers) == 1
assert logger.handlers[0].get_name() == 'console'
logger2 = Default.get_logger('worker-logger')
assert isinstance(logger2, logging.Logger)
assert logger2.name == 'worker-logger'
assert len(logger2.handlers) == 2
async def register_by_id_pair(params: IDPairJob):
codepack_storage_service = Default.get_service('codepack',
'storage_service')
codepack = codepack_storage_service.load(params.codepack_id)
argpack_storage_service = Default.get_service('argpack', 'storage_service')
argpack = argpack_storage_service.load(params.argpack_id)
common.scheduler.add_codepack(codepack=codepack,
argpack=argpack,
job_id=params.job_id,
trigger=params.trigger,
**params.trigger_config)
return {'serial_number': codepack.serial_number}
async def startup():
Default.init()
config = Default.config.get_config(section='apiserver')
_scheduler = config.get('scheduler', 'self')
common.add(key='supervisor', value=Default.get_employee('supervisor'), destroy=destroy_supervisor)
if isinstance(_scheduler, Scheduler):
common.add(key='scheduler', value=_scheduler, destroy=destroy_scheduler)
common.scheduler.start()
elif isinstance(_scheduler, str):
if _scheduler == 'self':
common.add(key='scheduler', value=Default.get_scheduler(), destroy=destroy_scheduler)
common.scheduler.start()
else:
common.add(key='scheduler', value=_scheduler)
def test_default_init_with_nothing():
Default()
assert len(Default.instances) == 0
assert Default.config is not None
assert Default.config.config_path is None
assert Default.alias is not None
assert Default.alias.aliases is None
def test_default_init_with_config_path():
Default(config_path='config/test.ini')
assert len(Default.instances) == 0
assert Default.config is not None
assert Default.config.config_path == 'config/test.ini'
assert Default.alias is not None
assert Default.alias.aliases is None
def test_memory_worker_run_snapshot_with_image_and_callback(
mock_docker_client):
worker = Default.get_employee('worker')
worker.callback = dummy_callback_function
assert isinstance(worker.messenger, MemoryMessenger)
code = Code(add2, image='dummy')
assert code.get_state() == 'UNKNOWN'
sn = worker.run_snapshot(code.to_snapshot(kwargs={'a': 3, 'b': 5}))
assert sn == code.serial_number
default_config_dir = Config.get_default_config_dir()
script_dir = os.path.join(default_config_dir, 'scripts')
mock_docker_client.return_value.containers.run.assert_called_once_with(
auto_remove=True,
command=[
'python', 'run_snapshot.py',
'%s.json' % sn, '-p', '.', '-l', 'worker-logger', '-c',
'dummy_callback_function'
],
dns=['8.8.8.8'],
environment=['CODEPACK_LOGGER_LOG_DIR=/usr/logs'],
image='dummy',
name=id(worker.docker_manager),
volumes=[
'%s:/usr/src/codepack' % script_dir,
'%s:/usr/logs' % os.path.abspath(Config.get_log_dir())
],
working_dir='/usr/src/codepack')
worker.stop()
def test_get_default_scheduler_with_some_os_env(mock_client):
try:
os.environ['CODEPACK_SCHEDULER_SOURCE'] = 'mongodb'
os.environ['CODEPACK_SCHEDULER_DB'] = 'test_db'
os.environ['CODEPACK_SCHEDULER_COLLECTION'] = 'test_collection'
os.environ['CODEPACK_SCHEDULER_SUPERVISOR'] = 'dummy_supervisor'
os.environ['CODEPACK_MONGODB_REPLICASET'] = 'TEST'
scheduler = Default.get_scheduler()
assert isinstance(scheduler, Scheduler)
assert 'codepack' in scheduler.jobstores
jobstore = scheduler.jobstores['codepack']
assert hasattr(jobstore, 'storage')
assert isinstance(jobstore.storage, MongoStorage)
assert jobstore.storage.db == 'test_db'
assert jobstore.storage.collection == 'test_collection'
assert jobstore.storage.item_type == StorableJob
assert jobstore.storage.key == 'id'
assert scheduler.supervisor == 'dummy_supervisor'
mock_client.assert_called_once_with(host='localhost',
port=27017,
replicaset='TEST')
finally:
os.environ.pop('CODEPACK_SCHEDULER_SOURCE', None)
os.environ.pop('CODEPACK_SCHEDULER_DB', None)
os.environ.pop('CODEPACK_SCHEDULER_COLLECTION', None)
os.environ.pop('CODEPACK_SCHEDULER_SUPERVISOR', None)
os.environ.pop('CODEPACK_MONGODB_REPLICASET', None)
def test_memory_supervisor_organize():
supervisor = Default.get_employee('supervisor')
assert isinstance(supervisor.messenger, MemoryMessenger)
assert supervisor.messenger.topic == 'codepack'
assert supervisor.messenger.queues['codepack'].empty()
code1 = Code(add2)
code2 = Code(mul2)
code1 >> code2
code2(3, b=5)
assert code1.get_state() == 'UNKNOWN'
assert code2.get_state() == 'WAITING'
supervisor.organize()
assert code1.get_state() == 'UNKNOWN'
assert code2.get_state() == 'WAITING'
assert supervisor.messenger.queues['codepack'].empty()
code1(a=2, b=4)
assert code1.get_state() == 'TERMINATED'
assert code2.get_state() == 'WAITING'
assert supervisor.messenger.queues['codepack'].empty()
supervisor.organize()
assert code1.get_state() == 'TERMINATED'
assert code2.get_state() == 'READY'
assert not supervisor.messenger.queues['codepack'].empty(
) and supervisor.messenger.queues['codepack'].qsize() == 1
item = supervisor.messenger.queues['codepack'].get(block=False)
item.pop('source', None)
expected_item = code2.to_snapshot(args=(3, ),
kwargs={
'b': 5
},
timestamp=item['timestamp']).to_dict()
expected_item.pop('source', None)
assert item == expected_item
supervisor.close()
async def result(serial_number: str):
delivery_service = Default.get_service('delivery', 'delivery_service')
tmp = delivery_service.check(serial_number=serial_number)
if tmp:
_result = delivery_service.receive(serial_number=serial_number)
else:
_result = None
return {'serial_number': serial_number, 'result': _result}
def test_default_get_storage_config():
Default(config_path='config/test.ini')
assert len(Default.instances) == 0
config = Default.get_config_instance()
storage_config = config.get_storage_config('worker')
assert storage_config == {
'source': 'kafka',
'topic': 'test',
'kafka': {
'bootstrap_servers': 'localhost:9092'
},
'group_id': 'codepack_worker_test',
'interval': '5',
'supervisor': 'http://localhost:8000',
'script_path': 'scripts/run_snapshot.py',
'logger': 'worker-logger'
}
def test_default_init_with_alias_path():
Default(alias_path='codepack/utils/config/default/alias.ini')
assert len(Default.instances) == 0
assert Default.config is not None
assert Default.config.config_path is None
assert Default.alias is not None
assert Default.alias.aliases is not None and isinstance(
Default.alias.aliases, dict)
def default_os_env():
os.environ['CODEPACK_CONFIG_DIR'] = 'config'
os.environ['CODEPACK_CONFIG_PATH'] = 'test.ini'
Default.get_service('delivery', 'delivery_service').storage.init()
Default.get_service('code_snapshot', 'snapshot_service').storage.init()
Default.get_service('code', 'storage_service').storage.init()
Default.get_service('codepack', 'storage_service').storage.init()
yield
os.environ.pop('CODEPACK_CONFIG_DIR', None)
os.environ.pop('CODEPACK_CONFIG_PATH', None)
def test_get_default_scheduler():
scheduler = Default.get_scheduler()
assert isinstance(scheduler, Scheduler)
assert 'codepack' in scheduler.jobstores
jobstore = scheduler.jobstores['codepack']
assert hasattr(jobstore, 'storage')
assert isinstance(jobstore.storage, MemoryStorage)
assert jobstore.storage.item_type == StorableJob
assert jobstore.storage.key == 'id'
assert scheduler.supervisor is None
def test_memory_worker_run_snapshot_with_nothing():
worker = Default.get_employee('worker')
assert isinstance(worker.messenger, MemoryMessenger)
code = Code(add2)
assert code.get_state() == 'UNKNOWN'
sn = worker.run_snapshot(code.to_snapshot(kwargs={'a': 3, 'b': 5}))
assert sn == code.serial_number
assert code.get_state() == 'TERMINATED'
assert code.get_result() == 8
worker.stop()
async def result(serial_number: str):
snapshot_service = Default.get_service('codepack_snapshot',
'snapshot_service')
snapshot = snapshot_service.load(serial_number=serial_number)
_result = None
if snapshot and snapshot['subscribe']:
codepack_snapshot = CodePackSnapshot.from_dict(snapshot)
codepack = CodePack.from_snapshot(codepack_snapshot)
_result = codepack.get_result()
return {'serial_number': serial_number, 'result': _result}
async def register_by_id(params: CodePackIDJob):
storage_service = Default.get_service('codepack', 'storage_service')
codepack = storage_service.load(params.id)
argpack = ArgPack.from_json(params.argpack)
common.scheduler.add_codepack(codepack=codepack,
argpack=argpack,
job_id=params.job_id,
trigger=params.trigger,
**params.trigger_config)
return {'serial_number': codepack.serial_number}
def test_default_memory_delivery_service_with_os_env():
config = Config()
storage_config = config.get_storage_config('delivery')
assert storage_config == {'source': 'memory'}
env_source = 'CODEPACK_DELIVERY_SOURCE'
try:
os.environ[env_source] = 'memory'
mds = Default.get_service('delivery', 'delivery_service')
assert isinstance(mds.storage, MemoryStorage)
finally:
for env in [env_source]:
os.environ.pop(env, None)
def test_default_init_with_config_path_and_alias_path_os_env():
try:
os.environ[
'CODEPACK_ALIAS_PATH'] = 'codepack/utils/config/default/alias.ini'
Default(config_path='config/test.ini')
assert len(Default.instances) == 0
assert Default.config is not None
assert Default.config.config_path == 'config/test.ini'
assert Default.alias is not None
assert Default.alias.aliases is None
finally:
os.environ.pop('CODEPACK_ALIAS_PATH', None)
async def state(serial_number: str):
snapshot_service = Default.get_service('code_snapshot', 'snapshot_service')
snapshot = snapshot_service.load(serial_number=serial_number, projection={'state', 'message'})
ret = {'serial_number': serial_number}
if snapshot:
_state = snapshot['state']
if snapshot['message']:
ret['message'] = snapshot['message']
else:
_state = 'UNKNOWN'
ret['state'] = _state
return ret
def test_default_init_with_os_env():
try:
os.environ['CODEPACK_CONFIG_DIR'] = 'config'
os.environ['CODEPACK_CONFIG_PATH'] = 'test.ini'
Default()
assert len(Default.instances) == 0
assert Default.config is not None
assert Default.config.config_path is None
assert Default.alias is not None
assert Default.alias.aliases is None
finally:
os.environ.pop('CODEPACK_CONFIG_DIR', None)
os.environ.pop('CODEPACK_CONFIG_PATH', None)
def test_get_default_worker_without_os_env(mock_docker_client):
worker = Default.get_employee('worker')
assert isinstance(worker, Worker)
assert isinstance(worker.messenger, MemoryMessenger)
assert worker.messenger.topic == 'codepack'
assert worker.supervisor is None
mock_docker_client.assert_called_once_with(
base_url='unix://var/run/docker.sock')
assert worker.docker_manager.docker.session == mock_docker_client()
assert isinstance(worker.callback_service, CallbackService)
assert isinstance(worker.callback_service.storage, FileStorage)
default_dir = Default.get_config_instance().get_default_config_dir()
assert worker.callback_service.storage.path == os.path.join(
default_dir, 'scripts')
assert isinstance(worker.logger, logging.Logger)
assert worker.logger.name == 'worker-logger'
assert hasattr(worker, 'script') and worker.script == 'run_snapshot.py'
assert hasattr(worker, 'script_dir') and worker.script_dir == os.path.join(
default_dir, 'scripts')
assert hasattr(worker,
'script_path') and worker.script_path == os.path.join(
default_dir, 'scripts/run_snapshot.py')
def test_default_memory_code_storage_service_with_os_env():
config = Config()
storage_config = config.get_storage_config('code')
assert storage_config == {'source': 'memory'}
env_source = 'CODEPACK_CODE_SOURCE'
try:
os.environ[env_source] = 'memory'
mss = Default.get_service('code', 'storage_service')
assert hasattr(mss.storage, 'memory')
assert mss.storage.item_type == Code
finally:
if env_source in os.environ:
os.environ.pop(env_source, None)
def test_get_default_supervisor():
try:
os.environ['CODEPACK_CODESNAPSHOT_SOURCE'] = 'mongodb'
os.environ['CODEPACK_CODESNAPSHOT_DB'] = 'test_db'
os.environ['CODEPACK_CODESNAPSHOT_COLLECTION'] = 'test_collection'
supervisor = Default.get_employee('supervisor')
assert isinstance(supervisor, Supervisor)
assert isinstance(supervisor.messenger, MemoryMessenger)
assert supervisor.messenger.topic == 'codepack'
assert isinstance(supervisor.snapshot_service, SnapshotService)
assert isinstance(supervisor.snapshot_service.storage, MongoStorage)
finally:
os.environ.pop('CODEPACK_CODESNAPSHOT_SOURCE', None)
os.environ.pop('CODEPACK_CODESNAPSHOT_DB', None)
os.environ.pop('CODEPACK_CODESNAPSHOT_COLLECTION', None)
async def state(serial_number: str):
snapshot_service = Default.get_service('codepack_snapshot',
'snapshot_service')
tmp = snapshot_service.load(serial_number=serial_number)
ret = {'serial_number': serial_number}
if tmp:
codepack_snapshot = CodePackSnapshot.from_dict(tmp)
codepack = CodePack.from_snapshot(codepack_snapshot)
_state = codepack.get_state()
if _state == 'ERROR':
ret['message'] = codepack.get_message()
else:
_state = 'UNKNOWN'
ret['state'] = _state
return ret
def test_default_get_alias_from_source():
assert Default.get_alias_from_source(source='memory') == 'memory'
assert Default.get_alias_from_source(source='memory',
prefix='front') == 'front_memory'
assert Default.get_alias_from_source(source='memory',
suffix='back') == 'memory_back'
assert Default.get_alias_from_source(source='memory',
prefix='front',
suffix='back') == 'front_memory_back'
assert Default.get_alias_from_source(source='mongodb') == 'mongo'
assert Default.get_alias_from_source(source='mongodb',
prefix='front') == 'front_mongo'
assert Default.get_alias_from_source(source='mongodb',
suffix='back') == 'mongo_back'
assert Default.get_alias_from_source(source='mongodb',
prefix='front',
suffix='back') == 'front_mongo_back'
def test_default_file_code_snapshot_service_with_os_env():
config = Config()
storage_config = config.get_storage_config('code_snapshot')
assert storage_config == {'source': 'memory'}
env_source = 'CODEPACK_CODESNAPSHOT_SOURCE'
env_path = 'CODEPACK_CODESNAPSHOT_PATH'
try:
os.environ[env_source] = 'file'
os.environ[env_path] = 'tmp/'
fss = Default.get_service('code_snapshot', 'snapshot_service')
assert hasattr(fss.storage, 'path')
assert fss.storage.path == 'tmp/'
finally:
for env in [env_source, env_path]:
os.environ.pop(env, None)
def test_default_file_delivery_service_with_os_env():
config = Config()
storage_config = config.get_storage_config('delivery')
assert storage_config == {'source': 'memory'}
env_source = 'CODEPACK_DELIVERY_SOURCE'
env_path = 'CODEPACK_DELIVERY_PATH'
try:
os.environ[env_source] = 'file'
os.environ[env_path] = 'tmp/'
fds = Default.get_service('delivery', 'delivery_service')
assert hasattr(fds.storage, 'path')
assert fds.storage.path == 'tmp/'
finally:
for env in [env_source, env_path]:
os.environ.pop(env, None)
