def test_collect_values_without_anything_but_os_env(): os_envs = { 'CODEPACK_CONFIG_PATH': 'config/test.ini', 'CODEPACK_WORKER_LOGGER': 'dummy-logger', 'CODEPACK_WORKER_DUMMY': 'dummy_value' } try: for k, v in os_envs.items(): os.environ[k] = v config = Config() ret = config.get_config('worker') assert ret == { 'background': 'True', 'dummy': 'dummy_value', 'group_id': 'codepack_worker_test', 'interval': '5', 'logger': 'dummy-logger', 'script_path': 'scripts/run_snapshot.py', 'source': 'kafka', 'supervisor': 'http://localhost:8000', 'topic': 'test' } finally: for k in os_envs.keys(): os.environ.pop(k, None)
def test_memory_worker_run_snapshot_with_image_and_callback( mock_docker_client): worker = Default.get_employee('worker') worker.callback = dummy_callback_function assert isinstance(worker.messenger, MemoryMessenger) code = Code(add2, image='dummy') assert code.get_state() == 'UNKNOWN' sn = worker.run_snapshot(code.to_snapshot(kwargs={'a': 3, 'b': 5})) assert sn == code.serial_number default_config_dir = Config.get_default_config_dir() script_dir = os.path.join(default_config_dir, 'scripts') mock_docker_client.return_value.containers.run.assert_called_once_with( auto_remove=True, command=[ 'python', 'run_snapshot.py', '%s.json' % sn, '-p', '.', '-l', 'worker-logger', '-c', 'dummy_callback_function' ], dns=['8.8.8.8'], environment=['CODEPACK_LOGGER_LOG_DIR=/usr/logs'], image='dummy', name=id(worker.docker_manager), volumes=[ '%s:/usr/src/codepack' % script_dir, '%s:/usr/logs' % os.path.abspath(Config.get_log_dir()) ], working_dir='/usr/src/codepack') worker.stop()
def test_config_path_priority(): try: config = Config() _config = config.get_config(section='logger', ignore_error=True) assert _config == { 'log_dir': 'logs', 'name': 'default-logger', 'config_path': os.path.join(config.get_default_config_dir(), 'logging.json') } with pytest.raises(AssertionError): Config(config_path='test.ini') os.environ['CODEPACK_CONFIG_DIR'] = 'config' config = Config(config_path='test.ini') assert config.config_path == os.path.join('config', 'test.ini') _config = config.get_config(section='logger') ref = { 'name': 'default-logger', 'config_path': 'logging.json', 'log_dir': 'logs' } assert _config == ref os.environ['CODEPACK_CONFIG_PATH'] = 'codepack.ini' _config = config.get_config(section='logger') assert _config == ref _config = config.get_config(section='logger', config_path='test.ini') assert _config == ref finally: os.environ.pop('CODEPACK_CONFIG_PATH', None) os.environ.pop('CODEPACK_CONFIG_DIR', None)
def test_get_storage_config_priority(): try: os.environ['CODEPACK_CONFIG_DIR'] = 'config' config = Config('test.ini') storage_config = config.get_storage_config('worker') ref = { 'group_id': 'codepack_worker_test', 'interval': '5', 'kafka': { 'bootstrap_servers': 'localhost:9092' }, 'script_path': 'scripts/run_snapshot.py', 'source': 'kafka', 'supervisor': 'http://localhost:8000', 'topic': 'test', 'logger': 'worker-logger' } assert storage_config == ref os.environ['CODEPACK_WORKER_SCRIPT'] = 'test_script.py' os.environ['CODEPACK_WORKER_TOPIC'] = 'test2' storage_config = config.get_storage_config('worker') assert storage_config == ref os.environ['CODEPACK_WORKER_TEST_KEY'] = 'test' storage_config = config.get_storage_config('worker') assert storage_config == ref finally: os.environ.pop('CODEPACK_CONFIG_DIR', None) os.environ.pop('CODEPACK_WORKER_SCRIPT', None) os.environ.pop('CODEPACK_WORKER_TOPIC', None) os.environ.pop('CODEPACK_WORKER_TEST_KEY', None)
def test_get_storage_config_with_os_env(): try: os.environ['CODEPACK_CONFIG_PATH'] = 'config/test.ini' config = Config() ret = config.get_storage_config('scheduler') ref = { 'collection': 'scheduler', 'db': 'codepack', 'source': 'mongodb', 'supervisor': 'http://localhost:8000', 'mongodb': { 'host': 'localhost', 'port': '27017' } } assert ret == ref os.environ['CODEPACK_SCHEDULER_DB'] = 'test' ret = config.get_storage_config('scheduler') ref['db'] = 'test' assert ret == ref os.environ['CODEPACK_MONGODB_HOST'] = '127.0.0.1' os.environ['CODEPACK_MONGODB_USER'] = '******' ret = config.get_storage_config('scheduler') ref['mongodb']['host'] = '127.0.0.1' ref['mongodb']['user'] = '******' assert ret == ref finally: os.environ.pop('CODEPACK_CONFIG_PATH', None) os.environ.pop('CODEPACK_SCHEDULER_DB', None) os.environ.pop('CODEPACK_MONGODB_HOST', None) os.environ.pop('CODEPACK_MONGODB_USER', None)
def test_collect_values_without_anything(): os_envs = { 'CODEPACK_WORKER_LOGGER': 'dummy-logger', 'CODEPACK_WORKER_DUMMY': 'dummy_value' } try: for k, v in os_envs.items(): os.environ[k] = v config = Config() ret = config.get_config('worker') assert ret == { 'dummy': 'dummy_value', 'background': 'True', 'interval': '1', 'logger': 'dummy-logger', 'source': 'memory', 'topic': 'codepack', 'script_path': os.path.join(config.get_default_config_dir(), 'scripts/run_snapshot.py') } finally: for k in os_envs.keys(): os.environ.pop(k, None)
def test_s3_storage_save(mock_client, dummy_deliveries): config = Config() s3_config = config.get_config('s3') ss = S3Storage(s3=s3_config, item_type=Delivery, key='id', bucket='test_bucket', path='test_path') mock_client.return_value.exceptions.ClientError = Exception mock_client.return_value.head_object.return_value = None obj_key1 = dummy_deliveries[0].get_path(key='obj1', path='test_path', posix=True) obj_key2 = dummy_deliveries[1].get_path(key='obj2', path='test_path', posix=True) ss.save(item=dummy_deliveries[0]) mock_client.return_value.put_object.assert_called_once_with( Body=dummy_deliveries[0].to_json(), Bucket='test_bucket', Key=obj_key1) mock_client.return_value.head_object.return_value = 'dummy' with pytest.raises(ValueError): ss.save(item=dummy_deliveries[1]) ss.save(item=dummy_deliveries[1], update=True) mock_client.return_value.put_object.assert_called_with( Body=dummy_deliveries[1].to_json(), Bucket='test_bucket', Key=obj_key2)
def test_get_config_with_constructor_argument_and_method_argument( parse_config): config = Config(config_path='config/test.ini') ret = config.get_config('worker', config_path='config/codepack.ini') assert ret is not None parse_config.assert_called_once_with(section='worker', config_path='config/codepack.ini')
def test_get_config_with_method_argument_and_os_env(parse_config): try: os.environ['CODEPACK_CONFIG_PATH'] = 'config/codepack.ini' config = Config() ret = config.get_config('worker', config_path='config/test.ini') assert ret is not None parse_config.assert_called_once_with(section='worker', config_path='config/test.ini') finally: os.environ.pop('CODEPACK_CONFIG_PATH')
def test_default_memory_delivery_service_with_os_env(): config = Config() storage_config = config.get_storage_config('delivery') assert storage_config == {'source': 'memory'} env_source = 'CODEPACK_DELIVERY_SOURCE' try: os.environ[env_source] = 'memory' mds = Default.get_service('delivery', 'delivery_service') assert isinstance(mds.storage, MemoryStorage) finally: for env in [env_source]: os.environ.pop(env, None)
def test_s3_storage_remove(mock_client): config = Config() s3_config = config.get_config('s3') ss = S3Storage(s3=s3_config, item_type=Delivery, key='id', bucket='test_bucket', path='test_path') ss.remove(key='test') mock_client.return_value.delete_object.assert_called_once_with( Bucket='test_bucket', Key='test_path/test.json') ss.remove(key=['test1', 'test2', 'test3']) assert mock_client.return_value.delete_object.call_count == 4
def test_default_memory_code_storage_service_with_os_env(): config = Config() storage_config = config.get_storage_config('code') assert storage_config == {'source': 'memory'} env_source = 'CODEPACK_CODE_SOURCE' try: os.environ[env_source] = 'memory' mss = Default.get_service('code', 'storage_service') assert hasattr(mss.storage, 'memory') assert mss.storage.item_type == Code finally: if env_source in os.environ: os.environ.pop(env_source, None)
def test_default_file_code_snapshot_service_with_os_env(): config = Config() storage_config = config.get_storage_config('code_snapshot') assert storage_config == {'source': 'memory'} env_source = 'CODEPACK_CODESNAPSHOT_SOURCE' env_path = 'CODEPACK_CODESNAPSHOT_PATH' try: os.environ[env_source] = 'file' os.environ[env_path] = 'tmp/' fss = Default.get_service('code_snapshot', 'snapshot_service') assert hasattr(fss.storage, 'path') assert fss.storage.path == 'tmp/' finally: for env in [env_source, env_path]: os.environ.pop(env, None)
def test_default_file_delivery_service_with_os_env(): config = Config() storage_config = config.get_storage_config('delivery') assert storage_config == {'source': 'memory'} env_source = 'CODEPACK_DELIVERY_SOURCE' env_path = 'CODEPACK_DELIVERY_PATH' try: os.environ[env_source] = 'file' os.environ[env_path] = 'tmp/' fds = Default.get_service('delivery', 'delivery_service') assert hasattr(fds.storage, 'path') assert fds.storage.path == 'tmp/' finally: for env in [env_source, env_path]: os.environ.pop(env, None)
def test_get_config_without_anything(parse_config): config = Config() ret = config.get_config('worker') assert ret is not None arg_list = parse_config.call_args_list assert len(arg_list) == 2 args, kwargs = arg_list[0] assert kwargs == { 'section': 'worker', 'config_path': config.get_default_config_path() } args, kwargs = arg_list[1] assert kwargs == { 'section': 'worker', 'config_path': config.get_default_config_path() }
def test_config_dir(): path = Config.collect_value(section='?', key='path', config={'path': 'config/test.ini'}) assert path == 'config/test.ini' with pytest.raises(AssertionError): Config.collect_value(section='conn', key='path', config={'path': 'test.ini'}) try: os.environ['CODEPACK_CONFIG_DIR'] = 'config' path = Config.collect_value(section='conn', key='path', config={'path': 'test.ini'}) assert path == os.path.join('config', 'test.ini') finally: os.environ.pop('CODEPACK_CONFIG_DIR', None)
def test_s3_storage_list_all(mock_client): config = Config() s3_config = config.get_config('s3') ss = S3Storage(s3=s3_config, item_type=Delivery, key='id', bucket='test_bucket', path='test_path') mock_client.return_value.get_paginator.return_value.paginate.return_value\ = [{'Contents': [{'Key': 'test_path/x1.json'}, {'Key': 'test_path/x2.json'}]}] k = sorted(ss.list_all()) mock_client.return_value.get_paginator.assert_called_once_with( 'list_objects') mock_client.return_value.get_paginator.return_value.paginate.assert_called_once_with( Bucket='test_bucket', Prefix='test_path/') assert k == ['x1', 'x2'] ss.close() assert not ss.s3
def test_s3_storage_update(mock_client, dummy_deliveries): config = Config() s3_config = config.get_config('s3') ss = S3Storage(s3=s3_config, item_type=Delivery, key='id', bucket='test_bucket', path='test_path') mock_client.return_value.exceptions.NoSuchKey = Exception mock_client.return_value.get_object.side_effect = partial( fake_get_object, dummy_deliveries) mock_client.return_value.exceptions.ClientError = Exception ss.update(key='obj2', values={'serial_number': 'test'}) obj1 = dummy_deliveries[0] obj_key1 = obj1.get_path(key='obj1', path='test_path', posix=True) obj2 = dummy_deliveries[1] obj_key2 = obj2.get_path(key='obj2', path='test_path', posix=True) obj3 = dummy_deliveries[2] obj_key3 = obj3.get_path(key='obj3', path='test_path', posix=True) obj1_dict = obj1.to_dict() obj1_dict['serial_number'] = 'hello' obj1_dict['_id'] = 'hello' obj2_dict = obj2.to_dict() obj2_dict['serial_number'] = 'test' obj2_dict['_id'] = 'test' obj3_dict = obj3.to_dict() obj3_dict['serial_number'] = 'hello' obj3_dict['_id'] = 'hello' mock_client.return_value.put_object.assert_called_once_with( Bucket='test_bucket', Key=obj_key2, Body=json.dumps(obj2_dict)) ss.update(key=['obj1', 'obj3'], values={'serial_number': 'hello'}) arg_list = mock_client.return_value.put_object.call_args_list assert len(arg_list) == 3 assert arg_list[1][1] == { 'Bucket': 'test_bucket', 'Key': obj_key1, 'Body': json.dumps(obj1_dict) } assert arg_list[2][1] == { 'Bucket': 'test_bucket', 'Key': obj_key3, 'Body': json.dumps(obj3_dict) }
def test_s3_storage_search(mock_client, dummy_deliveries): config = Config() s3_config = config.get_config('s3') ss = S3Storage(s3=s3_config, item_type=Delivery, key='id', bucket='test_bucket', path='test_path') ret = ss.search(key='item', value='y') mock_client.return_value.get_paginator.assert_called_once_with( 'list_objects') mock_client.return_value.get_paginator.return_value.paginate.assert_called_once_with( Bucket='test_bucket', Prefix='test_path/') assert ret == list() mock_client.return_value.get_paginator.return_value.paginate.return_value \ = [{'Contents': [{'Key': 'test_path/obj1.json'}, {'Key': 'test_path/obj2.json'}, {'Key': 'test_path/obj3.json'}]}] mock_client.return_value.exceptions.NoSuchKey = Exception mock_client.return_value.get_object.side_effect = partial( fake_get_object, dummy_deliveries) ret = ss.search(key='item', value=json.dumps('y')) assert len(ret) == 2 assert isinstance(ret[0], Delivery) assert sorted([obj.id for obj in ret]) == ['obj2', 'obj3'] arg_list = mock_client.return_value.get_object.call_args_list assert len(arg_list) == 3 args, kwargs = arg_list[0] assert kwargs.get('Bucket', '') == 'test_bucket' assert kwargs.get('Key', '') == 'test_path/obj1.json' args, kwargs = arg_list[1] assert kwargs.get('Bucket', '') == 'test_bucket' assert kwargs.get('Key', '') == 'test_path/obj2.json' args, kwargs = arg_list[2] assert kwargs.get('Bucket', '') == 'test_bucket' assert kwargs.get('Key', '') == 'test_path/obj3.json' ret = ss.search(key='item', value=json.dumps('y'), to_dict=True) assert len(ret) == 2 assert isinstance(ret[0], dict) ret = ss.search(key='item', value=json.dumps('y'), projection=['serial_number']) assert len(ret) == 2 assert isinstance(ret[0], dict) assert set(ret[0].keys()) == {'id', 'serial_number'}
def test_default_mongo_code_snapshot_service_with_os_env(): config = Config() storage_config = config.get_storage_config('code_snapshot') assert storage_config == {'source': 'memory'} env_source = 'CODEPACK_CODESNAPSHOT_SOURCE' env_db = 'CODEPACK_CODESNAPSHOT_DB' env_collection = 'CODEPACK_CODESNAPSHOT_COLLECTION' mss = None try: os.environ[env_source] = 'mongodb' os.environ[env_db] = 'test' os.environ[env_collection] = 'snapshot' mss = Default.get_service('code_snapshot', 'snapshot_service') finally: if mss is not None and not mss.storage.mongodb.closed(): mss.storage.mongodb.close() for env in [env_source, env_db, env_collection]: os.environ.pop(env, None) assert hasattr(mss.storage, 'mongodb')
def test_default_memory_code_snapshot_service_with_os_env(fake_mongodb): mss = None try: config = Config() storage_config = config.get_storage_config('code_snapshot') assert storage_config == {'source': 'memory'} os.environ['CODEPACK_CODESNAPSHOT_SOURCE'] = 'mongodb' os.environ['CODEPACK_CODESNAPSHOT_DB'] = 'test_db' os.environ['CODEPACK_CODESNAPSHOT_COLLECTION'] = 'test_collection' mss = Default.get_service('code_snapshot', 'snapshot_service') assert hasattr(mss.storage, 'mongodb') assert mss.storage.db == 'test_db' assert mss.storage.collection == 'test_collection' finally: if mss is not None and not mss.storage.mongodb.closed(): mss.storage.mongodb.close() os.environ.pop('CODEPACK_CODESNAPSHOT_SOURCE', None) os.environ.pop('CODEPACK_CODESNAPSHOT_DB', None) os.environ.pop('CODEPACK_CODESNAPSHOT_COLLECTION', None)
def test_get_config_without_anything_but_os_env(parse_config): os.environ['CODEPACK_CONFIG_PATH'] = 'config/test.ini' try: config = Config() ret = config.get_config('worker') assert ret is not None arg_list = parse_config.call_args_list assert len(arg_list) == 2 args, kwargs = arg_list[0] assert kwargs == { 'section': 'worker', 'config_path': 'config/test.ini' } args, kwargs = arg_list[1] assert kwargs == { 'section': 'worker', 'config_path': config.get_default_config_path() } finally: os.environ.pop('CODEPACK_CONFIG_PATH')
def test_s3_storage_exist(mock_client): config = Config() s3_config = config.get_config('s3') ss = S3Storage(s3=s3_config, item_type=Delivery, key='id', bucket='test_bucket', path='test_path') ret = ss.exist(key='test') mock_client.return_value.head_object.assert_called_once_with( Key='test_path/test.json', Bucket='test_bucket') assert ret is True ret = ss.exist(key=['test1', 'test2']) assert mock_client.return_value.head_object.call_count == 3 assert ret == [True, True] ret = ss.exist(key=['test1', 'test2'], summary='or') assert mock_client.return_value.head_object.call_count == 4 assert ret is True ret = ss.exist(key=['test1', 'test2'], summary='and') assert mock_client.return_value.head_object.call_count == 6 assert ret is True
def test_default_mongo_delivery_service_with_os_env(): config = Config() storage_config = config.get_storage_config('delivery') assert storage_config == {'source': 'memory'} env_source = 'CODEPACK_DELIVERY_SOURCE' env_db = 'CODEPACK_DELIVERY_DB' env_collection = 'CODEPACK_DELIVERY_COLLECTION' mds = None try: os.environ[env_source] = 'mongodb' os.environ[env_db] = 'test' os.environ[env_collection] = 'delivery' mds = Default.get_service('delivery', 'delivery_service') assert hasattr(mds.storage, 'mongodb') assert mds.storage.db == 'test' assert mds.storage.collection == 'delivery' finally: for env in [env_source, env_db, env_collection]: os.environ.pop(env, None) if mds is not None and not mds.storage.mongodb.closed(): mds.storage.mongodb.close()
def test_collect_values_with_constructor_argument(): os_envs = { 'CODEPACK_WORKER_LOGGER': 'dummy-logger', 'CODEPACK_WORKER_DUMMY': 'dummy_value' } try: for k, v in os_envs.items(): os.environ[k] = v config = Config(config_path='config/test.ini') ret = config.get_config('worker') assert ret == { 'group_id': 'codepack_worker_test', 'interval': '5', 'logger': 'worker-logger', 'script_path': 'scripts/run_snapshot.py', 'source': 'kafka', 'supervisor': 'http://localhost:8000', 'topic': 'test' } finally: for k in os_envs.keys(): os.environ.pop(k, None)
def test_s3_storage_init_with_dict(mock_client): config = Config() s3_config = config.get_config('s3') ss = S3Storage(s3=s3_config, item_type=Delivery, key='id') arg_list = mock_client.call_args_list assert len(arg_list) == 1 assert ss.new_connection args, kwargs = arg_list[0] assert kwargs.get('service_name', '') == 's3' assert kwargs.get('region_name', '') == '' assert kwargs.get('endpoint_url', '') == '' assert kwargs.get('aws_access_key_id', '') == 'codepack' assert kwargs.get('aws_secret_access_key', '') == 'codepack' assert 'config' in ss.s3.config and ss.s3.config['config'] == kwargs.get( 'config') assert kwargs.get('config').retries == {'max_attempts': 3} assert ss.s3.session is mock_client.return_value close_function = MagicMock() ss.s3.close = close_function ss.close() close_function.assert_called_once() assert not ss.s3
def test_default_mongo_code_storage_service_with_os_env(): config = Config() storage_config = config.get_storage_config('code') assert storage_config == {'source': 'memory'} env_source = 'CODEPACK_CODE_SOURCE' env_db = 'CODEPACK_CODE_DB' env_collection = 'CODEPACK_CODE_COLLECTION' mss = None try: os.environ[env_source] = 'mongodb' os.environ[env_db] = 'test' os.environ[env_collection] = 'codes' mss = Default.get_service('code', 'storage_service') assert hasattr(mss.storage, 'mongodb') assert mss.storage.item_type == Code assert mss.storage.db == 'test' assert mss.storage.collection == 'codes' finally: for env in [env_source, env_db, env_collection]: os.environ.pop(env, None) if mss is not None and not mss.storage.mongodb.closed(): mss.storage.mongodb.close()
def test_s3_storage_load(mock_client, dummy_deliveries): config = Config() s3_config = config.get_config('s3') ss = S3Storage(s3=s3_config, item_type=Delivery, key='id', bucket='test_bucket', path='test_path') mock_client.return_value.exceptions.NoSuchKey = Exception mock_client.return_value.get_object.side_effect = partial( fake_get_object, dummy_deliveries) ret = ss.load(key='obj2') assert isinstance(ret, Delivery) assert ret.to_dict() == dummy_deliveries[1].to_dict() ret = ss.load(key='obj1', to_dict=True) assert isinstance(ret, dict) assert ret == dummy_deliveries[0].to_dict() ret = ss.load(key='obj3', projection=['serial_number']) assert isinstance(ret, dict) assert set(ret.keys()) == {'id', 'serial_number'} ret = ss.load(key=['obj2', 'obj4', 'obj3']) assert len(ret) == 2 assert {x.id for x in ret} == {'obj2', 'obj3'}
def test_memory_worker_run_snapshot_with_env(mock_subprocess_run): worker = Default.get_employee('worker') assert isinstance(worker.messenger, MemoryMessenger) code = Code(add2, env='test_env', image='dummy') assert code.get_state() == 'UNKNOWN' sn = worker.run_snapshot(code.to_snapshot(kwargs={'a': 3, 'b': 5})) assert sn == code.serial_number default_config_dir = Config.get_default_config_dir() script_dir = os.path.join(default_config_dir, 'scripts') mock_subprocess_run.assert_called_once_with([ os.path.join(worker.interpreter_manager.path, 'test_env', 'bin', 'python'), os.path.join(script_dir, 'run_snapshot.py'), os.path.join(script_dir, '%s.json' % sn), '-p', script_dir, '-l', 'worker-logger' ]) worker.stop()
def test_config_get_value_priority(): try: config = Config() _config = config.get_config(section='logger') assert _config == { 'name': 'default-logger', 'log_dir': 'logs', 'config_path': os.path.join(config.get_default_config_dir(), 'logging.json') } default_value = Config.collect_value(section='logger', key='name', config=dict()) assert default_value == 'default-logger' name = Config.collect_value(section='logger', key='name', config=_config) assert name == 'default-logger' os.environ['CODEPACK_LOGGER_NAME'] = 'test-logger' name = Config.collect_value(section='logger', key='name', config=_config) assert name == 'test-logger' os.environ.pop('CODEPACK_LOGGER_NAME') os.environ['CODEPACK_CONFIG_PATH'] = 'config/test.ini' os.environ['CODEPACK_CONFIG_DIR'] = 'config' config = Config() _config = config.get_config(section='logger') assert _config == { 'name': 'default-logger', 'config_path': os.path.join('config', 'logging.json'), 'log_dir': 'logs' } name = Config.collect_value(section='logger', key='name', config=_config) assert name == 'default-logger' os.environ['CODEPACK_LOGGER_NAME'] = 'test-logger' name = Config.collect_value(section='logger', key='name', config=_config) assert name == 'test-logger' finally: os.environ.pop('CODEPACK_LOGGER_NAME', None) os.environ.pop('CODEPACK_CONFIG_PATH', None) os.environ.pop('CODEPACK_CONFIG_DIR', None)