def test_config_path_priority(): try: config = Config() _config = config.get_config(section='logger', ignore_error=True) assert _config == { 'log_dir': 'logs', 'name': 'default-logger', 'config_path': os.path.join(config.get_default_config_dir(), 'logging.json') } with pytest.raises(AssertionError): Config(config_path='test.ini') os.environ['CODEPACK_CONFIG_DIR'] = 'config' config = Config(config_path='test.ini') assert config.config_path == os.path.join('config', 'test.ini') _config = config.get_config(section='logger') ref = { 'name': 'default-logger', 'config_path': 'logging.json', 'log_dir': 'logs' } assert _config == ref os.environ['CODEPACK_CONFIG_PATH'] = 'codepack.ini' _config = config.get_config(section='logger') assert _config == ref _config = config.get_config(section='logger', config_path='test.ini') assert _config == ref finally: os.environ.pop('CODEPACK_CONFIG_PATH', None) os.environ.pop('CODEPACK_CONFIG_DIR', None)
def test_config_get_value_priority(): try: config = Config() _config = config.get_config(section='logger') assert _config == { 'name': 'default-logger', 'log_dir': 'logs', 'config_path': os.path.join(config.get_default_config_dir(), 'logging.json') } default_value = Config.collect_value(section='logger', key='name', config=dict()) assert default_value == 'default-logger' name = Config.collect_value(section='logger', key='name', config=_config) assert name == 'default-logger' os.environ['CODEPACK_LOGGER_NAME'] = 'test-logger' name = Config.collect_value(section='logger', key='name', config=_config) assert name == 'test-logger' os.environ.pop('CODEPACK_LOGGER_NAME') os.environ['CODEPACK_CONFIG_PATH'] = 'config/test.ini' os.environ['CODEPACK_CONFIG_DIR'] = 'config' config = Config() _config = config.get_config(section='logger') assert _config == { 'name': 'default-logger', 'config_path': os.path.join('config', 'logging.json'), 'log_dir': 'logs' } name = Config.collect_value(section='logger', key='name', config=_config) assert name == 'default-logger' os.environ['CODEPACK_LOGGER_NAME'] = 'test-logger' name = Config.collect_value(section='logger', key='name', config=_config) assert name == 'test-logger' finally: os.environ.pop('CODEPACK_LOGGER_NAME', None) os.environ.pop('CODEPACK_CONFIG_PATH', None) os.environ.pop('CODEPACK_CONFIG_DIR', None)
def test_collect_values_without_anything_but_os_env(): os_envs = { 'CODEPACK_CONFIG_PATH': 'config/test.ini', 'CODEPACK_WORKER_LOGGER': 'dummy-logger', 'CODEPACK_WORKER_DUMMY': 'dummy_value' } try: for k, v in os_envs.items(): os.environ[k] = v config = Config() ret = config.get_config('worker') assert ret == { 'background': 'True', 'dummy': 'dummy_value', 'group_id': 'codepack_worker_test', 'interval': '5', 'logger': 'dummy-logger', 'script_path': 'scripts/run_snapshot.py', 'source': 'kafka', 'supervisor': 'http://localhost:8000', 'topic': 'test' } finally: for k in os_envs.keys(): os.environ.pop(k, None)
def test_collect_values_without_anything(): os_envs = { 'CODEPACK_WORKER_LOGGER': 'dummy-logger', 'CODEPACK_WORKER_DUMMY': 'dummy_value' } try: for k, v in os_envs.items(): os.environ[k] = v config = Config() ret = config.get_config('worker') assert ret == { 'dummy': 'dummy_value', 'background': 'True', 'interval': '1', 'logger': 'dummy-logger', 'source': 'memory', 'topic': 'codepack', 'script_path': os.path.join(config.get_default_config_dir(), 'scripts/run_snapshot.py') } finally: for k in os_envs.keys(): os.environ.pop(k, None)
def test_get_config_with_constructor_argument_and_method_argument( parse_config): config = Config(config_path='config/test.ini') ret = config.get_config('worker', config_path='config/codepack.ini') assert ret is not None parse_config.assert_called_once_with(section='worker', config_path='config/codepack.ini')
def test_s3_storage_save(mock_client, dummy_deliveries): config = Config() s3_config = config.get_config('s3') ss = S3Storage(s3=s3_config, item_type=Delivery, key='id', bucket='test_bucket', path='test_path') mock_client.return_value.exceptions.ClientError = Exception mock_client.return_value.head_object.return_value = None obj_key1 = dummy_deliveries[0].get_path(key='obj1', path='test_path', posix=True) obj_key2 = dummy_deliveries[1].get_path(key='obj2', path='test_path', posix=True) ss.save(item=dummy_deliveries[0]) mock_client.return_value.put_object.assert_called_once_with( Body=dummy_deliveries[0].to_json(), Bucket='test_bucket', Key=obj_key1) mock_client.return_value.head_object.return_value = 'dummy' with pytest.raises(ValueError): ss.save(item=dummy_deliveries[1]) ss.save(item=dummy_deliveries[1], update=True) mock_client.return_value.put_object.assert_called_with( Body=dummy_deliveries[1].to_json(), Bucket='test_bucket', Key=obj_key2)
def test_get_config_with_method_argument_and_os_env(parse_config): try: os.environ['CODEPACK_CONFIG_PATH'] = 'config/codepack.ini' config = Config() ret = config.get_config('worker', config_path='config/test.ini') assert ret is not None parse_config.assert_called_once_with(section='worker', config_path='config/test.ini') finally: os.environ.pop('CODEPACK_CONFIG_PATH')
def test_s3_storage_remove(mock_client): config = Config() s3_config = config.get_config('s3') ss = S3Storage(s3=s3_config, item_type=Delivery, key='id', bucket='test_bucket', path='test_path') ss.remove(key='test') mock_client.return_value.delete_object.assert_called_once_with( Bucket='test_bucket', Key='test_path/test.json') ss.remove(key=['test1', 'test2', 'test3']) assert mock_client.return_value.delete_object.call_count == 4
def test_get_config_without_anything(parse_config): config = Config() ret = config.get_config('worker') assert ret is not None arg_list = parse_config.call_args_list assert len(arg_list) == 2 args, kwargs = arg_list[0] assert kwargs == { 'section': 'worker', 'config_path': config.get_default_config_path() } args, kwargs = arg_list[1] assert kwargs == { 'section': 'worker', 'config_path': config.get_default_config_path() }
def test_s3_storage_list_all(mock_client): config = Config() s3_config = config.get_config('s3') ss = S3Storage(s3=s3_config, item_type=Delivery, key='id', bucket='test_bucket', path='test_path') mock_client.return_value.get_paginator.return_value.paginate.return_value\ = [{'Contents': [{'Key': 'test_path/x1.json'}, {'Key': 'test_path/x2.json'}]}] k = sorted(ss.list_all()) mock_client.return_value.get_paginator.assert_called_once_with( 'list_objects') mock_client.return_value.get_paginator.return_value.paginate.assert_called_once_with( Bucket='test_bucket', Prefix='test_path/') assert k == ['x1', 'x2'] ss.close() assert not ss.s3
def test_s3_storage_update(mock_client, dummy_deliveries): config = Config() s3_config = config.get_config('s3') ss = S3Storage(s3=s3_config, item_type=Delivery, key='id', bucket='test_bucket', path='test_path') mock_client.return_value.exceptions.NoSuchKey = Exception mock_client.return_value.get_object.side_effect = partial( fake_get_object, dummy_deliveries) mock_client.return_value.exceptions.ClientError = Exception ss.update(key='obj2', values={'serial_number': 'test'}) obj1 = dummy_deliveries[0] obj_key1 = obj1.get_path(key='obj1', path='test_path', posix=True) obj2 = dummy_deliveries[1] obj_key2 = obj2.get_path(key='obj2', path='test_path', posix=True) obj3 = dummy_deliveries[2] obj_key3 = obj3.get_path(key='obj3', path='test_path', posix=True) obj1_dict = obj1.to_dict() obj1_dict['serial_number'] = 'hello' obj1_dict['_id'] = 'hello' obj2_dict = obj2.to_dict() obj2_dict['serial_number'] = 'test' obj2_dict['_id'] = 'test' obj3_dict = obj3.to_dict() obj3_dict['serial_number'] = 'hello' obj3_dict['_id'] = 'hello' mock_client.return_value.put_object.assert_called_once_with( Bucket='test_bucket', Key=obj_key2, Body=json.dumps(obj2_dict)) ss.update(key=['obj1', 'obj3'], values={'serial_number': 'hello'}) arg_list = mock_client.return_value.put_object.call_args_list assert len(arg_list) == 3 assert arg_list[1][1] == { 'Bucket': 'test_bucket', 'Key': obj_key1, 'Body': json.dumps(obj1_dict) } assert arg_list[2][1] == { 'Bucket': 'test_bucket', 'Key': obj_key3, 'Body': json.dumps(obj3_dict) }
def test_s3_storage_search(mock_client, dummy_deliveries): config = Config() s3_config = config.get_config('s3') ss = S3Storage(s3=s3_config, item_type=Delivery, key='id', bucket='test_bucket', path='test_path') ret = ss.search(key='item', value='y') mock_client.return_value.get_paginator.assert_called_once_with( 'list_objects') mock_client.return_value.get_paginator.return_value.paginate.assert_called_once_with( Bucket='test_bucket', Prefix='test_path/') assert ret == list() mock_client.return_value.get_paginator.return_value.paginate.return_value \ = [{'Contents': [{'Key': 'test_path/obj1.json'}, {'Key': 'test_path/obj2.json'}, {'Key': 'test_path/obj3.json'}]}] mock_client.return_value.exceptions.NoSuchKey = Exception mock_client.return_value.get_object.side_effect = partial( fake_get_object, dummy_deliveries) ret = ss.search(key='item', value=json.dumps('y')) assert len(ret) == 2 assert isinstance(ret[0], Delivery) assert sorted([obj.id for obj in ret]) == ['obj2', 'obj3'] arg_list = mock_client.return_value.get_object.call_args_list assert len(arg_list) == 3 args, kwargs = arg_list[0] assert kwargs.get('Bucket', '') == 'test_bucket' assert kwargs.get('Key', '') == 'test_path/obj1.json' args, kwargs = arg_list[1] assert kwargs.get('Bucket', '') == 'test_bucket' assert kwargs.get('Key', '') == 'test_path/obj2.json' args, kwargs = arg_list[2] assert kwargs.get('Bucket', '') == 'test_bucket' assert kwargs.get('Key', '') == 'test_path/obj3.json' ret = ss.search(key='item', value=json.dumps('y'), to_dict=True) assert len(ret) == 2 assert isinstance(ret[0], dict) ret = ss.search(key='item', value=json.dumps('y'), projection=['serial_number']) assert len(ret) == 2 assert isinstance(ret[0], dict) assert set(ret[0].keys()) == {'id', 'serial_number'}
def test_get_config_without_anything_but_os_env(parse_config): os.environ['CODEPACK_CONFIG_PATH'] = 'config/test.ini' try: config = Config() ret = config.get_config('worker') assert ret is not None arg_list = parse_config.call_args_list assert len(arg_list) == 2 args, kwargs = arg_list[0] assert kwargs == { 'section': 'worker', 'config_path': 'config/test.ini' } args, kwargs = arg_list[1] assert kwargs == { 'section': 'worker', 'config_path': config.get_default_config_path() } finally: os.environ.pop('CODEPACK_CONFIG_PATH')
def test_s3_storage_exist(mock_client): config = Config() s3_config = config.get_config('s3') ss = S3Storage(s3=s3_config, item_type=Delivery, key='id', bucket='test_bucket', path='test_path') ret = ss.exist(key='test') mock_client.return_value.head_object.assert_called_once_with( Key='test_path/test.json', Bucket='test_bucket') assert ret is True ret = ss.exist(key=['test1', 'test2']) assert mock_client.return_value.head_object.call_count == 3 assert ret == [True, True] ret = ss.exist(key=['test1', 'test2'], summary='or') assert mock_client.return_value.head_object.call_count == 4 assert ret is True ret = ss.exist(key=['test1', 'test2'], summary='and') assert mock_client.return_value.head_object.call_count == 6 assert ret is True
def test_s3_storage_init_with_dict(mock_client): config = Config() s3_config = config.get_config('s3') ss = S3Storage(s3=s3_config, item_type=Delivery, key='id') arg_list = mock_client.call_args_list assert len(arg_list) == 1 assert ss.new_connection args, kwargs = arg_list[0] assert kwargs.get('service_name', '') == 's3' assert kwargs.get('region_name', '') == '' assert kwargs.get('endpoint_url', '') == '' assert kwargs.get('aws_access_key_id', '') == 'codepack' assert kwargs.get('aws_secret_access_key', '') == 'codepack' assert 'config' in ss.s3.config and ss.s3.config['config'] == kwargs.get( 'config') assert kwargs.get('config').retries == {'max_attempts': 3} assert ss.s3.session is mock_client.return_value close_function = MagicMock() ss.s3.close = close_function ss.close() close_function.assert_called_once() assert not ss.s3
def test_collect_values_with_constructor_argument(): os_envs = { 'CODEPACK_WORKER_LOGGER': 'dummy-logger', 'CODEPACK_WORKER_DUMMY': 'dummy_value' } try: for k, v in os_envs.items(): os.environ[k] = v config = Config(config_path='config/test.ini') ret = config.get_config('worker') assert ret == { 'group_id': 'codepack_worker_test', 'interval': '5', 'logger': 'worker-logger', 'script_path': 'scripts/run_snapshot.py', 'source': 'kafka', 'supervisor': 'http://localhost:8000', 'topic': 'test' } finally: for k in os_envs.keys(): os.environ.pop(k, None)
def test_s3_storage_load(mock_client, dummy_deliveries): config = Config() s3_config = config.get_config('s3') ss = S3Storage(s3=s3_config, item_type=Delivery, key='id', bucket='test_bucket', path='test_path') mock_client.return_value.exceptions.NoSuchKey = Exception mock_client.return_value.get_object.side_effect = partial( fake_get_object, dummy_deliveries) ret = ss.load(key='obj2') assert isinstance(ret, Delivery) assert ret.to_dict() == dummy_deliveries[1].to_dict() ret = ss.load(key='obj1', to_dict=True) assert isinstance(ret, dict) assert ret == dummy_deliveries[0].to_dict() ret = ss.load(key='obj3', projection=['serial_number']) assert isinstance(ret, dict) assert set(ret.keys()) == {'id', 'serial_number'} ret = ss.load(key=['obj2', 'obj4', 'obj3']) assert len(ret) == 2 assert {x.id for x in ret} == {'obj2', 'obj3'}
def test_get_config_without_anything_but_method_argument(parse_config): config = Config() ret = config.get_config('worker', config_path='config/test.ini') assert ret is not None parse_config.assert_called_once_with(section='worker', config_path='config/test.ini')