def test_config_path_priority():
try:
config = Config()
_config = config.get_config(section='logger', ignore_error=True)
assert _config == {
'log_dir':
'logs',
'name':
'default-logger',
'config_path':
os.path.join(config.get_default_config_dir(), 'logging.json')
}
with pytest.raises(AssertionError):
Config(config_path='test.ini')
os.environ['CODEPACK_CONFIG_DIR'] = 'config'
config = Config(config_path='test.ini')
assert config.config_path == os.path.join('config', 'test.ini')
_config = config.get_config(section='logger')
ref = {
'name': 'default-logger',
'config_path': 'logging.json',
'log_dir': 'logs'
}
assert _config == ref
os.environ['CODEPACK_CONFIG_PATH'] = 'codepack.ini'
_config = config.get_config(section='logger')
assert _config == ref
_config = config.get_config(section='logger', config_path='test.ini')
assert _config == ref
finally:
os.environ.pop('CODEPACK_CONFIG_PATH', None)
os.environ.pop('CODEPACK_CONFIG_DIR', None)
def test_config_get_value_priority():
try:
config = Config()
_config = config.get_config(section='logger')
assert _config == {
'name':
'default-logger',
'log_dir':
'logs',
'config_path':
os.path.join(config.get_default_config_dir(), 'logging.json')
}
default_value = Config.collect_value(section='logger',
key='name',
config=dict())
assert default_value == 'default-logger'
name = Config.collect_value(section='logger',
key='name',
config=_config)
assert name == 'default-logger'
os.environ['CODEPACK_LOGGER_NAME'] = 'test-logger'
name = Config.collect_value(section='logger',
key='name',
config=_config)
assert name == 'test-logger'
os.environ.pop('CODEPACK_LOGGER_NAME')
os.environ['CODEPACK_CONFIG_PATH'] = 'config/test.ini'
os.environ['CODEPACK_CONFIG_DIR'] = 'config'
config = Config()
_config = config.get_config(section='logger')
assert _config == {
'name': 'default-logger',
'config_path': os.path.join('config', 'logging.json'),
'log_dir': 'logs'
}
name = Config.collect_value(section='logger',
key='name',
config=_config)
assert name == 'default-logger'
os.environ['CODEPACK_LOGGER_NAME'] = 'test-logger'
name = Config.collect_value(section='logger',
key='name',
config=_config)
assert name == 'test-logger'
finally:
os.environ.pop('CODEPACK_LOGGER_NAME', None)
os.environ.pop('CODEPACK_CONFIG_PATH', None)
os.environ.pop('CODEPACK_CONFIG_DIR', None)
def test_collect_values_without_anything_but_os_env():
os_envs = {
'CODEPACK_CONFIG_PATH': 'config/test.ini',
'CODEPACK_WORKER_LOGGER': 'dummy-logger',
'CODEPACK_WORKER_DUMMY': 'dummy_value'
}
try:
for k, v in os_envs.items():
os.environ[k] = v
config = Config()
ret = config.get_config('worker')
assert ret == {
'background': 'True',
'dummy': 'dummy_value',
'group_id': 'codepack_worker_test',
'interval': '5',
'logger': 'dummy-logger',
'script_path': 'scripts/run_snapshot.py',
'source': 'kafka',
'supervisor': 'http://localhost:8000',
'topic': 'test'
}
finally:
for k in os_envs.keys():
os.environ.pop(k, None)
def test_collect_values_without_anything():
os_envs = {
'CODEPACK_WORKER_LOGGER': 'dummy-logger',
'CODEPACK_WORKER_DUMMY': 'dummy_value'
}
try:
for k, v in os_envs.items():
os.environ[k] = v
config = Config()
ret = config.get_config('worker')
assert ret == {
'dummy':
'dummy_value',
'background':
'True',
'interval':
'1',
'logger':
'dummy-logger',
'source':
'memory',
'topic':
'codepack',
'script_path':
os.path.join(config.get_default_config_dir(),
'scripts/run_snapshot.py')
}
finally:
for k in os_envs.keys():
os.environ.pop(k, None)
def test_get_config_with_constructor_argument_and_method_argument(
parse_config):
config = Config(config_path='config/test.ini')
ret = config.get_config('worker', config_path='config/codepack.ini')
assert ret is not None
parse_config.assert_called_once_with(section='worker',
config_path='config/codepack.ini')
def test_s3_storage_save(mock_client, dummy_deliveries):
config = Config()
s3_config = config.get_config('s3')
ss = S3Storage(s3=s3_config,
item_type=Delivery,
key='id',
bucket='test_bucket',
path='test_path')
mock_client.return_value.exceptions.ClientError = Exception
mock_client.return_value.head_object.return_value = None
obj_key1 = dummy_deliveries[0].get_path(key='obj1',
path='test_path',
posix=True)
obj_key2 = dummy_deliveries[1].get_path(key='obj2',
path='test_path',
posix=True)
ss.save(item=dummy_deliveries[0])
mock_client.return_value.put_object.assert_called_once_with(
Body=dummy_deliveries[0].to_json(), Bucket='test_bucket', Key=obj_key1)
mock_client.return_value.head_object.return_value = 'dummy'
with pytest.raises(ValueError):
ss.save(item=dummy_deliveries[1])
ss.save(item=dummy_deliveries[1], update=True)
mock_client.return_value.put_object.assert_called_with(
Body=dummy_deliveries[1].to_json(), Bucket='test_bucket', Key=obj_key2)
def test_get_config_with_method_argument_and_os_env(parse_config):
try:
os.environ['CODEPACK_CONFIG_PATH'] = 'config/codepack.ini'
config = Config()
ret = config.get_config('worker', config_path='config/test.ini')
assert ret is not None
parse_config.assert_called_once_with(section='worker',
config_path='config/test.ini')
finally:
os.environ.pop('CODEPACK_CONFIG_PATH')
def test_s3_storage_remove(mock_client):
config = Config()
s3_config = config.get_config('s3')
ss = S3Storage(s3=s3_config,
item_type=Delivery,
key='id',
bucket='test_bucket',
path='test_path')
ss.remove(key='test')
mock_client.return_value.delete_object.assert_called_once_with(
Bucket='test_bucket', Key='test_path/test.json')
ss.remove(key=['test1', 'test2', 'test3'])
assert mock_client.return_value.delete_object.call_count == 4
def test_get_config_without_anything(parse_config):
config = Config()
ret = config.get_config('worker')
assert ret is not None
arg_list = parse_config.call_args_list
assert len(arg_list) == 2
args, kwargs = arg_list[0]
assert kwargs == {
'section': 'worker',
'config_path': config.get_default_config_path()
}
args, kwargs = arg_list[1]
assert kwargs == {
'section': 'worker',
'config_path': config.get_default_config_path()
}
def test_s3_storage_list_all(mock_client):
config = Config()
s3_config = config.get_config('s3')
ss = S3Storage(s3=s3_config,
item_type=Delivery,
key='id',
bucket='test_bucket',
path='test_path')
mock_client.return_value.get_paginator.return_value.paginate.return_value\
= [{'Contents': [{'Key': 'test_path/x1.json'}, {'Key': 'test_path/x2.json'}]}]
k = sorted(ss.list_all())
mock_client.return_value.get_paginator.assert_called_once_with(
'list_objects')
mock_client.return_value.get_paginator.return_value.paginate.assert_called_once_with(
Bucket='test_bucket', Prefix='test_path/')
assert k == ['x1', 'x2']
ss.close()
assert not ss.s3
def test_s3_storage_update(mock_client, dummy_deliveries):
config = Config()
s3_config = config.get_config('s3')
ss = S3Storage(s3=s3_config,
item_type=Delivery,
key='id',
bucket='test_bucket',
path='test_path')
mock_client.return_value.exceptions.NoSuchKey = Exception
mock_client.return_value.get_object.side_effect = partial(
fake_get_object, dummy_deliveries)
mock_client.return_value.exceptions.ClientError = Exception
ss.update(key='obj2', values={'serial_number': 'test'})
obj1 = dummy_deliveries[0]
obj_key1 = obj1.get_path(key='obj1', path='test_path', posix=True)
obj2 = dummy_deliveries[1]
obj_key2 = obj2.get_path(key='obj2', path='test_path', posix=True)
obj3 = dummy_deliveries[2]
obj_key3 = obj3.get_path(key='obj3', path='test_path', posix=True)
obj1_dict = obj1.to_dict()
obj1_dict['serial_number'] = 'hello'
obj1_dict['_id'] = 'hello'
obj2_dict = obj2.to_dict()
obj2_dict['serial_number'] = 'test'
obj2_dict['_id'] = 'test'
obj3_dict = obj3.to_dict()
obj3_dict['serial_number'] = 'hello'
obj3_dict['_id'] = 'hello'
mock_client.return_value.put_object.assert_called_once_with(
Bucket='test_bucket', Key=obj_key2, Body=json.dumps(obj2_dict))
ss.update(key=['obj1', 'obj3'], values={'serial_number': 'hello'})
arg_list = mock_client.return_value.put_object.call_args_list
assert len(arg_list) == 3
assert arg_list[1][1] == {
'Bucket': 'test_bucket',
'Key': obj_key1,
'Body': json.dumps(obj1_dict)
}
assert arg_list[2][1] == {
'Bucket': 'test_bucket',
'Key': obj_key3,
'Body': json.dumps(obj3_dict)
}
def test_s3_storage_search(mock_client, dummy_deliveries):
config = Config()
s3_config = config.get_config('s3')
ss = S3Storage(s3=s3_config,
item_type=Delivery,
key='id',
bucket='test_bucket',
path='test_path')
ret = ss.search(key='item', value='y')
mock_client.return_value.get_paginator.assert_called_once_with(
'list_objects')
mock_client.return_value.get_paginator.return_value.paginate.assert_called_once_with(
Bucket='test_bucket', Prefix='test_path/')
assert ret == list()
mock_client.return_value.get_paginator.return_value.paginate.return_value \
= [{'Contents': [{'Key': 'test_path/obj1.json'}, {'Key': 'test_path/obj2.json'},
{'Key': 'test_path/obj3.json'}]}]
mock_client.return_value.exceptions.NoSuchKey = Exception
mock_client.return_value.get_object.side_effect = partial(
fake_get_object, dummy_deliveries)
ret = ss.search(key='item', value=json.dumps('y'))
assert len(ret) == 2
assert isinstance(ret[0], Delivery)
assert sorted([obj.id for obj in ret]) == ['obj2', 'obj3']
arg_list = mock_client.return_value.get_object.call_args_list
assert len(arg_list) == 3
args, kwargs = arg_list[0]
assert kwargs.get('Bucket', '') == 'test_bucket'
assert kwargs.get('Key', '') == 'test_path/obj1.json'
args, kwargs = arg_list[1]
assert kwargs.get('Bucket', '') == 'test_bucket'
assert kwargs.get('Key', '') == 'test_path/obj2.json'
args, kwargs = arg_list[2]
assert kwargs.get('Bucket', '') == 'test_bucket'
assert kwargs.get('Key', '') == 'test_path/obj3.json'
ret = ss.search(key='item', value=json.dumps('y'), to_dict=True)
assert len(ret) == 2
assert isinstance(ret[0], dict)
ret = ss.search(key='item',
value=json.dumps('y'),
projection=['serial_number'])
assert len(ret) == 2
assert isinstance(ret[0], dict)
assert set(ret[0].keys()) == {'id', 'serial_number'}
def test_get_config_without_anything_but_os_env(parse_config):
os.environ['CODEPACK_CONFIG_PATH'] = 'config/test.ini'
try:
config = Config()
ret = config.get_config('worker')
assert ret is not None
arg_list = parse_config.call_args_list
assert len(arg_list) == 2
args, kwargs = arg_list[0]
assert kwargs == {
'section': 'worker',
'config_path': 'config/test.ini'
}
args, kwargs = arg_list[1]
assert kwargs == {
'section': 'worker',
'config_path': config.get_default_config_path()
}
finally:
os.environ.pop('CODEPACK_CONFIG_PATH')
def test_s3_storage_exist(mock_client):
config = Config()
s3_config = config.get_config('s3')
ss = S3Storage(s3=s3_config,
item_type=Delivery,
key='id',
bucket='test_bucket',
path='test_path')
ret = ss.exist(key='test')
mock_client.return_value.head_object.assert_called_once_with(
Key='test_path/test.json', Bucket='test_bucket')
assert ret is True
ret = ss.exist(key=['test1', 'test2'])
assert mock_client.return_value.head_object.call_count == 3
assert ret == [True, True]
ret = ss.exist(key=['test1', 'test2'], summary='or')
assert mock_client.return_value.head_object.call_count == 4
assert ret is True
ret = ss.exist(key=['test1', 'test2'], summary='and')
assert mock_client.return_value.head_object.call_count == 6
assert ret is True
def test_s3_storage_init_with_dict(mock_client):
config = Config()
s3_config = config.get_config('s3')
ss = S3Storage(s3=s3_config, item_type=Delivery, key='id')
arg_list = mock_client.call_args_list
assert len(arg_list) == 1
assert ss.new_connection
args, kwargs = arg_list[0]
assert kwargs.get('service_name', '') == 's3'
assert kwargs.get('region_name', '') == ''
assert kwargs.get('endpoint_url', '') == ''
assert kwargs.get('aws_access_key_id', '') == 'codepack'
assert kwargs.get('aws_secret_access_key', '') == 'codepack'
assert 'config' in ss.s3.config and ss.s3.config['config'] == kwargs.get(
'config')
assert kwargs.get('config').retries == {'max_attempts': 3}
assert ss.s3.session is mock_client.return_value
close_function = MagicMock()
ss.s3.close = close_function
ss.close()
close_function.assert_called_once()
assert not ss.s3
def test_collect_values_with_constructor_argument():
os_envs = {
'CODEPACK_WORKER_LOGGER': 'dummy-logger',
'CODEPACK_WORKER_DUMMY': 'dummy_value'
}
try:
for k, v in os_envs.items():
os.environ[k] = v
config = Config(config_path='config/test.ini')
ret = config.get_config('worker')
assert ret == {
'group_id': 'codepack_worker_test',
'interval': '5',
'logger': 'worker-logger',
'script_path': 'scripts/run_snapshot.py',
'source': 'kafka',
'supervisor': 'http://localhost:8000',
'topic': 'test'
}
finally:
for k in os_envs.keys():
os.environ.pop(k, None)
def test_s3_storage_load(mock_client, dummy_deliveries):
config = Config()
s3_config = config.get_config('s3')
ss = S3Storage(s3=s3_config,
item_type=Delivery,
key='id',
bucket='test_bucket',
path='test_path')
mock_client.return_value.exceptions.NoSuchKey = Exception
mock_client.return_value.get_object.side_effect = partial(
fake_get_object, dummy_deliveries)
ret = ss.load(key='obj2')
assert isinstance(ret, Delivery)
assert ret.to_dict() == dummy_deliveries[1].to_dict()
ret = ss.load(key='obj1', to_dict=True)
assert isinstance(ret, dict)
assert ret == dummy_deliveries[0].to_dict()
ret = ss.load(key='obj3', projection=['serial_number'])
assert isinstance(ret, dict)
assert set(ret.keys()) == {'id', 'serial_number'}
ret = ss.load(key=['obj2', 'obj4', 'obj3'])
assert len(ret) == 2
assert {x.id for x in ret} == {'obj2', 'obj3'}
def test_get_config_without_anything_but_method_argument(parse_config):
config = Config()
ret = config.get_config('worker', config_path='config/test.ini')
assert ret is not None
parse_config.assert_called_once_with(section='worker',
config_path='config/test.ini')