def test_key_with_value_float_returns_key_and_value(self):
parameter_input = {self.random_key: self.random_float}
flattened_parameter_input = flatten_parameter_dictionary(
parameter_input)
self.assertEqual({self.random_key: self.random_float},
flattened_parameter_input)
def test_key_and_value_returns_key_and_value(self):
parameter_input = {self.random_key: self.random_string_literal}
flattened_parameter_input = flatten_parameter_dictionary(
parameter_input)
self.assertEqual({self.random_key: self.random_string_literal},
flattened_parameter_input)
def _assert_flattened_parameter_names_for_job_in_job_input_parameters(
self, job_id, expected_loaded_parameters):
from foundations.job_parameters import flatten_parameter_dictionary
from foundations_contrib.global_state import redis_connection
from foundations_internal.foundations_serializer import loads
flattened_parameters = flatten_parameter_dictionary(
expected_loaded_parameters)
flattened_parameters_data = []
for parameter_name in flattened_parameters.keys():
flattened_parameters_data.append({
'argument': {
'name': parameter_name,
'value': {
'type': 'dynamic',
'name': parameter_name
}
},
'stage_uuid': 'stageless'
})
logged_parameters = redis_connection.get(
'jobs:{}:input_parameters'.format(job_id))
self.assertEqual(flattened_parameters_data, loads(logged_parameters))
def test_logs_nested_params_from_file_after_flattening(self):
from foundations.job_parameters import flatten_parameter_dictionary
log_params(self.mock_nested_parameters)
for param_key, param_value in flatten_parameter_dictionary(
self.mock_nested_parameters).items():
self.mock_log_param.assert_any_call(param_key, param_value)
def test_value_is_list_of_lists_flattens_correctly(self):
parameter_input = {self.random_key: [[0, 1], [2, 3]]}
expected_output = {
self.random_key + "_0_0": 0,
self.random_key + "_0_1": 1,
self.random_key + "_1_0": 2,
self.random_key + "_1_1": 3,
}
self.assertEqual(expected_output,
flatten_parameter_dictionary(parameter_input))
def _assert_flattened_parameter_values_for_job_in_job_parameters(
self, job_id, expected_loaded_parameters):
from foundations.job_parameters import flatten_parameter_dictionary
from foundations_contrib.global_state import redis_connection
import json
flattened_parameters = flatten_parameter_dictionary(
expected_loaded_parameters)
logged_parameters = redis_connection.get(
'jobs:{}:parameters'.format(job_id))
self.assertEqual(flattened_parameters, json.loads(logged_parameters))
def _assert_flattened_parameter_keys_in_project_input_parameter_names_set(
self, project_name, expected_loaded_parameters):
from foundations.job_parameters import flatten_parameter_dictionary
flattened_parameters = flatten_parameter_dictionary(
expected_loaded_parameters)
parameter_names = set(
map(lambda param_key: bytes(param_key, 'ascii'),
flattened_parameters))
logged_parameter_names = self.redis_connection.smembers(
'projects:{}:input_parameter_names'.format(project_name))
self.assertEqual(parameter_names, logged_parameter_names)
def test_logs_nested_params_from_file_after_flattening(self):
import json
from foundations.job_parameters import flatten_parameter_dictionary
self.mock_file.read.return_value = json.dumps(
self.mock_nested_parameters)
load_parameters()
for param_key, param_value in flatten_parameter_dictionary(
self.mock_nested_parameters).items():
self.mock_log_param.assert_any_call(param_key, param_value)
def test_key_with_value_of_dict_of_literals_returns_key_concatenated_with_nested_dict_keys(
self, ):
parameter_input = {self.random_key: self.random_literal_dict_value}
flattened_parameter_input = flatten_parameter_dictionary(
parameter_input)
expected_output = {
f"{self.random_key}_{nested_key}": nested_value
for nested_key, nested_value in
self.random_literal_dict_value.items()
}
self.assertEqual(expected_output, flattened_parameter_input)
def test_multiple_keys_with_at_most_singly_nested_values(self):
none_literal_key = self.faker.word()
int_literal_key = self.faker.word()
float_literal_key = self.faker.word()
string_literal_key = self.faker.word()
non_empty_dictionary_key = self.faker.word()
non_empty_list_key = self.faker.word()
empty_dictionary_key = self.faker.word()
empty_list_key = self.faker.word()
parameter_input = {
none_literal_key: None,
int_literal_key: self.random_int,
float_literal_key: self.random_float,
string_literal_key: self.random_string_literal,
empty_dictionary_key: {},
empty_list_key: [],
non_empty_list_key: self.random_literal_list_value,
non_empty_dictionary_key: self.random_literal_dict_value
}
expected_output = {
none_literal_key: None,
int_literal_key: self.random_int,
float_literal_key: self.random_float,
string_literal_key: self.random_string_literal,
empty_dictionary_key: None,
empty_list_key: None
}
list_of_keys = map(
lambda list_index: '{}_{}'.format(non_empty_list_key, list_index),
range(self.random_length))
list_output = {
key: value
for key, value in zip(list_of_keys, self.random_literal_list_value)
}
expected_output.update(list_output)
expected_dict_output = {
'{}_{}'.format(non_empty_dictionary_key, nested_key): nested_value
for nested_key, nested_value in
self.random_literal_dict_value.items()
}
expected_output.update(expected_dict_output)
self.assertEqual(expected_output,
flatten_parameter_dictionary(parameter_input))
def test_key_with_value_of_list_of_literals_returns_key_concatenated_with_list_index(
self):
parameter_input = {self.random_key: self.random_literal_list_value}
flattened_parameter_input = flatten_parameter_dictionary(
parameter_input)
list_of_keys = map(
lambda list_index: '{}_{}'.format(self.random_key, list_index),
range(self.random_length))
expected_output = {
key: value
for key, value in zip(list_of_keys, self.random_literal_list_value)
}
self.assertEqual(expected_output, flattened_parameter_input)
def _assert_flattened_parameter_values_for_job_in_job_parameters(
self, job_id, expected_loaded_parameters):
from foundations.job_parameters import flatten_parameter_dictionary
import json
flattened_parameters = flatten_parameter_dictionary(
expected_loaded_parameters)
parameters_in_redis = self.redis_connection.get(
'jobs:{}:parameters'.format(job_id))
if parameters_in_redis is None:
logged_parameters = {}
else:
logged_parameters = json.loads(parameters_in_redis)
self.assertEqual(flattened_parameters, logged_parameters)
def test_value_is_dict_of_dicts_flattens_correctly(self):
parameter_input = {
self.random_key: {
'key_zero': {
'hello': 'there'
},
'key_one': {
'bye': 'bye'
}
}
}
expected_output = {
self.random_key + '_key_zero_hello': 'there',
self.random_key + '_key_one_bye': 'bye'
}
self.assertEqual(expected_output,
flatten_parameter_dictionary(parameter_input))
def test_value_is_dict_of_dicts_flattens_correctly(self):
parameter_input = {
self.random_key: {
"key_zero": {
"hello": "there"
},
"key_one": {
"bye": "bye"
}
}
}
expected_output = {
self.random_key + "_key_zero_hello": "there",
self.random_key + "_key_one_bye": "bye",
}
self.assertEqual(expected_output,
flatten_parameter_dictionary(parameter_input))
def test_multiple_keys_with_at_most_singly_nested_values(self):
parameter_input = {
"none_literal_key": None,
"int_literal_key": self.random_int,
"float_literal_key": self.random_float,
"string_literal_key": self.random_string_literal,
"empty_dictionary_key": {},
"empty_list_key": [],
"non_empty_list_key": self.random_literal_list_value,
"non_empty_dictionary_key": self.random_literal_dict_value,
}
expected_output = {
"none_literal_key": None,
"int_literal_key": self.random_int,
"float_literal_key": self.random_float,
"string_literal_key": self.random_string_literal,
"empty_dictionary_key": None,
"empty_list_key": None,
}
list_of_keys = map(
lambda list_index: f"non_empty_list_key_{list_index}",
range(self.random_length),
)
list_output = {
key: value
for key, value in zip(list_of_keys, self.random_literal_list_value)
}
expected_output.update(list_output)
expected_dict_output = {
f"non_empty_dictionary_key_{nested_key}": nested_value
for nested_key, nested_value in
self.random_literal_dict_value.items()
}
expected_output.update(expected_dict_output)
self.assertEqual(expected_output,
flatten_parameter_dictionary(parameter_input))
def test_key_with_value_empty_dict_turns_value_dict_into_none(self):
parameter_input = {self.random_key: {}}
flattened_parameter_input = flatten_parameter_dictionary(
parameter_input)
self.assertEqual({self.random_key: None}, flattened_parameter_input)
def test_flatten_empty_dictionary_returns_empty_dictionary(self):
parameter_input = {}
flattened_parameter_input = flatten_parameter_dictionary(
parameter_input)
self.assertEqual({}, flattened_parameter_input)
