def test_fast_deepcopy_success(self):
values = [
'a',
u'٩(̾●̮̮̃̾•̃̾)۶',
1,
[1, 2, '3', 'b'],
{
'a': 1,
'b': '3333',
'c': 'd'
},
]
expected_values = [
'a',
u'٩(̾●̮̮̃̾•̃̾)۶',
1,
[1, 2, '3', 'b'],
{
'a': 1,
'b': '3333',
'c': 'd'
},
]
for value, expected_value in zip(values, expected_values):
result = fast_deepcopy(value)
self.assertEqual(result, value)
self.assertEqual(result, expected_value)
def test_fast_deepcopy_success(self):
values = [
"a",
"٩(̾●̮̮̃̾•̃̾)۶",
1,
[1, 2, "3", "b"],
{
"a": 1,
"b": "3333",
"c": "d"
},
]
expected_values = [
"a",
"٩(̾●̮̮̃̾•̃̾)۶",
1,
[1, 2, "3", "b"],
{
"a": 1,
"b": "3333",
"c": "d"
},
]
for value, expected_value in zip(values, expected_values):
result = fast_deepcopy(value)
self.assertEqual(result, value)
self.assertEqual(result, expected_value)
def unescape_chars(field):
if not isinstance(field, dict) and not isinstance(field, list):
return field
value = fast_deepcopy(field)
return _translate_chars(value, UNESCAPE_TRANSLATION)
def get_params_view(action_db=None, runner_db=None, merged_only=False):
if runner_db:
runner_params = fast_deepcopy(
getattr(runner_db, 'runner_parameters', {})) or {}
else:
runner_params = {}
if action_db:
action_params = fast_deepcopy(getattr(action_db, 'parameters',
{})) or {}
else:
action_params = {}
parameters = set(runner_params.keys()).union(set(action_params.keys()))
merged_params = {}
for param in parameters:
merged_params[param] = _merge_param_meta_values(
action_meta=action_params.get(param),
runner_meta=runner_params.get(param))
if merged_only:
return merged_params
def is_required(param_meta):
return param_meta.get('required', False)
def is_immutable(param_meta):
return param_meta.get('immutable', False)
immutable = {
param
for param in parameters if is_immutable(merged_params.get(param))
}
required = {
param
for param in parameters if is_required(merged_params.get(param))
}
required = required - immutable
optional = parameters - required - immutable
required_params = {k: merged_params[k] for k in required}
optional_params = {k: merged_params[k] for k in optional}
immutable_params = {k: merged_params[k] for k in immutable}
return (required_params, optional_params, immutable_params)
def unescape_chars(field):
if not isinstance(field, dict):
return field
value = fast_deepcopy(field)
translated = _translate_chars(value, UNESCAPE_TRANSLATION)
translated = _translate_chars(value, RULE_CRITERIA_UNESCAPE_TRANSLATION)
return translated
def unescape_chars(field):
if not isinstance(field, dict):
return field
value = fast_deepcopy(field)
translated = _translate_chars(value, UNESCAPE_TRANSLATION)
translated = _translate_chars(value, RULE_CRITERIA_UNESCAPE_TRANSLATION)
return translated
def mask_secret_parameters(parameters, secret_parameters, result=None):
"""
Introspect the parameters dict and return a new dict with masked secret
parameters.
:param parameters: Parameters to process.
:type parameters: ``dict`` or ``list`` or ``string``
:param secret_parameters: Dict of parameter names which are secret.
The type must be the same type as ``parameters``
(or at least behave in the same way),
so that they can be traversed in the same way as
recurse down into the structure.
:type secret_parameters: ``dict``
:param result: Deep copy of parameters so that parameters is not modified
in place. Default = None, meaning this function will make a
deep copy before starting.
:type result: ``dict`` or ``list`` or ``string``
"""
# how we iterate depends on what data type was passed in
iterator = None
is_dict = isinstance(secret_parameters, dict)
is_list = isinstance(secret_parameters, list)
if is_dict:
iterator = six.iteritems(secret_parameters)
elif is_list:
iterator = enumerate(secret_parameters)
else:
return MASKED_ATTRIBUTE_VALUE
# only create a deep copy of parameters on the first call
# all other recursive calls pass back referneces to this result object
# so we can reuse it, saving memory and CPU cycles
if result is None:
result = fast_deepcopy(parameters)
# iterate over the secret parameters
for secret_param, secret_sub_params in iterator:
if is_dict:
if secret_param in result:
result[secret_param] = mask_secret_parameters(
parameters[secret_param],
secret_sub_params,
result=result[secret_param],
)
elif is_list:
# we're assuming lists contain the same data type for every element
for idx, value in enumerate(result):
result[idx] = mask_secret_parameters(
parameters[idx], secret_sub_params, result=result[idx]
)
else:
result[secret_param] = MASKED_ATTRIBUTE_VALUE
return result
def mask_secret_parameters(parameters, secret_parameters, result=None):
"""
Introspect the parameters dict and return a new dict with masked secret
parameters.
:param parameters: Parameters to process.
:type parameters: ``dict`` or ``list`` or ``string``
:param secret_parameters: Dict of parameter names which are secret.
The type must be the same type as ``parameters``
(or at least behave in the same way),
so that they can be traversed in the same way as
recurse down into the structure.
:type secret_parameters: ``dict``
:param result: Deep copy of parameters so that parameters is not modified
in place. Default = None, meaning this function will make a
deep copy before starting.
:type result: ``dict`` or ``list`` or ``string``
"""
# how we iterate depends on what data type was passed in
iterator = None
is_dict = isinstance(secret_parameters, dict)
is_list = isinstance(secret_parameters, list)
if is_dict:
iterator = six.iteritems(secret_parameters)
elif is_list:
iterator = enumerate(secret_parameters)
else:
return MASKED_ATTRIBUTE_VALUE
# only create a deep copy of parameters on the first call
# all other recursive calls pass back referneces to this result object
# so we can reuse it, saving memory and CPU cycles
if result is None:
result = fast_deepcopy(parameters)
# iterate over the secret parameters
for secret_param, secret_sub_params in iterator:
if is_dict:
if secret_param in result:
result[secret_param] = mask_secret_parameters(parameters[secret_param],
secret_sub_params,
result=result[secret_param])
elif is_list:
# we're assuming lists contain the same data type for every element
for idx, value in enumerate(result):
result[idx] = mask_secret_parameters(parameters[idx],
secret_sub_params,
result=result[idx])
else:
result[secret_param] = MASKED_ATTRIBUTE_VALUE
return result
def get_params_view(action_db=None, runner_db=None, merged_only=False):
if runner_db:
runner_params = fast_deepcopy(getattr(runner_db, 'runner_parameters', {})) or {}
else:
runner_params = {}
if action_db:
action_params = fast_deepcopy(getattr(action_db, 'parameters', {})) or {}
else:
action_params = {}
parameters = set(runner_params.keys()).union(set(action_params.keys()))
merged_params = {}
for param in parameters:
merged_params[param] = _merge_param_meta_values(action_meta=action_params.get(param),
runner_meta=runner_params.get(param))
if merged_only:
return merged_params
def is_required(param_meta):
return param_meta.get('required', False)
def is_immutable(param_meta):
return param_meta.get('immutable', False)
immutable = {param for param in parameters if is_immutable(merged_params.get(param))}
required = {param for param in parameters if is_required(merged_params.get(param))}
required = required - immutable
optional = parameters - required - immutable
required_params = {k: merged_params[k] for k in required}
optional_params = {k: merged_params[k] for k in optional}
immutable_params = {k: merged_params[k] for k in immutable}
return (required_params, optional_params, immutable_params)
def mask_inquiry_response(response, schema):
"""
Introspect an Inquiry's response dict and return a new dict with masked secret
values.
:param response: Inquiry response to process.
:type response: ``dict``
:param schema: Inquiry response schema
:type schema: ``dict``
"""
result = fast_deepcopy(response)
for prop_name, prop_attrs in schema['properties'].items():
if prop_attrs.get('secret') is True:
if prop_name in response:
result[prop_name] = MASKED_ATTRIBUTE_VALUE
return result
def mask_inquiry_response(response, schema):
"""
Introspect an Inquiry's response dict and return a new dict with masked secret
values.
:param response: Inquiry response to process.
:type response: ``dict``
:param schema: Inquiry response schema
:type schema: ``dict``
"""
result = fast_deepcopy(response)
for prop_name, prop_attrs in schema['properties'].items():
if prop_attrs.get('secret') is True:
if prop_name in response:
result[prop_name] = MASKED_ATTRIBUTE_VALUE
return result
def test_fast_deepcopy_success(self):
values = [
'a',
u'٩(̾●̮̮̃̾•̃̾)۶',
1,
[1, 2, '3', 'b'],
{'a': 1, 'b': '3333', 'c': 'd'},
]
expected_values = [
'a',
u'٩(̾●̮̮̃̾•̃̾)۶',
1,
[1, 2, '3', 'b'],
{'a': 1, 'b': '3333', 'c': 'd'},
]
for value, expected_value in zip(values, expected_values):
result = fast_deepcopy(value)
self.assertEqual(result, value)
self.assertEqual(result, expected_value)
def restore_vars(self, ctx_vars):
self.vars.update(fast_deepcopy(ctx_vars))
def restore_vars(self, ctx_vars):
self.vars.update(fast_deepcopy(ctx_vars))
def test_fast_deepcopy_is_faster_than_copy_deepcopy(self):
def random_string(N):
return ''.join(
random.choice(string.ascii_uppercase + string.digits)
for _ in range(N))
def random_dict(width=5, height=5, levels=2):
dct = {}
lst = [random_string(4) for i in range(width)]
lst2 = [random.randint(0, 10000) for i in range(width)]
lst3 = [bool(random.randint(0, 1)) for i in range(width)]
for j in range(height):
dct[str(j)] = lst
dct[str(width + j)] = lst2
dct[str(2 * width + j)] = lst3
for i in range(levels):
new_dct = {}
for j in range(height):
new_dct[str(j)] = dct
dct = json.loads(json.dumps(new_dct))
return new_dct
error_msg = 'fast_deepcopy is not faster than copy.deepcopy'
# 1. Smaller dict
data = random_dict(width=10, levels=2, height=2)
# fast_deepcopy
start = timer()
fast_deepcopy(data)
end = timer()
duration_1 = (end - start)
# copy.deepcopy
start = timer()
copy.deepcopy(data)
end = timer()
duration_2 = (end - start)
self.assertTrue(duration_1 < duration_2, error_msg)
# 2. Medium sized dict
data = random_dict(width=20, levels=3, height=4)
# fast_deepcopy
start = timer()
fast_deepcopy(data)
end = timer()
duration_1 = (end - start)
# copy.deepcopy
start = timer()
copy.deepcopy(data)
end = timer()
duration_2 = (end - start)
self.assertTrue(duration_1 < duration_2, error_msg)
# 3. Larger dict
data = random_dict(width=30, levels=5, height=4)
# fast_deepcopy
start = timer()
fast_deepcopy(data)
end = timer()
duration_1 = (end - start)
# copy.deepcopy
start = timer()
copy.deepcopy(data)
end = timer()
duration_2 = (end - start)
self.assertTrue(duration_1 < duration_2, error_msg)
def _construct_context(self, wf_ex):
ctx = ujson.fast_deepcopy(self.context)
ctx['workflow_execution'] = str(wf_ex.id)
return ctx
def _construct_context(self, wf_ex):
ctx = ujson.fast_deepcopy(self.context)
ctx['workflow_execution'] = str(wf_ex.id)
return ctx
def escape_chars(field):
if not isinstance(field, dict):
return field
value = fast_deepcopy(field)
return _translate_chars(value, ESCAPE_TRANSLATION)
