def test_fast_deepcopy_success(self): values = [ 'a', u'٩(̾●̮̮̃̾•̃̾)۶', 1, [1, 2, '3', 'b'], { 'a': 1, 'b': '3333', 'c': 'd' }, ] expected_values = [ 'a', u'٩(̾●̮̮̃̾•̃̾)۶', 1, [1, 2, '3', 'b'], { 'a': 1, 'b': '3333', 'c': 'd' }, ] for value, expected_value in zip(values, expected_values): result = fast_deepcopy(value) self.assertEqual(result, value) self.assertEqual(result, expected_value)
def test_fast_deepcopy_success(self): values = [ "a", "٩(̾●̮̮̃̾•̃̾)۶", 1, [1, 2, "3", "b"], { "a": 1, "b": "3333", "c": "d" }, ] expected_values = [ "a", "٩(̾●̮̮̃̾•̃̾)۶", 1, [1, 2, "3", "b"], { "a": 1, "b": "3333", "c": "d" }, ] for value, expected_value in zip(values, expected_values): result = fast_deepcopy(value) self.assertEqual(result, value) self.assertEqual(result, expected_value)
def unescape_chars(field): if not isinstance(field, dict) and not isinstance(field, list): return field value = fast_deepcopy(field) return _translate_chars(value, UNESCAPE_TRANSLATION)
def get_params_view(action_db=None, runner_db=None, merged_only=False): if runner_db: runner_params = fast_deepcopy( getattr(runner_db, 'runner_parameters', {})) or {} else: runner_params = {} if action_db: action_params = fast_deepcopy(getattr(action_db, 'parameters', {})) or {} else: action_params = {} parameters = set(runner_params.keys()).union(set(action_params.keys())) merged_params = {} for param in parameters: merged_params[param] = _merge_param_meta_values( action_meta=action_params.get(param), runner_meta=runner_params.get(param)) if merged_only: return merged_params def is_required(param_meta): return param_meta.get('required', False) def is_immutable(param_meta): return param_meta.get('immutable', False) immutable = { param for param in parameters if is_immutable(merged_params.get(param)) } required = { param for param in parameters if is_required(merged_params.get(param)) } required = required - immutable optional = parameters - required - immutable required_params = {k: merged_params[k] for k in required} optional_params = {k: merged_params[k] for k in optional} immutable_params = {k: merged_params[k] for k in immutable} return (required_params, optional_params, immutable_params)
def unescape_chars(field): if not isinstance(field, dict): return field value = fast_deepcopy(field) translated = _translate_chars(value, UNESCAPE_TRANSLATION) translated = _translate_chars(value, RULE_CRITERIA_UNESCAPE_TRANSLATION) return translated
def unescape_chars(field): if not isinstance(field, dict): return field value = fast_deepcopy(field) translated = _translate_chars(value, UNESCAPE_TRANSLATION) translated = _translate_chars(value, RULE_CRITERIA_UNESCAPE_TRANSLATION) return translated
def mask_secret_parameters(parameters, secret_parameters, result=None): """ Introspect the parameters dict and return a new dict with masked secret parameters. :param parameters: Parameters to process. :type parameters: ``dict`` or ``list`` or ``string`` :param secret_parameters: Dict of parameter names which are secret. The type must be the same type as ``parameters`` (or at least behave in the same way), so that they can be traversed in the same way as recurse down into the structure. :type secret_parameters: ``dict`` :param result: Deep copy of parameters so that parameters is not modified in place. Default = None, meaning this function will make a deep copy before starting. :type result: ``dict`` or ``list`` or ``string`` """ # how we iterate depends on what data type was passed in iterator = None is_dict = isinstance(secret_parameters, dict) is_list = isinstance(secret_parameters, list) if is_dict: iterator = six.iteritems(secret_parameters) elif is_list: iterator = enumerate(secret_parameters) else: return MASKED_ATTRIBUTE_VALUE # only create a deep copy of parameters on the first call # all other recursive calls pass back referneces to this result object # so we can reuse it, saving memory and CPU cycles if result is None: result = fast_deepcopy(parameters) # iterate over the secret parameters for secret_param, secret_sub_params in iterator: if is_dict: if secret_param in result: result[secret_param] = mask_secret_parameters( parameters[secret_param], secret_sub_params, result=result[secret_param], ) elif is_list: # we're assuming lists contain the same data type for every element for idx, value in enumerate(result): result[idx] = mask_secret_parameters( parameters[idx], secret_sub_params, result=result[idx] ) else: result[secret_param] = MASKED_ATTRIBUTE_VALUE return result
def mask_secret_parameters(parameters, secret_parameters, result=None): """ Introspect the parameters dict and return a new dict with masked secret parameters. :param parameters: Parameters to process. :type parameters: ``dict`` or ``list`` or ``string`` :param secret_parameters: Dict of parameter names which are secret. The type must be the same type as ``parameters`` (or at least behave in the same way), so that they can be traversed in the same way as recurse down into the structure. :type secret_parameters: ``dict`` :param result: Deep copy of parameters so that parameters is not modified in place. Default = None, meaning this function will make a deep copy before starting. :type result: ``dict`` or ``list`` or ``string`` """ # how we iterate depends on what data type was passed in iterator = None is_dict = isinstance(secret_parameters, dict) is_list = isinstance(secret_parameters, list) if is_dict: iterator = six.iteritems(secret_parameters) elif is_list: iterator = enumerate(secret_parameters) else: return MASKED_ATTRIBUTE_VALUE # only create a deep copy of parameters on the first call # all other recursive calls pass back referneces to this result object # so we can reuse it, saving memory and CPU cycles if result is None: result = fast_deepcopy(parameters) # iterate over the secret parameters for secret_param, secret_sub_params in iterator: if is_dict: if secret_param in result: result[secret_param] = mask_secret_parameters(parameters[secret_param], secret_sub_params, result=result[secret_param]) elif is_list: # we're assuming lists contain the same data type for every element for idx, value in enumerate(result): result[idx] = mask_secret_parameters(parameters[idx], secret_sub_params, result=result[idx]) else: result[secret_param] = MASKED_ATTRIBUTE_VALUE return result
def get_params_view(action_db=None, runner_db=None, merged_only=False): if runner_db: runner_params = fast_deepcopy(getattr(runner_db, 'runner_parameters', {})) or {} else: runner_params = {} if action_db: action_params = fast_deepcopy(getattr(action_db, 'parameters', {})) or {} else: action_params = {} parameters = set(runner_params.keys()).union(set(action_params.keys())) merged_params = {} for param in parameters: merged_params[param] = _merge_param_meta_values(action_meta=action_params.get(param), runner_meta=runner_params.get(param)) if merged_only: return merged_params def is_required(param_meta): return param_meta.get('required', False) def is_immutable(param_meta): return param_meta.get('immutable', False) immutable = {param for param in parameters if is_immutable(merged_params.get(param))} required = {param for param in parameters if is_required(merged_params.get(param))} required = required - immutable optional = parameters - required - immutable required_params = {k: merged_params[k] for k in required} optional_params = {k: merged_params[k] for k in optional} immutable_params = {k: merged_params[k] for k in immutable} return (required_params, optional_params, immutable_params)
def mask_inquiry_response(response, schema): """ Introspect an Inquiry's response dict and return a new dict with masked secret values. :param response: Inquiry response to process. :type response: ``dict`` :param schema: Inquiry response schema :type schema: ``dict`` """ result = fast_deepcopy(response) for prop_name, prop_attrs in schema['properties'].items(): if prop_attrs.get('secret') is True: if prop_name in response: result[prop_name] = MASKED_ATTRIBUTE_VALUE return result
def mask_inquiry_response(response, schema): """ Introspect an Inquiry's response dict and return a new dict with masked secret values. :param response: Inquiry response to process. :type response: ``dict`` :param schema: Inquiry response schema :type schema: ``dict`` """ result = fast_deepcopy(response) for prop_name, prop_attrs in schema['properties'].items(): if prop_attrs.get('secret') is True: if prop_name in response: result[prop_name] = MASKED_ATTRIBUTE_VALUE return result
def test_fast_deepcopy_success(self): values = [ 'a', u'٩(̾●̮̮̃̾•̃̾)۶', 1, [1, 2, '3', 'b'], {'a': 1, 'b': '3333', 'c': 'd'}, ] expected_values = [ 'a', u'٩(̾●̮̮̃̾•̃̾)۶', 1, [1, 2, '3', 'b'], {'a': 1, 'b': '3333', 'c': 'd'}, ] for value, expected_value in zip(values, expected_values): result = fast_deepcopy(value) self.assertEqual(result, value) self.assertEqual(result, expected_value)
def restore_vars(self, ctx_vars): self.vars.update(fast_deepcopy(ctx_vars))
def restore_vars(self, ctx_vars): self.vars.update(fast_deepcopy(ctx_vars))
def test_fast_deepcopy_is_faster_than_copy_deepcopy(self): def random_string(N): return ''.join( random.choice(string.ascii_uppercase + string.digits) for _ in range(N)) def random_dict(width=5, height=5, levels=2): dct = {} lst = [random_string(4) for i in range(width)] lst2 = [random.randint(0, 10000) for i in range(width)] lst3 = [bool(random.randint(0, 1)) for i in range(width)] for j in range(height): dct[str(j)] = lst dct[str(width + j)] = lst2 dct[str(2 * width + j)] = lst3 for i in range(levels): new_dct = {} for j in range(height): new_dct[str(j)] = dct dct = json.loads(json.dumps(new_dct)) return new_dct error_msg = 'fast_deepcopy is not faster than copy.deepcopy' # 1. Smaller dict data = random_dict(width=10, levels=2, height=2) # fast_deepcopy start = timer() fast_deepcopy(data) end = timer() duration_1 = (end - start) # copy.deepcopy start = timer() copy.deepcopy(data) end = timer() duration_2 = (end - start) self.assertTrue(duration_1 < duration_2, error_msg) # 2. Medium sized dict data = random_dict(width=20, levels=3, height=4) # fast_deepcopy start = timer() fast_deepcopy(data) end = timer() duration_1 = (end - start) # copy.deepcopy start = timer() copy.deepcopy(data) end = timer() duration_2 = (end - start) self.assertTrue(duration_1 < duration_2, error_msg) # 3. Larger dict data = random_dict(width=30, levels=5, height=4) # fast_deepcopy start = timer() fast_deepcopy(data) end = timer() duration_1 = (end - start) # copy.deepcopy start = timer() copy.deepcopy(data) end = timer() duration_2 = (end - start) self.assertTrue(duration_1 < duration_2, error_msg)
def _construct_context(self, wf_ex): ctx = ujson.fast_deepcopy(self.context) ctx['workflow_execution'] = str(wf_ex.id) return ctx
def _construct_context(self, wf_ex): ctx = ujson.fast_deepcopy(self.context) ctx['workflow_execution'] = str(wf_ex.id) return ctx
def escape_chars(field): if not isinstance(field, dict): return field value = fast_deepcopy(field) return _translate_chars(value, ESCAPE_TRANSLATION)