def _transform_input(self, transform_type, value, spec_param): """ Transforms an input according to the rules given in NarrativeMethodStore.ServiceMethodInputMapping Really, there are three types of transforms possible: 1. ref - turns the input string into a workspace ref. 2. int - tries to coerce the input string into an int. 3. list<type> - turns the given list into a list of the given type. (4.) none or None - doesn't transform. Returns a transformed (or not) value. """ return transform_param_value(transform_type, value, spec_param)
def _map_inputs(self, input_mapping, params, spec_params): """ Maps the dictionary of parameters and inputs based on rules provided in the input_mapping. This iterates over the list of input_mappings, and uses them as a filter to apply to each parameter. Returns a list of inputs that can be passed directly to NJSW.run_job input_mapping is a list of dicts, as defined by NarrativeMethodStore.ServiceMethodInputMapping. params is a dict of key-value-pairs, each key is the input_parameter field of some parameter. """ inputs_dict = dict() for p in input_mapping: # 2 steps - figure out the proper value, then figure out the # proper position. value first! p_value = None input_param_id = None if 'input_parameter' in p: input_param_id = p['input_parameter'] p_value = params.get(input_param_id, None) if spec_params[input_param_id].get('type', '') == 'group': p_value = self._map_group_inputs(p_value, spec_params[input_param_id], spec_params) # turn empty strings into None if isinstance(p_value, basestring) and len(p_value) == 0: p_value = None elif 'narrative_system_variable' in p: p_value = system_variable(p['narrative_system_variable']) if 'constant_value' in p and p_value is None: p_value = p['constant_value'] if 'generated_value' in p and p_value is None: p_value = self._generate_input(p['generated_value']) spec_param = None if input_param_id: spec_param = spec_params[input_param_id] p_value = transform_param_value(p.get('target_type_transform'), p_value, spec_param) # get position! arg_position = p.get('target_argument_position', 0) target_prop = p.get('target_property', None) if target_prop is not None: final_input = inputs_dict.get(arg_position, dict()) if '/' in target_prop: # This is case when slashes in target_prop separeate # elements in nested maps. We ignore escaped slashes # (separate backslashes should be escaped as well). bck_slash = u"\u244A" fwd_slash = u"\u20EB" temp_string = target_prop.replace("\\\\", bck_slash) temp_string = temp_string.replace("\\/", fwd_slash) temp_path = [] for part in temp_string.split("/"): part = part.replace(bck_slash, "\\") part = part.replace(fwd_slash, "/") temp_path.append(part.encode('ascii', 'ignore')) temp_map = final_input temp_key = None # We're going along the path and creating intermediate # dictionaries. for temp_path_item in temp_path: if temp_key: if temp_key not in temp_map: temp_map[temp_key] = {} temp_map = temp_map[temp_key] temp_key = temp_path_item # temp_map points to deepest nested map now, temp_key is # the last item in the path temp_map[temp_key] = p_value else: final_input[target_prop] = p_value inputs_dict[arg_position] = final_input else: inputs_dict[arg_position] = p_value inputs_list = list() keys = sorted(inputs_dict.keys()) for k in keys: inputs_list.append(inputs_dict[k]) return inputs_list
def test_transform_input_bad(self): with self.assertRaises(ValueError): app_util.transform_param_value('foo', 'bar', None)
def test_transform_input_good(self): ws_name = self.public_ws os.environ['KB_WORKSPACE_ID'] = ws_name test_data = [{ 'value': 'input_value', 'type': 'ref', 'expected': ws_name + '/' + 'input_value' }, { 'value': ws_name + "/input_value", 'type': 'ref', 'expected': ws_name + '/' + 'input_value' }, { 'value': 'input_value', 'type': 'unresolved-ref', 'expected': ws_name + '/' + 'input_value' }, { 'value': 'rhodobacterium.art.q20.int.PE.reads', 'type': 'resolved-ref', 'expected': '11635/9/1' }, { 'value': ws_name + "/rhodobacterium.art.q20.int.PE.reads", 'type': 'resolved-ref', 'expected': '11635/9/1' }, { 'value': None, 'type': 'int', 'expected': None }, { 'value': '5', 'type': 'int', 'expected': 5 }, { 'value': ['a', 'b', 'c'], 'type': 'list<ref>', 'expected': [ws_name + '/a', ws_name + '/b', ws_name + '/c'] }, { 'value': [ 'rhodobacterium.art.q20.int.PE.reads', 'rhodobacterium.art.q10.PE.reads' ], 'type': 'list<resolved-ref>', 'expected': ['11635/9/1', '11635/10/1'] }, { 'value': 'foo', 'type': 'list<ref>', 'expected': [ws_name + '/foo'] }, { 'value': ['1', '2', 3], 'type': 'list<int>', 'expected': [1, 2, 3] }, { 'value': 'bar', 'type': None, 'expected': 'bar' }, { 'value': 'rhodobacterium.art.q20.int.PE.reads', 'type': 'future-default', 'spec': { 'is_output': 0, 'allowed_types': ["Some.KnownType"] }, 'expected': '11635/9/1' }, { 'value': [123, 456], 'type': None, 'expected': [123, 456] }, { 'value': 123, 'type': 'string', 'expected': '123' }, { 'value': ['one', 'two'], 'type': None, 'spec': { 'type': 'textsubdata' }, 'expected': "one,two" }, { 'value': ['one', 'two'], 'type': "list<string>", 'spec': { 'type': 'textsubdata' }, 'expected': ['one', 'two'] }, { 'value': { 'one': 1 }, 'type': 'string', 'expected': "one=1" }] for test in test_data: spec = test.get('spec', None) ret = app_util.transform_param_value(test['type'], test['value'], spec) self.assertEqual(ret, test['expected']) del (os.environ['KB_WORKSPACE_ID'])
def test_input_mapping(self): self.maxDiff = None inputs = { "reads_tuple": [{ "input_reads_label": "reads file 1", "input_reads_obj": "rhodobacterium.art.q20.int.PE.reads", "input_reads_metadata": { "key1": "value1" } }, { "input_reads_label": "reads file 2", "input_reads_obj": "rhodobacterium.art.q10.PE.reads", "input_reads_metadata": { "key2": "value2" } }], "output_object": "MyReadsSet", "description": "New Reads Set" } app_id = "NarrativeTest/test_create_set" tag = "dev" ws_name = self.public_ws prev_ws_id = os.environ.get('KB_WORKSPACE_ID', None) os.environ['KB_WORKSPACE_ID'] = ws_name sm = specmanager.SpecManager() spec = sm.get_spec(app_id, tag=tag) spec_params = sm.app_params(spec) spec_params_map = dict((spec_params[i]['id'], spec_params[i]) for i in range(len(spec_params))) mapped_inputs = self.am._map_inputs( spec['behavior']['kb_service_input_mapping'], inputs, spec_params_map) expected = [{ u'output_object_name': 'MyReadsSet', u'data': { u'items': [{ u'label': 'reads file 1', u'metadata': { 'key1': 'value1' }, u'ref': '12345/7/1' }, { u'label': 'reads file 2', u'metadata': { 'key2': 'value2' }, u'ref': '12345/8/1' }], u'description': 'New Reads Set' }, u'workspace': ws_name }] self.assertDictEqual(expected[0], mapped_inputs[0]) ref_path = ws_name + '/MyReadsSet; ' + ws_name + "/rhodobacterium.art.q10.PE.reads" ret = app_util.transform_param_value("resolved-ref", ref_path, None) self.assertEqual(ret, ws_name + '/MyReadsSet;18836/5/1') if prev_ws_id is None: del (os.environ['KB_WORKSPACE_ID']) else: os.environ['KB_WORKSPACE_ID'] = prev_ws_id
def _map_inputs(self, input_mapping, params, spec_params): """ Maps the dictionary of parameters and inputs based on rules provided in the input_mapping. This iterates over the list of input_mappings, and uses them as a filter to apply to each parameter. Returns a list of inputs that can be passed directly to NJSW.run_job input_mapping is a list of dicts, as defined by NarrativeMethodStore.ServiceMethodInputMapping. params is a dict of key-value-pairs, each key is the input_parameter field of some parameter. """ inputs_dict = dict() for p in input_mapping: # 2 steps - figure out the proper value, then figure out the # proper position. value first! p_value = None input_param_id = None if 'input_parameter' in p: input_param_id = p['input_parameter'] p_value = params.get(input_param_id, None) if spec_params[input_param_id].get('type', '') == 'group': p_value = self._map_group_inputs( p_value, spec_params[input_param_id], spec_params) # turn empty strings into None if isinstance(p_value, str) and len(p_value) == 0: p_value = None elif 'narrative_system_variable' in p: p_value = system_variable(p['narrative_system_variable']) if 'constant_value' in p and p_value is None: p_value = p['constant_value'] if 'generated_value' in p and p_value is None: p_value = self._generate_input(p['generated_value']) spec_param = None if input_param_id: spec_param = spec_params[input_param_id] p_value = transform_param_value(p.get('target_type_transform'), p_value, spec_param) # get position! arg_position = p.get('target_argument_position', 0) target_prop = p.get('target_property', None) if target_prop is not None: final_input = inputs_dict.get(arg_position, dict()) if '/' in target_prop: # This is case when slashes in target_prop separate # elements in nested maps. We ignore escaped slashes # (separate backslashes should be escaped as well). bck_slash = "\u244A" fwd_slash = "\u20EB" temp_string = target_prop.replace("\\\\", bck_slash) temp_string = temp_string.replace("\\/", fwd_slash) temp_path = [] for part in temp_string.split("/"): part = part.replace(bck_slash, "\\") part = part.replace(fwd_slash, "/") temp_path.append( part.encode('ascii', 'ignore').decode("ascii")) temp_map = final_input temp_key = None # We're going along the path and creating intermediate # dictionaries. for temp_path_item in temp_path: if temp_key: if temp_key not in temp_map: temp_map[temp_key] = {} temp_map = temp_map[temp_key] temp_key = temp_path_item # temp_map points to deepest nested map now, temp_key is # the last item in the path temp_map[temp_key] = p_value else: final_input[target_prop] = p_value inputs_dict[arg_position] = final_input else: inputs_dict[arg_position] = p_value inputs_list = list() keys = sorted(inputs_dict.keys()) for k in keys: inputs_list.append(inputs_dict[k]) return inputs_list
def test_transform_input_good(self): ws_name = self.public_ws os.environ["KB_WORKSPACE_ID"] = ws_name test_data = [ { "value": "input_value", "type": "ref", "expected": ws_name + "/" + "input_value", }, { "value": ws_name + "/input_value", "type": "ref", "expected": ws_name + "/" + "input_value", }, { "value": "input_value", "type": "unresolved-ref", "expected": ws_name + "/" + "input_value", }, { "value": "rhodobacterium.art.q20.int.PE.reads", "type": "resolved-ref", "expected": "11635/9/1", }, { "value": ws_name + "/rhodobacterium.art.q20.int.PE.reads", "type": "resolved-ref", "expected": "11635/9/1", }, { "value": None, "type": "int", "expected": None }, { "value": "5", "type": "int", "expected": 5 }, { "value": ["a", "b", "c"], "type": "list<ref>", "expected": [ws_name + "/a", ws_name + "/b", ws_name + "/c"], }, { "value": [ "rhodobacterium.art.q20.int.PE.reads", "rhodobacterium.art.q10.PE.reads", ], "type": "list<resolved-ref>", "expected": ["11635/9/1", "11635/10/1"], }, { "value": "foo", "type": "list<ref>", "expected": [ws_name + "/foo"] }, { "value": ["1", "2", 3], "type": "list<int>", "expected": [1, 2, 3] }, { "value": "bar", "type": None, "expected": "bar" }, { "value": "rhodobacterium.art.q20.int.PE.reads", "type": "future-default", "spec": { "is_output": 0, "allowed_types": ["Some.KnownType"] }, "expected": "11635/9/1", }, { "value": [123, 456], "type": None, "expected": [123, 456] }, { "value": 123, "type": "string", "expected": "123" }, { "value": ["one", "two"], "type": None, "spec": { "type": "textsubdata" }, "expected": "one,two", }, { "value": ["one", "two"], "type": "list<string>", "spec": { "type": "textsubdata" }, "expected": ["one", "two"], }, { "value": { "one": 1 }, "type": "string", "expected": "one=1" }, ] for test in test_data: spec = test.get("spec", None) ret = app_util.transform_param_value(test["type"], test["value"], spec) self.assertEqual(ret, test["expected"]) del os.environ["KB_WORKSPACE_ID"]
def test_input_mapping(self): self.maxDiff = None inputs = { "reads_tuple": [ { "input_reads_label": "reads file 1", "input_reads_obj": "rhodobacterium.art.q20.int.PE.reads", "input_reads_metadata": { "key1": "value1" }, }, { "input_reads_label": "reads file 2", "input_reads_obj": "rhodobacterium.art.q10.PE.reads", "input_reads_metadata": { "key2": "value2" }, }, ], "output_object": "MyReadsSet", "description": "New Reads Set", } app_id = "NarrativeTest/test_create_set" tag = "dev" ws_name = self.public_ws prev_ws_id = os.environ.get("KB_WORKSPACE_ID", None) os.environ["KB_WORKSPACE_ID"] = ws_name sm = specmanager.SpecManager() spec = sm.get_spec(app_id, tag=tag) spec_params = sm.app_params(spec) spec_params_map = dict((spec_params[i]["id"], spec_params[i]) for i in range(len(spec_params))) mapped_inputs = self.am._map_inputs( spec["behavior"]["kb_service_input_mapping"], inputs, spec_params_map) expected = [{ "output_object_name": "MyReadsSet", "data": { "items": [ { "label": "reads file 1", "metadata": { "key1": "value1" }, "ref": "12345/7/1", }, { "label": "reads file 2", "metadata": { "key2": "value2" }, "ref": "12345/8/1", }, ], "description": "New Reads Set", }, "workspace": ws_name, }] self.assertDictEqual(expected[0], mapped_inputs[0]) ref_path = (ws_name + "/MyReadsSet; " + ws_name + "/rhodobacterium.art.q10.PE.reads") ret = app_util.transform_param_value("resolved-ref", ref_path, None) self.assertEqual(ret, ws_name + "/MyReadsSet;18836/5/1") if prev_ws_id is None: del os.environ["KB_WORKSPACE_ID"] else: os.environ["KB_WORKSPACE_ID"] = prev_ws_id
def test_transform_input_good(self): ws_name = self.public_ws os.environ['KB_WORKSPACE_ID'] = ws_name test_data = [ { 'value': 'input_value', 'type': 'ref', 'expected': ws_name + '/' + 'input_value' }, { 'value': ws_name + "/input_value", 'type': 'ref', 'expected': ws_name + '/' + 'input_value' }, { 'value': 'input_value', 'type': 'unresolved-ref', 'expected': ws_name + '/' + 'input_value' }, { 'value': 'rhodobacterium.art.q20.int.PE.reads', 'type': 'resolved-ref', 'expected': '11635/9/1' }, { 'value': ws_name + "/rhodobacterium.art.q20.int.PE.reads", 'type': 'resolved-ref', 'expected': '11635/9/1' }, { 'value': None, 'type': 'int', 'expected': None }, { 'value': '5', 'type': 'int', 'expected': 5 }, { 'value': ['a', 'b', 'c'], 'type': 'list<ref>', 'expected': [ws_name + '/a', ws_name + '/b', ws_name + '/c'] }, { 'value': ['rhodobacterium.art.q20.int.PE.reads', 'rhodobacterium.art.q10.PE.reads'], 'type': 'list<resolved-ref>', 'expected': ['11635/9/1', '11635/10/1'] }, { 'value': 'foo', 'type': 'list<ref>', 'expected': [ws_name + '/foo'] }, { 'value': ['1', '2', 3], 'type': 'list<int>', 'expected': [1, 2, 3] }, { 'value': 'bar', 'type': None, 'expected': 'bar' }, { 'value': 'rhodobacterium.art.q20.int.PE.reads', 'type': 'future-default', 'spec': {'is_output': 0, 'allowed_types': ["Some.KnownType"]}, 'expected': '11635/9/1' }, { 'value': [123, 456], 'type': None, 'expected': [123, 456] }, { 'value': 123, 'type': 'string', 'expected': '123' }, { 'value': ['one', 'two'], 'type': None, 'spec': {'type': 'textsubdata'}, 'expected': "one,two" }, { 'value': ['one', 'two'], 'type': "list<string>", 'spec': {'type': 'textsubdata'}, 'expected': ['one', 'two'] }, { 'value': {'one': 1}, 'type': 'string', 'expected': "one=1" } ] for test in test_data: spec = test.get('spec', None) ret = app_util.transform_param_value(test['type'], test['value'], spec) self.assertEqual(ret, test['expected']) del(os.environ['KB_WORKSPACE_ID'])
def test_input_mapping(self): self.maxDiff = None inputs = { "reads_tuple": [ { "input_reads_label": "reads file 1", "input_reads_obj": "rhodobacterium.art.q20.int.PE.reads", "input_reads_metadata": { "key1": "value1" } }, { "input_reads_label": "reads file 2", "input_reads_obj": "rhodobacterium.art.q10.PE.reads", "input_reads_metadata": { "key2": "value2" } } ], "output_object": "MyReadsSet", "description": "New Reads Set" } app_id = "NarrativeTest/test_create_set" tag = "dev" ws_name = self.public_ws prev_ws_id = os.environ.get('KB_WORKSPACE_ID', None) os.environ['KB_WORKSPACE_ID'] = ws_name sm = specmanager.SpecManager() spec = sm.get_spec(app_id, tag=tag) spec_params = sm.app_params(spec) spec_params_map = dict( (spec_params[i]['id'], spec_params[i]) for i in range(len(spec_params)) ) mapped_inputs = self.am._map_inputs( spec['behavior']['kb_service_input_mapping'], inputs, spec_params_map ) expected = [{ u'output_object_name': 'MyReadsSet', u'data': { u'items': [{ u'label': 'reads file 1', u'metadata': {'key1': 'value1'}, u'ref': '12345/7/1' }, { u'label': 'reads file 2', u'metadata': {'key2': 'value2'}, u'ref': '12345/8/1' }], u'description': 'New Reads Set' }, u'workspace': ws_name }] self.assertDictEqual(expected[0], mapped_inputs[0]) ref_path = ws_name + '/MyReadsSet; ' + ws_name + "/rhodobacterium.art.q10.PE.reads" ret = app_util.transform_param_value("resolved-ref", ref_path, None) self.assertEqual(ret, ws_name + '/MyReadsSet;18836/5/1') if prev_ws_id is None: del(os.environ['KB_WORKSPACE_ID']) else: os.environ['KB_WORKSPACE_ID'] = prev_ws_id
def test_transform_input_good(self): ws_name = self.public_ws test_data = [ { "value": "input_value", "type": "ref", "expected": ws_name + "/" + "input_value", }, { "value": ws_name + "/input_value", "type": "ref", "expected": ws_name + "/" + "input_value", }, { "value": "input_value", "type": "unresolved-ref", "expected": ws_name + "/" + "input_value", }, { "value": READS_OBJ_1, "type": "resolved-ref", "expected": "11635/9/1", }, { "value": ws_name + "/" + READS_OBJ_1, "type": "resolved-ref", "expected": "11635/9/1", }, { "value": None, "type": "int", "expected": None }, { "value": "5", "type": "int", "expected": 5 }, { "value": ["a", "b", "c"], "type": "list<ref>", "expected": [ws_name + "/a", ws_name + "/b", ws_name + "/c"], }, { "value": [ READS_OBJ_1, READS_OBJ_2, ], "type": "list<resolved-ref>", "expected": ["11635/9/1", "11635/10/1"], }, { "value": "foo", "type": "list<ref>", "expected": [ws_name + "/foo"] }, { "value": ["1", "2", 3], "type": "list<int>", "expected": [1, 2, 3] }, { "value": "bar", "type": None, "expected": "bar" }, { "value": READS_OBJ_1, "type": "future-default", "spec": { "is_output": 0, "allowed_types": ["Some.KnownType"] }, "expected": "11635/9/1", }, { "value": [123, 456], "type": None, "expected": [123, 456] }, { "value": 123, "type": "string", "expected": "123" }, { "value": ["one", "two"], "type": None, "spec": { "type": "textsubdata" }, "expected": "one,two", }, { "value": ["one", "two"], "type": "list<string>", "spec": { "type": "textsubdata" }, "expected": ["one", "two"], }, { "value": { "one": 1 }, "type": "string", "expected": "one=1" }, ] for test in test_data: spec = test.get("spec", None) ret = app_util.transform_param_value(test["type"], test["value"], spec) self.assertEqual(ret, test["expected"])
def test_input_mapping(self): self.maxDiff = None inputs = { "reads_tuple": [ { "input_reads_label": READS_FILE_1, "input_reads_obj": READS_OBJ_1, "input_reads_metadata": { "key1": "value1" }, }, { "input_reads_label": READS_FILE_2, "input_reads_obj": READS_OBJ_2, "input_reads_metadata": { "key2": "value2" }, }, ], "output_object": "MyReadsSet", "description": NEW_READS_SET, } app_id = "NarrativeTest/test_create_set" tag = "dev" ws_name = self.public_ws spec = self.am.spec_manager.get_spec(app_id, tag=tag) spec_params = self.am.spec_manager.app_params(spec) spec_params_map = dict((spec_params[i]["id"], spec_params[i]) for i in range(len(spec_params))) mapped_inputs = self.am._map_inputs( spec["behavior"]["kb_service_input_mapping"], inputs, spec_params_map) expected = [{ "output_object_name": "MyReadsSet", "data": { "items": [ { "label": READS_FILE_1, "metadata": { "key1": "value1" }, "ref": "12345/7/1", }, { "label": READS_FILE_2, "metadata": { "key2": "value2" }, "ref": "12345/8/1", }, ], "description": NEW_READS_SET, }, "workspace": ws_name, }] self.assertDictEqual(expected[0], mapped_inputs[0]) ref_path = (ws_name + "/MyReadsSet; " + ws_name + "/rhodobacterium.art.q10.PE.reads") ret = app_util.transform_param_value("resolved-ref", ref_path, None) self.assertEqual(ret, ws_name + "/MyReadsSet;18836/5/1")