def _transform_input(self, transform_type, value, spec_param):
"""
Transforms an input according to the rules given in
NarrativeMethodStore.ServiceMethodInputMapping
Really, there are three types of transforms possible:
1. ref - turns the input string into a workspace ref.
2. int - tries to coerce the input string into an int.
3. list<type> - turns the given list into a list of the given type.
(4.) none or None - doesn't transform.
Returns a transformed (or not) value.
"""
return transform_param_value(transform_type, value, spec_param)
def _map_inputs(self, input_mapping, params, spec_params):
"""
Maps the dictionary of parameters and inputs based on rules provided in
the input_mapping. This iterates over the list of input_mappings, and
uses them as a filter to apply to each parameter.
Returns a list of inputs that can be passed directly to NJSW.run_job
input_mapping is a list of dicts, as defined by
NarrativeMethodStore.ServiceMethodInputMapping.
params is a dict of key-value-pairs, each key is the input_parameter
field of some parameter.
"""
inputs_dict = dict()
for p in input_mapping:
# 2 steps - figure out the proper value, then figure out the
# proper position. value first!
p_value = None
input_param_id = None
if 'input_parameter' in p:
input_param_id = p['input_parameter']
p_value = params.get(input_param_id, None)
if spec_params[input_param_id].get('type', '') == 'group':
p_value = self._map_group_inputs(p_value, spec_params[input_param_id],
spec_params)
# turn empty strings into None
if isinstance(p_value, basestring) and len(p_value) == 0:
p_value = None
elif 'narrative_system_variable' in p:
p_value = system_variable(p['narrative_system_variable'])
if 'constant_value' in p and p_value is None:
p_value = p['constant_value']
if 'generated_value' in p and p_value is None:
p_value = self._generate_input(p['generated_value'])
spec_param = None
if input_param_id:
spec_param = spec_params[input_param_id]
p_value = transform_param_value(p.get('target_type_transform'), p_value, spec_param)
# get position!
arg_position = p.get('target_argument_position', 0)
target_prop = p.get('target_property', None)
if target_prop is not None:
final_input = inputs_dict.get(arg_position, dict())
if '/' in target_prop:
# This is case when slashes in target_prop separeate
# elements in nested maps. We ignore escaped slashes
# (separate backslashes should be escaped as well).
bck_slash = u"\u244A"
fwd_slash = u"\u20EB"
temp_string = target_prop.replace("\\\\", bck_slash)
temp_string = temp_string.replace("\\/", fwd_slash)
temp_path = []
for part in temp_string.split("/"):
part = part.replace(bck_slash, "\\")
part = part.replace(fwd_slash, "/")
temp_path.append(part.encode('ascii', 'ignore'))
temp_map = final_input
temp_key = None
# We're going along the path and creating intermediate
# dictionaries.
for temp_path_item in temp_path:
if temp_key:
if temp_key not in temp_map:
temp_map[temp_key] = {}
temp_map = temp_map[temp_key]
temp_key = temp_path_item
# temp_map points to deepest nested map now, temp_key is
# the last item in the path
temp_map[temp_key] = p_value
else:
final_input[target_prop] = p_value
inputs_dict[arg_position] = final_input
else:
inputs_dict[arg_position] = p_value
inputs_list = list()
keys = sorted(inputs_dict.keys())
for k in keys:
inputs_list.append(inputs_dict[k])
return inputs_list
def test_transform_input_bad(self):
with self.assertRaises(ValueError):
app_util.transform_param_value('foo', 'bar', None)
def test_transform_input_good(self):
ws_name = self.public_ws
os.environ['KB_WORKSPACE_ID'] = ws_name
test_data = [{
'value': 'input_value',
'type': 'ref',
'expected': ws_name + '/' + 'input_value'
}, {
'value': ws_name + "/input_value",
'type': 'ref',
'expected': ws_name + '/' + 'input_value'
}, {
'value': 'input_value',
'type': 'unresolved-ref',
'expected': ws_name + '/' + 'input_value'
}, {
'value': 'rhodobacterium.art.q20.int.PE.reads',
'type': 'resolved-ref',
'expected': '11635/9/1'
}, {
'value': ws_name + "/rhodobacterium.art.q20.int.PE.reads",
'type': 'resolved-ref',
'expected': '11635/9/1'
}, {
'value': None,
'type': 'int',
'expected': None
}, {
'value': '5',
'type': 'int',
'expected': 5
}, {
'value': ['a', 'b', 'c'],
'type':
'list<ref>',
'expected': [ws_name + '/a', ws_name + '/b', ws_name + '/c']
}, {
'value': [
'rhodobacterium.art.q20.int.PE.reads',
'rhodobacterium.art.q10.PE.reads'
],
'type':
'list<resolved-ref>',
'expected': ['11635/9/1', '11635/10/1']
}, {
'value': 'foo',
'type': 'list<ref>',
'expected': [ws_name + '/foo']
}, {
'value': ['1', '2', 3],
'type': 'list<int>',
'expected': [1, 2, 3]
}, {
'value': 'bar',
'type': None,
'expected': 'bar'
}, {
'value': 'rhodobacterium.art.q20.int.PE.reads',
'type': 'future-default',
'spec': {
'is_output': 0,
'allowed_types': ["Some.KnownType"]
},
'expected': '11635/9/1'
}, {
'value': [123, 456],
'type': None,
'expected': [123, 456]
}, {
'value': 123,
'type': 'string',
'expected': '123'
}, {
'value': ['one', 'two'],
'type': None,
'spec': {
'type': 'textsubdata'
},
'expected': "one,two"
}, {
'value': ['one', 'two'],
'type': "list<string>",
'spec': {
'type': 'textsubdata'
},
'expected': ['one', 'two']
}, {
'value': {
'one': 1
},
'type': 'string',
'expected': "one=1"
}]
for test in test_data:
spec = test.get('spec', None)
ret = app_util.transform_param_value(test['type'], test['value'],
spec)
self.assertEqual(ret, test['expected'])
del (os.environ['KB_WORKSPACE_ID'])
def test_input_mapping(self):
self.maxDiff = None
inputs = {
"reads_tuple": [{
"input_reads_label": "reads file 1",
"input_reads_obj": "rhodobacterium.art.q20.int.PE.reads",
"input_reads_metadata": {
"key1": "value1"
}
}, {
"input_reads_label": "reads file 2",
"input_reads_obj": "rhodobacterium.art.q10.PE.reads",
"input_reads_metadata": {
"key2": "value2"
}
}],
"output_object":
"MyReadsSet",
"description":
"New Reads Set"
}
app_id = "NarrativeTest/test_create_set"
tag = "dev"
ws_name = self.public_ws
prev_ws_id = os.environ.get('KB_WORKSPACE_ID', None)
os.environ['KB_WORKSPACE_ID'] = ws_name
sm = specmanager.SpecManager()
spec = sm.get_spec(app_id, tag=tag)
spec_params = sm.app_params(spec)
spec_params_map = dict((spec_params[i]['id'], spec_params[i])
for i in range(len(spec_params)))
mapped_inputs = self.am._map_inputs(
spec['behavior']['kb_service_input_mapping'], inputs,
spec_params_map)
expected = [{
u'output_object_name': 'MyReadsSet',
u'data': {
u'items': [{
u'label': 'reads file 1',
u'metadata': {
'key1': 'value1'
},
u'ref': '12345/7/1'
}, {
u'label': 'reads file 2',
u'metadata': {
'key2': 'value2'
},
u'ref': '12345/8/1'
}],
u'description':
'New Reads Set'
},
u'workspace': ws_name
}]
self.assertDictEqual(expected[0], mapped_inputs[0])
ref_path = ws_name + '/MyReadsSet; ' + ws_name + "/rhodobacterium.art.q10.PE.reads"
ret = app_util.transform_param_value("resolved-ref", ref_path, None)
self.assertEqual(ret, ws_name + '/MyReadsSet;18836/5/1')
if prev_ws_id is None:
del (os.environ['KB_WORKSPACE_ID'])
else:
os.environ['KB_WORKSPACE_ID'] = prev_ws_id
def _map_inputs(self, input_mapping, params, spec_params):
"""
Maps the dictionary of parameters and inputs based on rules provided in
the input_mapping. This iterates over the list of input_mappings, and
uses them as a filter to apply to each parameter.
Returns a list of inputs that can be passed directly to NJSW.run_job
input_mapping is a list of dicts, as defined by
NarrativeMethodStore.ServiceMethodInputMapping.
params is a dict of key-value-pairs, each key is the input_parameter
field of some parameter.
"""
inputs_dict = dict()
for p in input_mapping:
# 2 steps - figure out the proper value, then figure out the
# proper position. value first!
p_value = None
input_param_id = None
if 'input_parameter' in p:
input_param_id = p['input_parameter']
p_value = params.get(input_param_id, None)
if spec_params[input_param_id].get('type', '') == 'group':
p_value = self._map_group_inputs(
p_value, spec_params[input_param_id], spec_params)
# turn empty strings into None
if isinstance(p_value, str) and len(p_value) == 0:
p_value = None
elif 'narrative_system_variable' in p:
p_value = system_variable(p['narrative_system_variable'])
if 'constant_value' in p and p_value is None:
p_value = p['constant_value']
if 'generated_value' in p and p_value is None:
p_value = self._generate_input(p['generated_value'])
spec_param = None
if input_param_id:
spec_param = spec_params[input_param_id]
p_value = transform_param_value(p.get('target_type_transform'),
p_value, spec_param)
# get position!
arg_position = p.get('target_argument_position', 0)
target_prop = p.get('target_property', None)
if target_prop is not None:
final_input = inputs_dict.get(arg_position, dict())
if '/' in target_prop:
# This is case when slashes in target_prop separate
# elements in nested maps. We ignore escaped slashes
# (separate backslashes should be escaped as well).
bck_slash = "\u244A"
fwd_slash = "\u20EB"
temp_string = target_prop.replace("\\\\", bck_slash)
temp_string = temp_string.replace("\\/", fwd_slash)
temp_path = []
for part in temp_string.split("/"):
part = part.replace(bck_slash, "\\")
part = part.replace(fwd_slash, "/")
temp_path.append(
part.encode('ascii', 'ignore').decode("ascii"))
temp_map = final_input
temp_key = None
# We're going along the path and creating intermediate
# dictionaries.
for temp_path_item in temp_path:
if temp_key:
if temp_key not in temp_map:
temp_map[temp_key] = {}
temp_map = temp_map[temp_key]
temp_key = temp_path_item
# temp_map points to deepest nested map now, temp_key is
# the last item in the path
temp_map[temp_key] = p_value
else:
final_input[target_prop] = p_value
inputs_dict[arg_position] = final_input
else:
inputs_dict[arg_position] = p_value
inputs_list = list()
keys = sorted(inputs_dict.keys())
for k in keys:
inputs_list.append(inputs_dict[k])
return inputs_list
def test_transform_input_good(self):
ws_name = self.public_ws
os.environ["KB_WORKSPACE_ID"] = ws_name
test_data = [
{
"value": "input_value",
"type": "ref",
"expected": ws_name + "/" + "input_value",
},
{
"value": ws_name + "/input_value",
"type": "ref",
"expected": ws_name + "/" + "input_value",
},
{
"value": "input_value",
"type": "unresolved-ref",
"expected": ws_name + "/" + "input_value",
},
{
"value": "rhodobacterium.art.q20.int.PE.reads",
"type": "resolved-ref",
"expected": "11635/9/1",
},
{
"value": ws_name + "/rhodobacterium.art.q20.int.PE.reads",
"type": "resolved-ref",
"expected": "11635/9/1",
},
{
"value": None,
"type": "int",
"expected": None
},
{
"value": "5",
"type": "int",
"expected": 5
},
{
"value": ["a", "b", "c"],
"type": "list<ref>",
"expected": [ws_name + "/a", ws_name + "/b", ws_name + "/c"],
},
{
"value": [
"rhodobacterium.art.q20.int.PE.reads",
"rhodobacterium.art.q10.PE.reads",
],
"type":
"list<resolved-ref>",
"expected": ["11635/9/1", "11635/10/1"],
},
{
"value": "foo",
"type": "list<ref>",
"expected": [ws_name + "/foo"]
},
{
"value": ["1", "2", 3],
"type": "list<int>",
"expected": [1, 2, 3]
},
{
"value": "bar",
"type": None,
"expected": "bar"
},
{
"value": "rhodobacterium.art.q20.int.PE.reads",
"type": "future-default",
"spec": {
"is_output": 0,
"allowed_types": ["Some.KnownType"]
},
"expected": "11635/9/1",
},
{
"value": [123, 456],
"type": None,
"expected": [123, 456]
},
{
"value": 123,
"type": "string",
"expected": "123"
},
{
"value": ["one", "two"],
"type": None,
"spec": {
"type": "textsubdata"
},
"expected": "one,two",
},
{
"value": ["one", "two"],
"type": "list<string>",
"spec": {
"type": "textsubdata"
},
"expected": ["one", "two"],
},
{
"value": {
"one": 1
},
"type": "string",
"expected": "one=1"
},
]
for test in test_data:
spec = test.get("spec", None)
ret = app_util.transform_param_value(test["type"], test["value"],
spec)
self.assertEqual(ret, test["expected"])
del os.environ["KB_WORKSPACE_ID"]
def test_input_mapping(self):
self.maxDiff = None
inputs = {
"reads_tuple": [
{
"input_reads_label": "reads file 1",
"input_reads_obj": "rhodobacterium.art.q20.int.PE.reads",
"input_reads_metadata": {
"key1": "value1"
},
},
{
"input_reads_label": "reads file 2",
"input_reads_obj": "rhodobacterium.art.q10.PE.reads",
"input_reads_metadata": {
"key2": "value2"
},
},
],
"output_object":
"MyReadsSet",
"description":
"New Reads Set",
}
app_id = "NarrativeTest/test_create_set"
tag = "dev"
ws_name = self.public_ws
prev_ws_id = os.environ.get("KB_WORKSPACE_ID", None)
os.environ["KB_WORKSPACE_ID"] = ws_name
sm = specmanager.SpecManager()
spec = sm.get_spec(app_id, tag=tag)
spec_params = sm.app_params(spec)
spec_params_map = dict((spec_params[i]["id"], spec_params[i])
for i in range(len(spec_params)))
mapped_inputs = self.am._map_inputs(
spec["behavior"]["kb_service_input_mapping"], inputs,
spec_params_map)
expected = [{
"output_object_name": "MyReadsSet",
"data": {
"items": [
{
"label": "reads file 1",
"metadata": {
"key1": "value1"
},
"ref": "12345/7/1",
},
{
"label": "reads file 2",
"metadata": {
"key2": "value2"
},
"ref": "12345/8/1",
},
],
"description":
"New Reads Set",
},
"workspace": ws_name,
}]
self.assertDictEqual(expected[0], mapped_inputs[0])
ref_path = (ws_name + "/MyReadsSet; " + ws_name +
"/rhodobacterium.art.q10.PE.reads")
ret = app_util.transform_param_value("resolved-ref", ref_path, None)
self.assertEqual(ret, ws_name + "/MyReadsSet;18836/5/1")
if prev_ws_id is None:
del os.environ["KB_WORKSPACE_ID"]
else:
os.environ["KB_WORKSPACE_ID"] = prev_ws_id
def test_transform_input_bad(self):
with self.assertRaises(ValueError):
app_util.transform_param_value('foo', 'bar', None)
def test_transform_input_good(self):
ws_name = self.public_ws
os.environ['KB_WORKSPACE_ID'] = ws_name
test_data = [
{
'value': 'input_value',
'type': 'ref',
'expected': ws_name + '/' + 'input_value'
},
{
'value': ws_name + "/input_value",
'type': 'ref',
'expected': ws_name + '/' + 'input_value'
},
{
'value': 'input_value',
'type': 'unresolved-ref',
'expected': ws_name + '/' + 'input_value'
},
{
'value': 'rhodobacterium.art.q20.int.PE.reads',
'type': 'resolved-ref',
'expected': '11635/9/1'
},
{
'value': ws_name + "/rhodobacterium.art.q20.int.PE.reads",
'type': 'resolved-ref',
'expected': '11635/9/1'
},
{
'value': None,
'type': 'int',
'expected': None
},
{
'value': '5',
'type': 'int',
'expected': 5
},
{
'value': ['a', 'b', 'c'],
'type': 'list<ref>',
'expected': [ws_name + '/a', ws_name + '/b', ws_name + '/c']
},
{
'value': ['rhodobacterium.art.q20.int.PE.reads',
'rhodobacterium.art.q10.PE.reads'],
'type': 'list<resolved-ref>',
'expected': ['11635/9/1', '11635/10/1']
},
{
'value': 'foo',
'type': 'list<ref>',
'expected': [ws_name + '/foo']
},
{
'value': ['1', '2', 3],
'type': 'list<int>',
'expected': [1, 2, 3]
},
{
'value': 'bar',
'type': None,
'expected': 'bar'
},
{
'value': 'rhodobacterium.art.q20.int.PE.reads',
'type': 'future-default',
'spec': {'is_output': 0, 'allowed_types': ["Some.KnownType"]},
'expected': '11635/9/1'
},
{
'value': [123, 456],
'type': None,
'expected': [123, 456]
},
{
'value': 123,
'type': 'string',
'expected': '123'
},
{
'value': ['one', 'two'],
'type': None,
'spec': {'type': 'textsubdata'},
'expected': "one,two"
},
{
'value': ['one', 'two'],
'type': "list<string>",
'spec': {'type': 'textsubdata'},
'expected': ['one', 'two']
},
{
'value': {'one': 1},
'type': 'string',
'expected': "one=1"
}
]
for test in test_data:
spec = test.get('spec', None)
ret = app_util.transform_param_value(test['type'], test['value'], spec)
self.assertEqual(ret, test['expected'])
del(os.environ['KB_WORKSPACE_ID'])
def test_input_mapping(self):
self.maxDiff = None
inputs = {
"reads_tuple": [
{
"input_reads_label": "reads file 1",
"input_reads_obj": "rhodobacterium.art.q20.int.PE.reads",
"input_reads_metadata": {
"key1": "value1"
}
},
{
"input_reads_label": "reads file 2",
"input_reads_obj": "rhodobacterium.art.q10.PE.reads",
"input_reads_metadata": {
"key2": "value2"
}
}
],
"output_object": "MyReadsSet",
"description": "New Reads Set"
}
app_id = "NarrativeTest/test_create_set"
tag = "dev"
ws_name = self.public_ws
prev_ws_id = os.environ.get('KB_WORKSPACE_ID', None)
os.environ['KB_WORKSPACE_ID'] = ws_name
sm = specmanager.SpecManager()
spec = sm.get_spec(app_id, tag=tag)
spec_params = sm.app_params(spec)
spec_params_map = dict(
(spec_params[i]['id'], spec_params[i])
for i in range(len(spec_params))
)
mapped_inputs = self.am._map_inputs(
spec['behavior']['kb_service_input_mapping'],
inputs,
spec_params_map
)
expected = [{
u'output_object_name': 'MyReadsSet',
u'data': {
u'items': [{
u'label': 'reads file 1',
u'metadata': {'key1': 'value1'},
u'ref': '12345/7/1'
}, {
u'label': 'reads file 2',
u'metadata': {'key2': 'value2'},
u'ref': '12345/8/1'
}],
u'description': 'New Reads Set'
},
u'workspace': ws_name
}]
self.assertDictEqual(expected[0], mapped_inputs[0])
ref_path = ws_name + '/MyReadsSet; ' + ws_name + "/rhodobacterium.art.q10.PE.reads"
ret = app_util.transform_param_value("resolved-ref", ref_path, None)
self.assertEqual(ret, ws_name + '/MyReadsSet;18836/5/1')
if prev_ws_id is None:
del(os.environ['KB_WORKSPACE_ID'])
else:
os.environ['KB_WORKSPACE_ID'] = prev_ws_id
def test_transform_input_good(self):
ws_name = self.public_ws
test_data = [
{
"value": "input_value",
"type": "ref",
"expected": ws_name + "/" + "input_value",
},
{
"value": ws_name + "/input_value",
"type": "ref",
"expected": ws_name + "/" + "input_value",
},
{
"value": "input_value",
"type": "unresolved-ref",
"expected": ws_name + "/" + "input_value",
},
{
"value": READS_OBJ_1,
"type": "resolved-ref",
"expected": "11635/9/1",
},
{
"value": ws_name + "/" + READS_OBJ_1,
"type": "resolved-ref",
"expected": "11635/9/1",
},
{
"value": None,
"type": "int",
"expected": None
},
{
"value": "5",
"type": "int",
"expected": 5
},
{
"value": ["a", "b", "c"],
"type": "list<ref>",
"expected": [ws_name + "/a", ws_name + "/b", ws_name + "/c"],
},
{
"value": [
READS_OBJ_1,
READS_OBJ_2,
],
"type": "list<resolved-ref>",
"expected": ["11635/9/1", "11635/10/1"],
},
{
"value": "foo",
"type": "list<ref>",
"expected": [ws_name + "/foo"]
},
{
"value": ["1", "2", 3],
"type": "list<int>",
"expected": [1, 2, 3]
},
{
"value": "bar",
"type": None,
"expected": "bar"
},
{
"value": READS_OBJ_1,
"type": "future-default",
"spec": {
"is_output": 0,
"allowed_types": ["Some.KnownType"]
},
"expected": "11635/9/1",
},
{
"value": [123, 456],
"type": None,
"expected": [123, 456]
},
{
"value": 123,
"type": "string",
"expected": "123"
},
{
"value": ["one", "two"],
"type": None,
"spec": {
"type": "textsubdata"
},
"expected": "one,two",
},
{
"value": ["one", "two"],
"type": "list<string>",
"spec": {
"type": "textsubdata"
},
"expected": ["one", "two"],
},
{
"value": {
"one": 1
},
"type": "string",
"expected": "one=1"
},
]
for test in test_data:
spec = test.get("spec", None)
ret = app_util.transform_param_value(test["type"], test["value"],
spec)
self.assertEqual(ret, test["expected"])
def test_input_mapping(self):
self.maxDiff = None
inputs = {
"reads_tuple": [
{
"input_reads_label": READS_FILE_1,
"input_reads_obj": READS_OBJ_1,
"input_reads_metadata": {
"key1": "value1"
},
},
{
"input_reads_label": READS_FILE_2,
"input_reads_obj": READS_OBJ_2,
"input_reads_metadata": {
"key2": "value2"
},
},
],
"output_object":
"MyReadsSet",
"description":
NEW_READS_SET,
}
app_id = "NarrativeTest/test_create_set"
tag = "dev"
ws_name = self.public_ws
spec = self.am.spec_manager.get_spec(app_id, tag=tag)
spec_params = self.am.spec_manager.app_params(spec)
spec_params_map = dict((spec_params[i]["id"], spec_params[i])
for i in range(len(spec_params)))
mapped_inputs = self.am._map_inputs(
spec["behavior"]["kb_service_input_mapping"], inputs,
spec_params_map)
expected = [{
"output_object_name": "MyReadsSet",
"data": {
"items": [
{
"label": READS_FILE_1,
"metadata": {
"key1": "value1"
},
"ref": "12345/7/1",
},
{
"label": READS_FILE_2,
"metadata": {
"key2": "value2"
},
"ref": "12345/8/1",
},
],
"description":
NEW_READS_SET,
},
"workspace": ws_name,
}]
self.assertDictEqual(expected[0], mapped_inputs[0])
ref_path = (ws_name + "/MyReadsSet; " + ws_name +
"/rhodobacterium.art.q10.PE.reads")
ret = app_util.transform_param_value("resolved-ref", ref_path, None)
self.assertEqual(ret, ws_name + "/MyReadsSet;18836/5/1")
