def test_workflow_with_generated_task_classes(self):
wf = specs.Workflow()
AddTwo = spec_class_generator("AddTwo", self.add_two)
AddThree = spec_class_generator("AddThree", self.add_three)
Multiply = spec_class_generator("Multiply", self.multiply)
wf.add_task(AddTwo(), "a2")
wf.add_task(AddThree(), "a3")
wf.add_task(Multiply(), "m")
wf.connect_tasks("a3", "m", {"b": "in1"})
wf.connect_tasks("a2", "m", {"b": "in2"})
# Add default as defined in self.workflow
wf.set_default("a3.a", {"format": "number", "data": 10})
self.assertEquals(wf, self.workflow)
inputs = {"a2.a": {"format": "json", "data": "1"},
"a3.a": {"format": "number", "data": 2}}
ground = girder_worker.run(self.workflow, inputs=inputs)
system = girder_worker.run(wf, inputs=inputs)
self.assertEquals(system, ground)
def test_workflow_with_generated_task_classes(self):
wf = specs.Workflow()
AddTwo = spec_class_generator('AddTwo', self.add_two)
AddThree = spec_class_generator('AddThree', self.add_three)
Multiply = spec_class_generator('Multiply', self.multiply)
wf.add_task(AddTwo(), 'a2')
wf.add_task(AddThree(), 'a3')
wf.add_task(Multiply(), 'm')
wf.connect_tasks('a3', 'm', {'b': 'in1'})
wf.connect_tasks('a2', 'm', {'b': 'in2'})
# Add default as defined in self.workflow
wf.set_default('a3.a', {'format': 'number', 'data': 10})
self.assertEquals(wf, self.workflow)
inputs = {'a2.a': {'format': 'json', 'data': '1'},
'a3.a': {'format': 'number', 'data': 2}}
ground = girder_worker.run(self.workflow, inputs=inputs)
system = girder_worker.run(wf, inputs=inputs)
self.assertEquals(system, ground)
def testConvertingStatus(self):
job_mgr = girder_worker.utils.JobManager(True, url='http://jobstatus/')
status_changes = []
task = {
'mode': 'python',
'script': 'y = x',
'inputs': [{
'id': 'x',
'format': 'tsv',
'type': 'table'
}],
'outputs': [{
'id': 'y',
'format': 'tsv',
'type': 'table'
}]
}
inputs = {
'x': {
'mode': 'inline',
'data': 'a,b,c\nd,e,f\n',
'type': 'table',
'format': 'csv'
}
}
outputs = {
'y': {
'mode': 'inline',
'type': 'table',
'format': 'csv'
}
}
@httmock.all_requests
def fetchMock(url, request):
if (url.netloc == 'jobstatus' and url.path == '/' and
request.method == 'PUT'):
status_changes.append(request.body)
return ''
else:
raise Exception('Unexpected url ' + repr(url))
with httmock.HTTMock(fetchMock):
girder_worker.run(
task, inputs=inputs, outputs=outputs, _job_manager=job_mgr,
status=JobStatus.RUNNING)
# We should have received 3 status changes
expected_statuses = [
JobStatus.CONVERTING_INPUT,
JobStatus.RUNNING,
JobStatus.CONVERTING_OUTPUT,
JobStatus.PUSHING_OUTPUT
]
self.assertEqual(status_changes, [
'status=%d' % i for i in expected_statuses])
def test_kwargs_debug(self):
"""Runs the table json test but with asserts for kwarg debugging"""
global _tmpfiles
girder_worker.run(self.analysis, self.inputs, self.outputs, write_script=True)
# Should have generated serveral files ()
self.assertGreater(len(_tmpfiles), 1)
def test_kwargs_debug(self):
"""Runs the table json test but with asserts for kwarg debugging"""
global _tmpfiles
girder_worker.run(self.analysis, self.inputs, self.outputs, debug=True)
# Should have generated serveral files ()
self.assertGreater(len(_tmpfiles), 1)
def test_function(self):
outputs = girder_worker.run(
self.function_out,
inputs={},
outputs={'output': {'format': 'object'}})
self.assertEqual(outputs['output']['data'](3)[0], 9)
outputs = girder_worker.run(
self.function_in, inputs={'input': outputs['output']})
self.assertEqual(outputs['output']['data'], 16)
def test_function(self):
outputs = girder_worker.run(
self.function_out,
inputs={},
outputs={"output": {"format": "object"}})
self.assertEqual(outputs["output"]["data"](3)[0], 9)
outputs = girder_worker.run(
self.function_in, inputs={"input": outputs["output"]})
self.assertEqual(outputs["output"]["data"], 16)
def testInlineFilepath(self):
task = {
'mode': 'python',
'script': """
fname = file
with open(file) as f:
out = f.read()
""",
'inputs': [{
'id': 'file',
'format': 'text',
'type': 'string',
'target': 'filepath'
}],
'outputs': [
{
'id': 'out',
'format': 'text',
'type': 'string'
},
{
'id': 'fname',
'format': 'text',
'type': 'string'
}
]
}
# Test filepath input
inputs = {
'file': {
'data': 'a,b,c\n1,2,3\n',
'format': 'text',
'type': 'string'
}
}
outputs = girder_worker.run(task, inputs)
self.assertEqual(outputs['out']['data'], 'a,b,c\n1,2,3\n')
# Test filepath input with filename specified
task['inputs'][0]['filename'] = 'file.csv'
inputs = {
'file': {
'data': 'a,b,c\n1,2,3\n',
'format': 'text',
'type': 'string'
}
}
outputs = girder_worker.run(task, inputs)
self.assertEqual(outputs['out']['data'], 'a,b,c\n1,2,3\n')
self.assertEqual(outputs['fname']['data'][-8:], 'file.csv')
def test_analysis_debug(self):
"""Runs the table json test but with asserts for analysis debugging"""
global _tmpfiles
girder_worker.run(dict(self.analysis.items() + [('debug', 1)]),
inputs=self.inputs,
outputs=self.outputs)
# Should have generated just one debug file
self.assertEquals(len(_tmpfiles), 1)
# File contents equals script
with open(_tmpfiles[0], 'r') as fh:
self.assertEquals(fh.read(), self.script)
def test_analysis_debug(self):
"""Runs the table json test but with asserts for analysis debugging"""
global _tmpfiles
girder_worker.run(
dict(self.analysis.items() + [("write_script", 1)]),
inputs=self.inputs, outputs=self.outputs)
# Should have generated just one debug file
self.assertEquals(len(_tmpfiles), 1)
# File contents equals script
with open(_tmpfiles[0], "r") as fh:
self.assertEquals(fh.read(), self.script)
def test_pgls(self):
pgls = girder_worker.load(os.path.join(self.arbor_path, 'pgls.json'))
tree_file = os.path.join('data', 'anolis.phy')
table_file = os.path.join('data', 'anolisDataAppended.csv')
girder_worker.run(
pgls,
{
'tree': {'format': 'newick', 'url': 'file://' + tree_file},
'table': {'format': 'csv', 'url': 'file://' + table_file},
'correlation': {'format': 'text', 'data': 'BM'},
'ind_variable': {'format': 'text', 'data': 'SVL'},
'dep_variable': {'format': 'text', 'data': 'PCI_limbs'}
}
)
def test_array(self):
outputs = girder_worker.run(
self.array_out,
inputs={},
outputs={"output": {"format": "serialized"}})
self.assertEqual('\n'.join(outputs["output"]["data"].split('\n')[3:]),
"131840\n14\n3\n1\n2\n3\n")
outputs = girder_worker.run(
self.array_in,
inputs={"input": outputs["output"]},
outputs={"output": {"format": "serialized"}})
self.assertEqual('\n'.join(outputs["output"]["data"].split('\n')[3:]),
"131840\n14\n5\n1\n2\n3\n4\n5\n")
def test_pgls(self):
pgls = girder_worker.load(os.path.join(self.arbor_path, "pgls.json"))
tree_file = os.path.join("data", "anolis.phy")
table_file = os.path.join("data", "anolisDataAppended.csv")
girder_worker.run(
pgls,
{
"tree": {"format": "newick", "url": "file://" + tree_file},
"table": {"format": "csv", "url": "file://" + table_file},
"correlation": {"format": "text", "data": "BM"},
"ind_variable": {"format": "text", "data": "SVL"},
"dep_variable": {"format": "text", "data": "PCI_limbs"}
}
)
def test_array(self):
outputs = girder_worker.run(
self.array_out,
inputs={},
outputs={'output': {'format': 'serialized'}})
self.assertEqual('\n'.join(outputs['output']['data'].split('\n')[3:]),
'131840\n14\n3\n1\n2\n3\n')
outputs = girder_worker.run(
self.array_in,
inputs={'input': outputs['output']},
outputs={'output': {'format': 'serialized'}})
self.assertEqual('\n'.join(outputs['output']['data'].split('\n')[3:]),
'131840\n14\n5\n1\n2\n3\n4\n5\n')
def test_cont2disc(self):
cont2disc = girder_worker.load(
os.path.join(self.arbor_path, 'continuous_to_discrete.json'))
table_file = os.path.join('data', 'anolisDataAppended.csv')
girder_worker.run(
cont2disc,
{
'table': {'format': 'csv', 'url': 'file://' + table_file},
'column': {'format': 'text', 'data': 'SVL'},
'thresh': {'format': 'number', 'data': 3.5}
},
{
'newtable': {'format': 'rows'}
}
)
def test_cont2disc(self):
cont2disc = girder_worker.load(
os.path.join(self.arbor_path, "continuous_to_discrete.json"))
table_file = os.path.join("data", "anolisDataAppended.csv")
girder_worker.run(
cont2disc,
{
"table": {"format": "csv", "url": "file://" + table_file},
"column": {"format": "text", "data": "SVL"},
"thresh": {"format": "number", "data": 3.5}
},
{
"newtable": {"format": "rows"}
}
)
def test_sniffer(self):
output = girder_worker.convert(
"table",
{
"format": "csv",
"url": "file://" + os.path.join("data", "test.csv")
},
{"format": "rows"}
)
self.assertEqual(len(output["data"]["fields"]), 32)
self.assertEqual(output["data"]["fields"][:3], [
"FACILITY", "ADDRESS", "DATE OF INSPECTION"
])
self.assertEqual(len(output["data"]["rows"]), 14)
flu = girder_worker.load(os.path.join(
self.analysis_path, "xdata", "flu.json"))
output = girder_worker.run(
flu,
inputs={},
outputs={"data": {"type": "table", "format": "rows"}}
)
self.assertEqual(output["data"]["data"]["fields"][:3], [
"Date", "United States", "Alabama"
])
def testCreateZip(self):
task = {
'inputs': [{
'id': 'myArchive',
'type': 'directory',
'format': 'zip'
}],
'outputs': [{
'id': 'output',
'type': 'string',
'format': 'text'
}],
'script': 'output = myArchive',
'mode': 'python'
}
outputs = girder_worker.run(task, inputs={
'myArchive': {
'format': 'path',
'data': self.createFromDir
}
})
path = outputs['output']['data']
self.assertEqual(path, os.path.join(DATA_DIR, 'shapefile.zip'))
self.assertTrue(os.path.isfile(path))
with zipfile.ZipFile(path, 'r') as zf:
names = zf.namelist()
self.assertTrue('shapefile/shapefile.cpg' in names)
self.assertTrue('shapefile/shapefile.prj' in names)
def testCreateZip(self):
task = {
"inputs": [{
"id": "myArchive",
"type": "directory",
"format": "zip"
}],
"outputs": [{
"id": "output",
"type": "string",
"format": "text"
}],
"script": "output = myArchive",
"mode": "python"
}
outputs = girder_worker.run(task, inputs={
"myArchive": {
"format": "path",
"data": self.createFromDir
}
})
path = outputs['output']['data']
self.assertEqual(path, os.path.join(DATA_DIR, 'shapefile.zip'))
self.assertTrue(os.path.isfile(path))
with zipfile.ZipFile(path, 'r') as zf:
names = zf.namelist()
self.assertTrue('shapefile/shapefile.cpg' in names)
self.assertTrue('shapefile/shapefile.prj' in names)
def testDefaultInline(self):
task = {
'mode': 'python',
'script': 'b = a ** 2',
'inputs': [{
'id': 'a',
'format': 'number',
'type': 'number'
}],
'outputs': [{
'id': 'b',
'format': 'number',
'type': 'number'
}]
}
# Mode should default to "inline" if data key is set
inputs = {
'a': {
'data': 5,
'format': 'number',
'type': 'number'
}
}
outputs = girder_worker.run(task, inputs)
self.assertEqual(outputs['b']['data'], 25)
def testSparkRddToJson(self):
analysis = {
'name': 'map',
'inputs': [
{'name': 'a', 'type': 'collection', 'format': 'spark.rdd'}
],
'outputs': [
{'name': 'b', 'type': 'collection', 'format': 'spark.rdd'}
],
'mode': 'spark.python',
'spark_conf': {
'spark.app.name': 'test_square',
'spark.master': os.environ['SPARK_TEST_MASTER_URL']
}
}
with open('data/spark_square_script.py', 'r') as fp:
script = fp.read()
analysis['script'] = script
outputs = girder_worker.run(analysis,
{'a': {'format': 'json',
'data': '[1,2,3,4,5,6,7,8,9]'}},
{'b': {'format': 'json'}})
expected = {
'b': {
'data': '[1, 4, 9, 16, 25, 36, 49, 64, 81]',
'format': 'json'
}
}
self.assertEqual(outputs, expected)
def test_bson(self):
import pymongo
outputs = girder_worker.run(
self.analysis,
inputs={
"a": {
"format": "objectlist.bson",
"mode": "mongodb",
"db": "test",
"collection": "a"
},
"b": {
"format": "objectlist.bson",
"mode": "mongodb",
"db": "test",
"collection": "b"
}
},
outputs={
"c": {
"format": "objectlist.bson",
"mode": "mongodb",
"db": "test",
"collection": "temp"
}
})
self.assertEqual(outputs["c"]["format"], "objectlist.bson")
coll = pymongo.MongoClient("mongodb://localhost")["test"]["temp"]
self.assertEqual([d for d in coll.find()], [self.aobj, self.bobj])
def test_base64(self):
outputs = girder_worker.run(
self.analysis,
inputs={
"a": {"format": "png.base64", "data": self.image},
})
self.assertEqual(outputs["pixels"]["format"], "number")
self.assertEqual(outputs["pixels"]["data"], 256)
def test_mammal_tree(self):
mammal = girder_worker.load(
os.path.join("data", "Mammal tree extraction.json"))
girder_worker.run(
mammal,
{
"table": {
"format": "csv",
"url": "file://" +
os.path.join("data", "mammal_lnMass_tiny.csv")
},
"outRowCount": {"format": "number", "data": 19}
},
{
"tree": {"format": "nested"}
}
)
def test_newick(self):
outputs = girder_worker.run(
self.analysis,
inputs={"a": {"format": "newick", "data": self.newick}},
outputs={"b": {"format": "newick"}}
)
self.assertEqual(outputs["b"]["format"], "newick")
self.assertEqual(outputs["b"]["data"], self.newick)
def test_base64(self):
outputs = girder_worker.run(
self.analysis,
inputs={
'a': {'format': 'png.base64', 'data': self.image},
})
self.assertEqual(outputs['pixels']['format'], 'number')
self.assertEqual(outputs['pixels']['data'], 256)
def test_newick(self):
outputs = girder_worker.run(
self.analysis,
inputs={'a': {'format': 'newick', 'data': self.newick}},
outputs={'b': {'format': 'newick'}}
)
self.assertEqual(outputs['b']['format'], 'newick')
self.assertEqual(outputs['b']['data'], self.newick)
def test_multi_input(self):
outputs = girder_worker.run(
self.multi_input,
inputs={
"x": {"format": "number", "data": 2},
"y": {"format": "number", "data": 3}
})
self.assertEqual(outputs["result"]["format"], "number")
self.assertEqual(outputs["result"]["data"], (2*2)+(3*3))
def test_workflow(self):
outputs = girder_worker.run(
self.workflow,
inputs={
"x": {"format": "json", "data": "1"},
"y": {"format": "number", "data": 2}
})
self.assertEqual(outputs["result"]["format"], "number")
self.assertEqual(outputs["result"]["data"], (1+3)*(2+2))
# Test using the default value for x (10).
outputs = girder_worker.run(
self.workflow,
inputs={
"y": {"format": "number", "data": 2}
})
self.assertEqual(outputs["result"]["format"], "number")
self.assertEqual(outputs["result"]["data"], (10+3)*(2+2))
def test_r_apetree(self):
outputs = girder_worker.run(
self.analysis,
inputs={'a': {'format': 'newick', 'data': self.newick}},
outputs={'b': {'format': 'r.apetree'}}
)
self.assertEqual(outputs['b']['format'], 'r.apetree')
self.assertEqual(
str(outputs['b']['data'])[:52],
'\nPhylogenetic tree with 3 tips and 2 internal nodes.')
def test_r_apetree(self):
outputs = girder_worker.run(
self.analysis,
inputs={"a": {"format": "newick", "data": self.newick}},
outputs={"b": {"format": "r.apetree"}}
)
self.assertEqual(outputs["b"]["format"], "r.apetree")
self.assertEqual(
str(outputs["b"]["data"])[:52],
'\nPhylogenetic tree with 3 tips and 2 internal nodes.')
def test_cone(self):
outputs = girder_worker.run(
self.cone,
inputs={
"resolution": {"format": "number", "data": 100},
"radius": {"format": "number", "data": 1}
})
self.assertEqual(outputs["cone"]["format"], "vtkpolydata")
cone = outputs["cone"]["data"]
self.assertTrue(isinstance(cone, vtk.vtkPolyData))
self.assertEqual(cone.GetNumberOfCells(), 101)
self.assertEqual(cone.GetNumberOfPoints(), 101)
