def test_workflow_with_generated_task_classes(self): wf = specs.Workflow() AddTwo = spec_class_generator("AddTwo", self.add_two) AddThree = spec_class_generator("AddThree", self.add_three) Multiply = spec_class_generator("Multiply", self.multiply) wf.add_task(AddTwo(), "a2") wf.add_task(AddThree(), "a3") wf.add_task(Multiply(), "m") wf.connect_tasks("a3", "m", {"b": "in1"}) wf.connect_tasks("a2", "m", {"b": "in2"}) # Add default as defined in self.workflow wf.set_default("a3.a", {"format": "number", "data": 10}) self.assertEquals(wf, self.workflow) inputs = {"a2.a": {"format": "json", "data": "1"}, "a3.a": {"format": "number", "data": 2}} ground = girder_worker.run(self.workflow, inputs=inputs) system = girder_worker.run(wf, inputs=inputs) self.assertEquals(system, ground)
def test_workflow_with_generated_task_classes(self): wf = specs.Workflow() AddTwo = spec_class_generator('AddTwo', self.add_two) AddThree = spec_class_generator('AddThree', self.add_three) Multiply = spec_class_generator('Multiply', self.multiply) wf.add_task(AddTwo(), 'a2') wf.add_task(AddThree(), 'a3') wf.add_task(Multiply(), 'm') wf.connect_tasks('a3', 'm', {'b': 'in1'}) wf.connect_tasks('a2', 'm', {'b': 'in2'}) # Add default as defined in self.workflow wf.set_default('a3.a', {'format': 'number', 'data': 10}) self.assertEquals(wf, self.workflow) inputs = {'a2.a': {'format': 'json', 'data': '1'}, 'a3.a': {'format': 'number', 'data': 2}} ground = girder_worker.run(self.workflow, inputs=inputs) system = girder_worker.run(wf, inputs=inputs) self.assertEquals(system, ground)
def testConvertingStatus(self): job_mgr = girder_worker.utils.JobManager(True, url='http://jobstatus/') status_changes = [] task = { 'mode': 'python', 'script': 'y = x', 'inputs': [{ 'id': 'x', 'format': 'tsv', 'type': 'table' }], 'outputs': [{ 'id': 'y', 'format': 'tsv', 'type': 'table' }] } inputs = { 'x': { 'mode': 'inline', 'data': 'a,b,c\nd,e,f\n', 'type': 'table', 'format': 'csv' } } outputs = { 'y': { 'mode': 'inline', 'type': 'table', 'format': 'csv' } } @httmock.all_requests def fetchMock(url, request): if (url.netloc == 'jobstatus' and url.path == '/' and request.method == 'PUT'): status_changes.append(request.body) return '' else: raise Exception('Unexpected url ' + repr(url)) with httmock.HTTMock(fetchMock): girder_worker.run( task, inputs=inputs, outputs=outputs, _job_manager=job_mgr, status=JobStatus.RUNNING) # We should have received 3 status changes expected_statuses = [ JobStatus.CONVERTING_INPUT, JobStatus.RUNNING, JobStatus.CONVERTING_OUTPUT, JobStatus.PUSHING_OUTPUT ] self.assertEqual(status_changes, [ 'status=%d' % i for i in expected_statuses])
def test_kwargs_debug(self): """Runs the table json test but with asserts for kwarg debugging""" global _tmpfiles girder_worker.run(self.analysis, self.inputs, self.outputs, write_script=True) # Should have generated serveral files () self.assertGreater(len(_tmpfiles), 1)
def test_kwargs_debug(self): """Runs the table json test but with asserts for kwarg debugging""" global _tmpfiles girder_worker.run(self.analysis, self.inputs, self.outputs, debug=True) # Should have generated serveral files () self.assertGreater(len(_tmpfiles), 1)
def test_function(self): outputs = girder_worker.run( self.function_out, inputs={}, outputs={'output': {'format': 'object'}}) self.assertEqual(outputs['output']['data'](3)[0], 9) outputs = girder_worker.run( self.function_in, inputs={'input': outputs['output']}) self.assertEqual(outputs['output']['data'], 16)
def test_function(self): outputs = girder_worker.run( self.function_out, inputs={}, outputs={"output": {"format": "object"}}) self.assertEqual(outputs["output"]["data"](3)[0], 9) outputs = girder_worker.run( self.function_in, inputs={"input": outputs["output"]}) self.assertEqual(outputs["output"]["data"], 16)
def testInlineFilepath(self): task = { 'mode': 'python', 'script': """ fname = file with open(file) as f: out = f.read() """, 'inputs': [{ 'id': 'file', 'format': 'text', 'type': 'string', 'target': 'filepath' }], 'outputs': [ { 'id': 'out', 'format': 'text', 'type': 'string' }, { 'id': 'fname', 'format': 'text', 'type': 'string' } ] } # Test filepath input inputs = { 'file': { 'data': 'a,b,c\n1,2,3\n', 'format': 'text', 'type': 'string' } } outputs = girder_worker.run(task, inputs) self.assertEqual(outputs['out']['data'], 'a,b,c\n1,2,3\n') # Test filepath input with filename specified task['inputs'][0]['filename'] = 'file.csv' inputs = { 'file': { 'data': 'a,b,c\n1,2,3\n', 'format': 'text', 'type': 'string' } } outputs = girder_worker.run(task, inputs) self.assertEqual(outputs['out']['data'], 'a,b,c\n1,2,3\n') self.assertEqual(outputs['fname']['data'][-8:], 'file.csv')
def test_analysis_debug(self): """Runs the table json test but with asserts for analysis debugging""" global _tmpfiles girder_worker.run(dict(self.analysis.items() + [('debug', 1)]), inputs=self.inputs, outputs=self.outputs) # Should have generated just one debug file self.assertEquals(len(_tmpfiles), 1) # File contents equals script with open(_tmpfiles[0], 'r') as fh: self.assertEquals(fh.read(), self.script)
def test_analysis_debug(self): """Runs the table json test but with asserts for analysis debugging""" global _tmpfiles girder_worker.run( dict(self.analysis.items() + [("write_script", 1)]), inputs=self.inputs, outputs=self.outputs) # Should have generated just one debug file self.assertEquals(len(_tmpfiles), 1) # File contents equals script with open(_tmpfiles[0], "r") as fh: self.assertEquals(fh.read(), self.script)
def test_pgls(self): pgls = girder_worker.load(os.path.join(self.arbor_path, 'pgls.json')) tree_file = os.path.join('data', 'anolis.phy') table_file = os.path.join('data', 'anolisDataAppended.csv') girder_worker.run( pgls, { 'tree': {'format': 'newick', 'url': 'file://' + tree_file}, 'table': {'format': 'csv', 'url': 'file://' + table_file}, 'correlation': {'format': 'text', 'data': 'BM'}, 'ind_variable': {'format': 'text', 'data': 'SVL'}, 'dep_variable': {'format': 'text', 'data': 'PCI_limbs'} } )
def test_array(self): outputs = girder_worker.run( self.array_out, inputs={}, outputs={"output": {"format": "serialized"}}) self.assertEqual('\n'.join(outputs["output"]["data"].split('\n')[3:]), "131840\n14\n3\n1\n2\n3\n") outputs = girder_worker.run( self.array_in, inputs={"input": outputs["output"]}, outputs={"output": {"format": "serialized"}}) self.assertEqual('\n'.join(outputs["output"]["data"].split('\n')[3:]), "131840\n14\n5\n1\n2\n3\n4\n5\n")
def test_pgls(self): pgls = girder_worker.load(os.path.join(self.arbor_path, "pgls.json")) tree_file = os.path.join("data", "anolis.phy") table_file = os.path.join("data", "anolisDataAppended.csv") girder_worker.run( pgls, { "tree": {"format": "newick", "url": "file://" + tree_file}, "table": {"format": "csv", "url": "file://" + table_file}, "correlation": {"format": "text", "data": "BM"}, "ind_variable": {"format": "text", "data": "SVL"}, "dep_variable": {"format": "text", "data": "PCI_limbs"} } )
def test_array(self): outputs = girder_worker.run( self.array_out, inputs={}, outputs={'output': {'format': 'serialized'}}) self.assertEqual('\n'.join(outputs['output']['data'].split('\n')[3:]), '131840\n14\n3\n1\n2\n3\n') outputs = girder_worker.run( self.array_in, inputs={'input': outputs['output']}, outputs={'output': {'format': 'serialized'}}) self.assertEqual('\n'.join(outputs['output']['data'].split('\n')[3:]), '131840\n14\n5\n1\n2\n3\n4\n5\n')
def test_cont2disc(self): cont2disc = girder_worker.load( os.path.join(self.arbor_path, 'continuous_to_discrete.json')) table_file = os.path.join('data', 'anolisDataAppended.csv') girder_worker.run( cont2disc, { 'table': {'format': 'csv', 'url': 'file://' + table_file}, 'column': {'format': 'text', 'data': 'SVL'}, 'thresh': {'format': 'number', 'data': 3.5} }, { 'newtable': {'format': 'rows'} } )
def test_cont2disc(self): cont2disc = girder_worker.load( os.path.join(self.arbor_path, "continuous_to_discrete.json")) table_file = os.path.join("data", "anolisDataAppended.csv") girder_worker.run( cont2disc, { "table": {"format": "csv", "url": "file://" + table_file}, "column": {"format": "text", "data": "SVL"}, "thresh": {"format": "number", "data": 3.5} }, { "newtable": {"format": "rows"} } )
def test_sniffer(self): output = girder_worker.convert( "table", { "format": "csv", "url": "file://" + os.path.join("data", "test.csv") }, {"format": "rows"} ) self.assertEqual(len(output["data"]["fields"]), 32) self.assertEqual(output["data"]["fields"][:3], [ "FACILITY", "ADDRESS", "DATE OF INSPECTION" ]) self.assertEqual(len(output["data"]["rows"]), 14) flu = girder_worker.load(os.path.join( self.analysis_path, "xdata", "flu.json")) output = girder_worker.run( flu, inputs={}, outputs={"data": {"type": "table", "format": "rows"}} ) self.assertEqual(output["data"]["data"]["fields"][:3], [ "Date", "United States", "Alabama" ])
def testCreateZip(self): task = { 'inputs': [{ 'id': 'myArchive', 'type': 'directory', 'format': 'zip' }], 'outputs': [{ 'id': 'output', 'type': 'string', 'format': 'text' }], 'script': 'output = myArchive', 'mode': 'python' } outputs = girder_worker.run(task, inputs={ 'myArchive': { 'format': 'path', 'data': self.createFromDir } }) path = outputs['output']['data'] self.assertEqual(path, os.path.join(DATA_DIR, 'shapefile.zip')) self.assertTrue(os.path.isfile(path)) with zipfile.ZipFile(path, 'r') as zf: names = zf.namelist() self.assertTrue('shapefile/shapefile.cpg' in names) self.assertTrue('shapefile/shapefile.prj' in names)
def testCreateZip(self): task = { "inputs": [{ "id": "myArchive", "type": "directory", "format": "zip" }], "outputs": [{ "id": "output", "type": "string", "format": "text" }], "script": "output = myArchive", "mode": "python" } outputs = girder_worker.run(task, inputs={ "myArchive": { "format": "path", "data": self.createFromDir } }) path = outputs['output']['data'] self.assertEqual(path, os.path.join(DATA_DIR, 'shapefile.zip')) self.assertTrue(os.path.isfile(path)) with zipfile.ZipFile(path, 'r') as zf: names = zf.namelist() self.assertTrue('shapefile/shapefile.cpg' in names) self.assertTrue('shapefile/shapefile.prj' in names)
def testDefaultInline(self): task = { 'mode': 'python', 'script': 'b = a ** 2', 'inputs': [{ 'id': 'a', 'format': 'number', 'type': 'number' }], 'outputs': [{ 'id': 'b', 'format': 'number', 'type': 'number' }] } # Mode should default to "inline" if data key is set inputs = { 'a': { 'data': 5, 'format': 'number', 'type': 'number' } } outputs = girder_worker.run(task, inputs) self.assertEqual(outputs['b']['data'], 25)
def testSparkRddToJson(self): analysis = { 'name': 'map', 'inputs': [ {'name': 'a', 'type': 'collection', 'format': 'spark.rdd'} ], 'outputs': [ {'name': 'b', 'type': 'collection', 'format': 'spark.rdd'} ], 'mode': 'spark.python', 'spark_conf': { 'spark.app.name': 'test_square', 'spark.master': os.environ['SPARK_TEST_MASTER_URL'] } } with open('data/spark_square_script.py', 'r') as fp: script = fp.read() analysis['script'] = script outputs = girder_worker.run(analysis, {'a': {'format': 'json', 'data': '[1,2,3,4,5,6,7,8,9]'}}, {'b': {'format': 'json'}}) expected = { 'b': { 'data': '[1, 4, 9, 16, 25, 36, 49, 64, 81]', 'format': 'json' } } self.assertEqual(outputs, expected)
def test_bson(self): import pymongo outputs = girder_worker.run( self.analysis, inputs={ "a": { "format": "objectlist.bson", "mode": "mongodb", "db": "test", "collection": "a" }, "b": { "format": "objectlist.bson", "mode": "mongodb", "db": "test", "collection": "b" } }, outputs={ "c": { "format": "objectlist.bson", "mode": "mongodb", "db": "test", "collection": "temp" } }) self.assertEqual(outputs["c"]["format"], "objectlist.bson") coll = pymongo.MongoClient("mongodb://localhost")["test"]["temp"] self.assertEqual([d for d in coll.find()], [self.aobj, self.bobj])
def test_base64(self): outputs = girder_worker.run( self.analysis, inputs={ "a": {"format": "png.base64", "data": self.image}, }) self.assertEqual(outputs["pixels"]["format"], "number") self.assertEqual(outputs["pixels"]["data"], 256)
def test_mammal_tree(self): mammal = girder_worker.load( os.path.join("data", "Mammal tree extraction.json")) girder_worker.run( mammal, { "table": { "format": "csv", "url": "file://" + os.path.join("data", "mammal_lnMass_tiny.csv") }, "outRowCount": {"format": "number", "data": 19} }, { "tree": {"format": "nested"} } )
def test_newick(self): outputs = girder_worker.run( self.analysis, inputs={"a": {"format": "newick", "data": self.newick}}, outputs={"b": {"format": "newick"}} ) self.assertEqual(outputs["b"]["format"], "newick") self.assertEqual(outputs["b"]["data"], self.newick)
def test_base64(self): outputs = girder_worker.run( self.analysis, inputs={ 'a': {'format': 'png.base64', 'data': self.image}, }) self.assertEqual(outputs['pixels']['format'], 'number') self.assertEqual(outputs['pixels']['data'], 256)
def test_newick(self): outputs = girder_worker.run( self.analysis, inputs={'a': {'format': 'newick', 'data': self.newick}}, outputs={'b': {'format': 'newick'}} ) self.assertEqual(outputs['b']['format'], 'newick') self.assertEqual(outputs['b']['data'], self.newick)
def test_multi_input(self): outputs = girder_worker.run( self.multi_input, inputs={ "x": {"format": "number", "data": 2}, "y": {"format": "number", "data": 3} }) self.assertEqual(outputs["result"]["format"], "number") self.assertEqual(outputs["result"]["data"], (2*2)+(3*3))
def test_workflow(self): outputs = girder_worker.run( self.workflow, inputs={ "x": {"format": "json", "data": "1"}, "y": {"format": "number", "data": 2} }) self.assertEqual(outputs["result"]["format"], "number") self.assertEqual(outputs["result"]["data"], (1+3)*(2+2)) # Test using the default value for x (10). outputs = girder_worker.run( self.workflow, inputs={ "y": {"format": "number", "data": 2} }) self.assertEqual(outputs["result"]["format"], "number") self.assertEqual(outputs["result"]["data"], (10+3)*(2+2))
def test_r_apetree(self): outputs = girder_worker.run( self.analysis, inputs={'a': {'format': 'newick', 'data': self.newick}}, outputs={'b': {'format': 'r.apetree'}} ) self.assertEqual(outputs['b']['format'], 'r.apetree') self.assertEqual( str(outputs['b']['data'])[:52], '\nPhylogenetic tree with 3 tips and 2 internal nodes.')
def test_r_apetree(self): outputs = girder_worker.run( self.analysis, inputs={"a": {"format": "newick", "data": self.newick}}, outputs={"b": {"format": "r.apetree"}} ) self.assertEqual(outputs["b"]["format"], "r.apetree") self.assertEqual( str(outputs["b"]["data"])[:52], '\nPhylogenetic tree with 3 tips and 2 internal nodes.')
def test_cone(self): outputs = girder_worker.run( self.cone, inputs={ "resolution": {"format": "number", "data": 100}, "radius": {"format": "number", "data": 1} }) self.assertEqual(outputs["cone"]["format"], "vtkpolydata") cone = outputs["cone"]["data"] self.assertTrue(isinstance(cone, vtk.vtkPolyData)) self.assertEqual(cone.GetNumberOfCells(), 101) self.assertEqual(cone.GetNumberOfPoints(), 101)