def test_vector(self):
rows = {
'fields': ['a', 'b'],
'rows': [
{'a': [1, 2, 3], 'b': ['1', '2']},
{'a': [4, 5, 6], 'b': ['3', '4']}
]
}
# Conversion to vtkTable
vtktable = convert(
'table',
{'format': 'rows', 'data': rows},
{'format': 'vtktable'}
)['data']
self.assertEqual(vtktable.GetNumberOfRows(), 2)
self.assertEqual(vtktable.GetNumberOfColumns(), 2)
a = vtktable.GetColumnByName('a')
b = vtktable.GetColumnByName('b')
self.assertEqual(a.GetNumberOfComponents(), 3)
self.assertEqual(b.GetNumberOfComponents(), 2)
self.assertTrue(isinstance(a, vtk.vtkDoubleArray))
self.assertTrue(isinstance(b, vtk.vtkStringArray))
for i in range(6):
self.assertEqual(a.GetValue(i), i + 1)
for i in range(4):
self.assertEqual(b.GetValue(i), str(i + 1))
# Conversion back to rows
rows2 = convert(
'table',
{'format': 'vtktable', 'data': vtktable},
{'format': 'rows'}
)['data']
self.assertEqual(rows2, rows)
def test_adjacencylist(self):
output = convert(
'graph', self.test_input['distances'], {'format': 'adjacencylist'})
expected_edges = set(self.test_input['distances']['data'].edges())
actual_edges = set()
for line in output['data'].splitlines():
parts = line.split(' ', 1)
if len(parts) > 1:
source, targets = parts
for target in targets.split(' '):
edge = (source, target)
self.assertNotIn(edge, actual_edges)
actual_edges.add(edge)
self.assertEqual(expected_edges, actual_edges)
output = convert(
'graph', output, {'format': 'networkx'})
# Don't take edges into consideration, because they were lost in the
# original conversion
self.assertTrue(
is_isomorphic(output['data'],
self.test_input['distances']['data'],
edge_match=None))
def test_treestore(self):
output = convert(
'tree',
{'format': 'newick', 'data': self.newick},
{'format': 'r.apetree'})
output = convert('tree', output, {'format': 'treestore'})
self.assertEqual(output['format'], 'treestore')
rows = bson.decode_all(output['data'])
for d in rows:
if 'rooted' in d:
root = d
self.assertNotEqual(root, None)
self.assertEqual(len(root['clades']), 1)
def findId(id):
for d in rows:
if d['_id'] == id:
return d
top = findId(root['clades'][0])
self.assertEqual(len(top['clades']), 2)
internal = findId(top['clades'][0])
rubribarbus = findId(top['clades'][1])
ahli = findId(internal['clades'][0])
allogus = findId(internal['clades'][1])
self.assertEqual(internal['branch_length'], 2)
self.assertEqual(ahli['name'], 'ahli')
self.assertEqual(ahli['branch_length'], 0)
self.assertEqual(allogus['name'], 'allogus')
self.assertEqual(allogus['branch_length'], 1)
self.assertEqual(rubribarbus['name'], 'rubribarbus')
self.assertEqual(rubribarbus['branch_length'], 3)
def test_graphml(self):
output = convert(
'graph', self.test_input['distances'], {'format': 'graphml'})
expected_edges = set(self.test_input['distances']['data'].edges(
data='distance'))
actual_edges = set()
self.assertIsInstance(output['data'], (str, unicode))
tree = etree.fromstring(output['data'])
self.assertEqual(len(tree), 2)
self.assertEqual(tree[0].tag, self.GRAPHML_NS + 'key')
self.assertEqual(tree[1].tag, self.GRAPHML_NS + 'graph')
for edge in tree[1].findall(self.GRAPHML_NS + 'edge'):
edge = (edge.attrib['source'],
edge.attrib['target'],
int(edge.find(self.GRAPHML_NS + 'data').text))
self.assertNotIn(edge, actual_edges)
actual_edges.add(edge)
self.assertEqual(expected_edges, actual_edges)
output = convert(
'graph', output, {'format': 'networkx'})
self.assertTrue(
is_isomorphic(output['data'],
self.test_input['distances']['data'],
edge_match=numerical_edge_match('distance', 1)))
def test_convert(self):
tmp = tempfile.mktemp()
output = convert(
"image",
{"format": "png.base64", "data": self.image},
{"format": "png", "url": "file://" + tmp, "mode": "auto"},
)
value = open(tmp).read()
os.remove(tmp)
self.assertEqual(output["format"], "png")
self.assertEqual(base64.b64encode(value), self.image)
output = convert("image", {"format": "png.base64", "data": self.image}, {"format": "pil"})
tmp = tempfile.mktemp()
output = convert("image", output, {"format": "png"})
io1 = StringIO(base64.b64decode(self.image))
im1 = Image.open(io1)
io2 = StringIO(output["data"])
im2 = Image.open(io2)
self.assertEqual(compareImages(im1, im2), 0)
output = convert("image", {"format": "png.base64", "data": self.image}, {"format": "jpeg"})
data = StringIO(output["data"])
jpeg = Image.open(data)
self.assertTrue(isinstance(jpeg, JpegImageFile))
def test_non_binary_tree(self):
convert(
'tree',
{
'format': 'newick',
'url': 'file://' +
os.path.join('data', 'geospiza_from_otl.phy')
},
{'format': 'nested'})
def test_vtkgraph(self):
# Test vtkgraph -> vtkgraph.serialized on a simple digraph
output = convert(
'graph', self.test_input['simpleVtkDiGraph'],
{'format': 'vtkgraph.serialized'})
with open(os.path.join(
'tests', 'data', 'vtkDiGraph.txt'), 'rb') as fixture:
self.assertEqual(output['data'].splitlines()[1:],
fixture.read().splitlines()[1:])
# Test networkx -> vtkgraph.serialized on an undirected
# graph w/ edge data
output = convert(
'graph', self.test_input['distances'],
{'format': 'vtkgraph.serialized'})
with open(os.path.join(
'tests', 'data', 'vtkDistancesUndirectedGraph.txt'),
'rb') as fixture:
self.assertEqual(output['data'].splitlines()[1:],
fixture.read().splitlines()[1:])
# Test networkx -> vtkgraph with missing edge attributes
output = convert(
'graph', self.test_input['grants'],
{'format': 'vtkgraph.serialized'})
with open(os.path.join('tests', 'data', 'vtkGrantsDirectedGraph.txt'),
'rb') as fixture:
self.assertEqual(output['data'].splitlines()[1:],
fixture.read().splitlines()[1:])
# Test networkx -> vtkgraph throws errors for different types
# of metadata
with self.assertRaises(Exception):
output = convert(
'graph', {'format': 'networkx', 'data': nx.Graph([
('A', 'B', {'value': 10}),
('B', 'C', {'value': '10'})
])}, {'format': 'vtkgraph'})
# Test vtkgraph -> networkx
output = convert(
'graph', self.test_input['simpleVtkDiGraph'],
{'format': 'networkx'})
self.assertIsInstance(output['data'], nx.DiGraph)
self.assertEqual(len(output['data'].nodes()), 3)
self.assertEqual(len(output['data'].edges()), 3)
self.assertEqual(sorted(output['data'].edges(data=True)),
[(0, 1, {'Weights': 1.0}),
(0, 2, {'Weights': 2.0}),
(1, 2, {'Weights': 1.0})])
def round_trip(self, obj):
"""Convert an object to base64 and back returning the new object."""
b64 = convert(
'python',
{'format': 'object', 'data': obj},
{'format': 'pickle.base64'}
)['data']
newobj = convert(
'python',
{'format': 'pickle.base64', 'data': b64},
{'format': 'object'}
)
return newobj['data']
def test_objectlist_to_rows(self):
objlist = [{'a': {'b': 5}}, {'a': {'b': {'c': 3}}}]
output = convert('table', {
'format': 'objectlist',
'data': objlist
}, {'format': 'rows'})
self.assertEqual(output['format'], 'rows')
self.assertEqual(output['data'], {
'fields': ['a.b', 'a.b.c'],
'rows': [{'a.b': 5}, {'a.b.c': 3}]})
output = convert('table', {
'format': 'rows',
'data': output['data']
}, {'format': 'objectlist'})
self.assertEqual(output['data'], objlist)
def test_column_names_csv(self):
output = convert('table', {
'format': 'csv',
'data': ',a,b,longer name\n1,1,1,1\n2,2,2,2\n3,3,3,3\n'
}, {'format': 'column.names'})
self.assertEqual(output['format'], 'column.names')
self.assertEqual(output['data'], ['', 'a', 'b', 'longer name'])
def test_sniffer(self):
output = convert(
'table',
{
'format': 'csv',
'url': 'file://' + os.path.join('data', 'test.csv')
},
{'format': 'rows'}
)
self.assertEqual(len(output['data']['fields']), 32)
self.assertEqual(output['data']['fields'][:3], [
'FACILITY', 'ADDRESS', 'DATE OF INSPECTION'
])
self.assertEqual(len(output['data']['rows']), 14)
flu = load(os.path.join(
self.analysis_path, 'xdata', 'flu.json'))
output = run(
flu,
inputs={},
outputs={'data': {'type': 'table', 'format': 'rows'}}
)
self.assertEqual(output['data']['data']['fields'][:3], [
'Date', 'United States', 'Alabama'
])
def test_jsonlines(self):
output = convert('table', {
'format': 'jsonlines',
'data': '{"a": 1, "b": 2}\n{"a": 3, "b": 4}'
}, {'format': 'objectlist'})
self.assertEqual(output['format'], 'objectlist')
self.assertEqual(output['data'], [{'a': 1, 'b': 2}, {'a': 3, 'b': 4}])
def test_column_names(self):
output = convert('table', {
'format': 'rows',
'data': {'fields': ['a', 'b'], 'rows': [{'a': 6, 'b': 5}]}
}, {'format': 'column.names'})
self.assertEqual(output['format'], 'column.names')
self.assertEqual(output['data'], ['a', 'b'])
def test_header_detection(self):
output = convert(
'table',
{'format': 'csv', 'data': 'a,b,c\n7,1,c\n8,2,f\n9,3,i'},
{'format': 'rows'}
)
self.assertEqual(output['data']['fields'], ['a', 'b', 'c'])
self.assertEqual(len(output['data']['rows']), 3)
output = convert(
'table',
{'format': 'csv', 'data': '1,2,3\n7,10,\n,11,\n,12,'},
{'format': 'rows'}
)
self.assertEqual(
output['data']['fields'],
['1', '2', '3']
)
self.assertEqual(len(output['data']['rows']), 3)
def test_clique(self):
# clique.json -> NetworkX
output = convert(
'graph', self.test_input['alphabetGraph'], {'format': 'networkx'})
self.assertEqual(
set([n[1]['name'] for n in output['data'].nodes(data=True)]),
set(['a', 'b', 'c', 'd']))
self.assertEqual(len(output['data'].edges()), 3)
self.assertEqual(output['data'].degree('55ba5019f8883b5bf35f3e30'), 0)
# NetworkX -> clique.json
output = convert(
'graph', output, {'format': 'clique.json'})
# Since the id of the nodes are lost, only test the structure
# Check nodes with names a, b, c, and d
# Check the following edges
# a -> b
# a -> c
# b -> c
output['data'] = json.loads(output['data'])
nodes = [item for item in output['data'] if item['type'] == 'node']
edges = [(item['source']['$oid'], item['target']['$oid'])
for item in output['data'] if item['type'] == 'link']
oid_by_name = {}
for node in nodes:
oid_by_name[node['data']['name']] = node['_id']['$oid']
# Check nodes
self.assertEqual(sorted(oid_by_name.keys()),
['a', 'b', 'c', 'd'])
# Check edges
self.assertEqual(len(edges), 3)
self.assertIn((oid_by_name['a'], oid_by_name['b']), edges)
self.assertIn((oid_by_name['a'], oid_by_name['c']), edges)
self.assertIn((oid_by_name['b'], oid_by_name['c']), edges)
def test_big_header(self):
output = convert(
'table',
{
'format': 'csv',
'url': 'file://' + os.path.join('data', 'RadiomicsData.csv')
},
{'format': 'rows'}
)
self.assertEqual(len(output['data']['fields']), 454)
self.assertEqual(output['data']['fields'][:3], [
'GLCM_autocorr', 'GLCM_clusProm', 'GLCM_clusShade'
])
self.assertEqual(len(output['data']['rows']), 99)
def test_objectlist(self):
rows = {
'fields': ['a', 'b'],
'rows': [
{'a': 1, 'b': 'x'},
{'a': 4, 'b': 'y'}
]
}
objectlist = convert(
'table',
{'format': 'rows', 'data': rows},
{'format': 'objectlist'}
)['data']
# Should have same row data
self.assertEqual(objectlist, rows['rows'])
rows2 = convert(
'table',
{'format': 'objectlist', 'data': objectlist},
{'format': 'rows'}
)['data']
# Should have same fields but could be in different order
self.assertEqual(set(rows['fields']), set(rows2['fields']))
# Should have same row data
self.assertEqual(rows['rows'], rows2['rows'])
# Make sure we can go back and forth to JSON
objectlist = convert(
'table',
convert(
'table',
{'format': 'objectlist', 'data': rows['rows']},
{'format': 'objectlist.json'}
),
{'format': 'objectlist'}
)['data']
self.assertEqual(rows['rows'], objectlist)
def test_flu(self):
output = convert(
'table',
{
'format': 'csv',
'url': 'file://' + os.path.join('data', 'flu.csv')
},
{'format': 'column.names'}
)
self.assertEqual(output['format'], 'column.names')
self.assertEqual(len(output['data']), 162)
self.assertEqual(
output['data'][:3],
['Date', 'United States', 'Alabama']
)
def test_inputs_from_file(self):
"""Run a task with base64 inputs in a file."""
a = tempfile.NamedTemporaryFile()
b = tempfile.NamedTemporaryFile()
convert(
'python',
{'format': 'object', 'data': (0, 1)},
{'format': 'pickle.base64', 'mode': 'local', 'path': a.name}
)
convert(
'python',
{'format': 'object', 'data': 2},
{'format': 'pickle.base64', 'mode': 'local', 'path': b.name}
)
outputs = self.run_basic_task({
'a': {'format': 'pickle.base64', 'mode': 'local', 'path': a.name},
'b': {'format': 'pickle.base64', 'mode': 'local', 'path': b.name}
})
self.assertEqual(outputs.get('c'), (0, 1, 0, 1))
self.assertEqual(outputs.get('d'), 4)
def test_convert(self):
outputs = run(
self.cone,
inputs={
'resolution': {'format': 'number', 'data': 100},
'radius': {'format': 'number', 'data': 1}
},
outputs={
'cone': {'format': 'vtkpolydata.serialized'}
})
cone = outputs['cone']['data']
self.assertTrue(isinstance(cone, str))
converted = convert(
'geometry',
outputs['cone'],
{'format': 'vtkpolydata'}
)['data']
self.assertTrue(isinstance(converted, vtk.vtkPolyData))
self.assertEqual(converted.GetNumberOfCells(), 101)
self.assertEqual(converted.GetNumberOfPoints(), 101)
def test_nan(self):
output = convert(
'table',
{
'format': 'csv',
'url': 'file://' + os.path.join('data', 'RadiomicsData.csv')
},
{'format': 'rows.json'}
)
data = json.loads(output['data'])
self.assertEqual(len(data['fields']), 454)
self.assertEqual(data['fields'][:3], [
'GLCM_autocorr', 'GLCM_clusProm', 'GLCM_clusShade'
])
self.assertEqual(len(data['rows']), 99)
for row in data['rows']:
for field in row:
if isinstance(row[field], float):
self.assertFalse(math.isnan(row[field]))
self.assertFalse(math.isinf(row[field]))
