def test_csv_numeric(self):
"""Uses CSVFile and ExtractColumn to load a numeric array.
"""
with intercept_result(ExtractColumn, 'value') as results:
with intercept_result(CSVFile, 'column_count') as columns:
self.assertFalse(execute([
('read|CSVFile', identifier, [
('file', [('File', self._test_dir + '/test.csv')]),
]),
('ExtractColumn', identifier, [
('column_index', [('Integer', '1')]),
('column_name', [('String', 'col 2')]),
('numeric', [('Boolean', 'True')]),
]),
('PythonSource', 'org.vistrails.vistrails.basic', [
('source', [('String', '')]),
]),
],
[
(0, 'value', 1, 'table'),
(1, 'value', 2, 'l'),
],
add_port_specs=[
(2, 'input', 'l',
'org.vistrails.vistrails.basic:List'),
]))
# Here we use a PythonSource just to check that a numpy array
# can be passed on a List port
self.assertEqual(columns, [3])
self.assertEqual(len(results), 1)
self.assertEqual(list(results[0]), [2.0, 3.0, 14.5])
def test_xls_header_numeric(self):
"""Uses ExcelSpreadsheet to load a numeric array.
"""
with intercept_result(ExtractColumn, "value") as results:
with intercept_result(ExcelSpreadsheet, "column_count") as cols:
self.assertFalse(
execute(
[
(
"read|ExcelSpreadsheet",
identifier,
[
("file", [("File", self._test_dir + "/xl.xls")]),
# Will default to first sheet
("header_present", [("Boolean", "True")]),
],
),
(
"ExtractColumn",
identifier,
[("column_name", [("String", "data2")]), ("numeric", [("Boolean", "True")])],
),
],
[(0, "self", 1, "table")],
)
)
self.assertEqual(cols, [2])
self.assertEqual(len(results), 1)
self.assertAlmostEqual_lists(list(results[0]), [1, -2.8, 3.4, 3.3])
def test_xls_header_numeric(self):
"""Uses ExcelSpreadsheet to load a numeric array.
"""
with intercept_result(ExtractColumn, 'value') as results:
with intercept_result(ExcelSpreadsheet, 'column_count') as cols:
self.assertFalse(
execute(
[
(
'read|ExcelSpreadsheet',
identifier,
[
('file', [
('File', self._test_dir + '/xl.xls')
]),
# Will default to first sheet
('header_present', [('Boolean', 'True')])
]),
('ExtractColumn', identifier, [
('column_name', [('String', 'data2')]),
('numeric', [('Boolean', 'True')]),
]),
],
[
(0, 'value', 1, 'table'),
]))
self.assertEqual(cols, [2])
self.assertEqual(len(results), 1)
self.assertAlmostEqual_lists(list(results[0]), [1, -2.8, 3.4, 3.3])
def test_xls_numeric(self):
"""Uses ExcelSpreadsheet to load a numeric array.
"""
with intercept_result(ExtractColumn, "value") as results:
with intercept_result(ExcelSpreadsheet, "column_count") as cols:
self.assertFalse(
execute(
[
(
"read|ExcelSpreadsheet",
identifier,
[
("file", [("File", self._test_dir + "/xl.xls")]),
("sheet_index", [("Integer", "1")]),
("sheet_name", [("String", "Feuil2")]),
("header_present", [("Boolean", "False")]),
],
),
(
"ExtractColumn",
identifier,
[("column_index", [("Integer", "0")]), ("numeric", [("Boolean", "True")])],
),
],
[(0, "self", 1, "table")],
)
)
self.assertEqual(cols, [1])
self.assertEqual(len(results), 1)
self.assertAlmostEqual_lists(list(results[0]), [1, 2, 2, 3, -7.6])
def test_xls_header_nonnumeric(self):
"""Uses ExcelSpreadsheet to load data.
"""
with intercept_result(ExtractColumn, "value") as results:
with intercept_result(ExcelSpreadsheet, "column_count") as cols:
self.assertFalse(
execute(
[
(
"read|ExcelSpreadsheet",
identifier,
[
("file", [("File", self._test_dir + "/xl.xls")]),
("sheet_name", [("String", "Feuil1")]),
("header_present", [("Boolean", "True")]),
],
),
(
"ExtractColumn",
identifier,
[
("column_index", [("Integer", "0")]),
("column_name", [("String", "data1")]),
("numeric", [("Boolean", "False")]),
],
),
],
[(0, "self", 1, "table")],
)
)
self.assertEqual(cols, [2])
self.assertEqual(len(results), 1)
self.assertEqual(list(results[0]), ["here", "is", "some", "text"])
def test_csv_numeric(self):
"""Uses CSVFile and ExtractColumn to load a numeric array.
"""
with intercept_result(ExtractColumn, 'value') as results:
with intercept_result(CSVFile, 'column_count') as columns:
self.assertFalse(
execute([
('read|CSVFile', identifier, [
('file', [('File', self._test_dir + '/test.csv')]),
]),
('ExtractColumn', identifier, [
('column_index', [('Integer', '1')]),
('column_name', [('String', 'col 2')]),
('numeric', [('Boolean', 'True')]),
]),
('PythonSource', 'org.vistrails.vistrails.basic', [
('source', [('String', '')]),
]),
], [
(0, 'value', 1, 'table'),
(1, 'value', 2, 'l'),
],
add_port_specs=[
(2, 'input', 'l',
'org.vistrails.vistrails.basic:List'),
]))
# Here we use a PythonSource just to check that a numpy array
# can be passed on a List port
self.assertEqual(columns, [3])
self.assertEqual(len(results), 1)
self.assertEqual(list(results[0]), [2.0, 3.0, 14.5])
def test_simple(self):
"""Test converting timestamps into matplotlib's format.
"""
try:
import matplotlib
except ImportError: # pragma: no cover
self.skipTest("matplotlib is not available")
from matplotlib.dates import date2num
with intercept_result(TimestampsToMatplotlib, "dates") as results:
self.assertFalse(
execute(
[
(
"convert|dates|TimestampsToMatplotlib",
identifier,
[("timestamps", [("List", "[1324842375, 1369842877]")])],
)
]
)
)
self.assertEqual(len(results), 1)
results = results[0]
self.assertEqual(
list(results),
list(
date2num(
[datetime.datetime.utcfromtimestamp(1324842375), datetime.datetime.utcfromtimestamp(1369842877)]
)
),
)
def test_timezone(self):
"""Test reading timezone-aware dates by supplying an offset.
"""
dates = ["2013-05-20 9:25", "2013-05-20 09:31", "2013-01-02 19:05"]
in_fmt = "%Y-%m-%d %H:%M"
with intercept_result(StringsToDates, "dates") as results:
self.assertFalse(
execute(
[
(
"convert|dates|StringsToDates",
identifier,
[
("strings", [("List", repr(dates))]),
("format", [("String", in_fmt)]),
("timezone", [("String", "-0500")]),
],
)
]
)
)
self.assertEqual(len(results), 1)
results = results[0]
self.assertTrue(all(d.tzinfo is not None for d in results))
fmt = "%Y-%m-%d %H:%M:%S %z"
self.assertEqual(
[d.strftime(fmt) for d in results],
["2013-05-20 09:25:00 -0500", "2013-05-20 09:31:00 -0500", "2013-01-02 19:05:00 -0500"],
)
def test_dateutil(self):
"""Test reading non-timezone-aware dates without providing the format.
dateutil is required for this one.
"""
try:
import dateutil
except ImportError: # pragma: no cover
self.skipTest("dateutil is not available")
dates = [
"2013-05-20 9:25",
"Thu Sep 25 10:36:26 2003",
"2003 10:36:28 CET 25 Sep Thu",
] # Timezone will be ignored
with intercept_result(StringsToDates, "dates") as results:
self.assertFalse(
execute([("convert|dates|StringsToDates", identifier, [("strings", [("List", repr(dates))])])])
)
self.assertEqual(len(results), 1)
results = results[0]
fmt = "%Y-%m-%d %H:%M:%S %Z %z"
self.assertEqual(
[d.strftime(fmt) for d in results],
["2013-05-20 09:25:00 ", "2003-09-25 10:36:26 ", "2003-09-25 10:36:28 "],
)
def test_strings(self):
from vistrails.tests.utils import execute, intercept_result
from ..common import ExtractColumn
from ..identifiers import identifier
with intercept_result(ExtractColumn, 'value') as results:
self.assertFalse(execute([
('BuildTable', identifier, [
('a', [('List', "['a', '2', 'c']")]),
('b', [('List', "[4, 5, 6]")]),
]),
(self.WRITER_MODULE, identifier, []),
(self.READER_MODULE, identifier, []),
('ExtractColumn', identifier, [
('column_index', [('Integer', '0')]),
('numeric', [('Boolean', 'False')]),
]),
], [
(0, 'value', 1, 'table'),
(1, 'file', 2, 'file'),
(2, 'value', 3, 'table'),
],
add_port_specs=[
(0, 'input', 'a',
'(org.vistrails.vistrails.basic:List)'),
(0, 'input', 'b',
'(org.vistrails.vistrails.basic:List)'),
]))
self.assertEqual(len(results), 1)
self.assertEqual(results[0], ['a', '2', 'c'])
def test_pythonsource(self):
import urllib2
source = ('o = i * 2\n'
"r = \"it's %d!!!\" % o\n"
'go_on = o < 100')
source = urllib2.quote(source)
from vistrails.tests.utils import execute, intercept_result
with intercept_result(While, 'Result') as results:
self.assertFalse(execute([
('PythonSource', 'org.vistrails.vistrails.basic', [
('source', [('String', source)]),
('i', [('Integer', '5')]),
]),
('While', 'org.vistrails.vistrails.control_flow', [
('ConditionPort', [('String', 'go_on')]),
('OutputPort', [('String', 'r')]),
('StateInputPorts', [('List', "['i']")]),
('StateOutputPorts', [('List', "['o']")]),
]),
],
[
(0, 'self', 1, 'FunctionPort'),
],
add_port_specs=[
(0, 'input', 'i',
'org.vistrails.vistrails.basic:Integer'),
(0, 'output', 'o',
'org.vistrails.vistrails.basic:Integer'),
(0, 'output', 'r',
'org.vistrails.vistrails.basic:String'),
(0, 'output', 'go_on',
'org.vistrails.vistrails.basic:Boolean'),
]))
self.assertEqual(results, ["it's 160!!!"])
def test_multiple(self):
src = urllib2.quote("o = i + j")
with intercept_result(Map, "Result") as results:
self.assertFalse(
execute(
[
("PythonSource", "org.vistrails.vistrails.basic", [("source", [("String", src)])]),
(
"Map",
"org.vistrails.vistrails.control_flow",
[
("InputPort", [("List", "['i', 'j']")]),
("OutputPort", [("String", "o")]),
("InputList", [("List", "[(1, 2), (3, 8), (-2, 3)]")]),
],
),
],
[(0, "self", 1, "FunctionPort")],
add_port_specs=[
(0, "input", "i", "org.vistrails.vistrails.basic:Integer"),
(0, "input", "j", "org.vistrails.vistrails.basic:Integer"),
(0, "output", "o", "org.vistrails.vistrails.basic:Integer"),
],
)
)
self.assertEqual(results, [[3, 11, 1]])
def test_tuple(self):
src = urllib2.quote("o = len(i[0]) + i[1]")
with intercept_result(Map, "Result") as results:
self.assertFalse(
execute(
[
("PythonSource", "org.vistrails.vistrails.basic", [("source", [("String", src)])]),
(
"Map",
"org.vistrails.vistrails.control_flow",
[
("InputPort", [("List", "['i']")]),
("OutputPort", [("String", "o")]),
("InputList", [("List", '[("aa", 1), ("", 8), ("a", 4)]')]),
],
),
],
[(0, "self", 1, "FunctionPort")],
add_port_specs=[
(0, "input", "i", "org.vistrails.vistrails.basic:String,org.vistrails.vistrails.basic:Integer"),
(0, "output", "o", "org.vistrails.vistrails.basic:Float"),
],
)
)
self.assertEqual(results, [[3, 8, 5]])
def test_filter(self):
src = urllib2.quote("o = bool(i)")
with intercept_result(Filter, "Result") as results:
self.assertFalse(
execute(
[
("PythonSource", "org.vistrails.vistrails.basic", [("source", [("String", src)])]),
(
"Filter",
"org.vistrails.vistrails.control_flow",
[
("InputPort", [("List", "['i']")]),
("OutputPort", [("String", "o")]),
("InputList", [("List", "[0, 1, 2, 3, '', 'foo', True, False]")]),
],
),
],
[(0, "self", 1, "FunctionPort")],
add_port_specs=[
(0, "input", "i", "org.vistrails.vistrails.basic:Module"),
(0, "output", "o", "org.vistrails.vistrails.basic:Boolean"),
],
)
)
self.assertEqual(results, [[1, 2, 3, "foo", True]])
def test_simple(self):
src = urllib2.quote("o = i + 1")
with intercept_result(Map, "Result") as results:
self.assertFalse(
execute(
[
("PythonSource", "org.vistrails.vistrails.basic", [("source", [("String", src)])]),
(
"Map",
"org.vistrails.vistrails.control_flow",
[
("InputPort", [("List", "['i']")]),
("OutputPort", [("String", "o")]),
("InputList", [("List", "[1, 2, 8, 9.1]")]),
],
),
],
[(0, "self", 1, "FunctionPort")],
add_port_specs=[
(0, "input", "i", "org.vistrails.vistrails.basic:Float"),
(0, "output", "o", "org.vistrails.vistrails.basic:Float"),
],
)
)
self.assertEqual(results, [[2, 3, 9, 10.1]])
def test_npy_numpy(self):
"""Uses NumPyArray to load an array in .NPY format.
"""
from ..identifiers import identifier
from vistrails.tests.utils import execute, intercept_result
with intercept_result(NumPyArray, 'value') as results:
self.assertFalse(execute([
('read|numpy|NumPyArray', identifier, [
('datatype', [('String', 'npy')]),
('file', [('File', self._test_dir + '/random.npy')]),
]),
('PythonSource', 'org.vistrails.vistrails.basic', [
('source', [('String', '')]),
]),
],
[
(0, 'value', 1, 'l'),
],
add_port_specs=[
(1, 'input', 'l',
'org.vistrails.vistrails.basic:List'),
]))
self.assertEqual(len(results), 1)
self.assertEqual(list(results[0]), [1.0, 7.0, 5.0, 3.0, 6.0, 1.0])
def test_dateutil(self):
"""Test reading non-timezone-aware dates without providing the format.
dateutil is required for this one.
"""
try:
import dateutil
except ImportError: # pragma: no cover
self.skipTest("dateutil is not available")
dates = ['2013-05-20 9:25',
'Thu Sep 25 10:36:26 2003',
'2003 10:36:28 CET 25 Sep Thu'] # Timezone will be ignored
with intercept_result(StringsToDates, 'dates') as results:
self.assertFalse(execute([
('convert|dates|StringsToDates', identifier, [
('strings', [('List', repr(dates))]),
]),
]))
self.assertEqual(len(results), 1)
results = results[0]
fmt = '%Y-%m-%d %H:%M:%S %Z %z'
self.assertEqual(
[d.strftime(fmt) for d in results],
['2013-05-20 09:25:00 ',
'2003-09-25 10:36:26 ',
'2003-09-25 10:36:28 '])
def test_tuple(self):
src = urllib2.quote('o = len(i[0]) + i[1]')
with intercept_result(Map, 'Result') as results:
self.assertFalse(
execute(
[
('PythonSource', 'org.vistrails.vistrails.basic', [
('source', [('String', src)]),
]),
('Map', 'org.vistrails.vistrails.control_flow', [
('InputPort', [('List', "['i']")]),
('OutputPort', [('String', 'o')]),
('InputList', [
('List', '[("aa", 1), ("", 8), ("a", 4)]')
]),
]),
], [
(0, 'self', 1, 'FunctionPort'),
],
add_port_specs=[
(0, 'input', 'i',
'org.vistrails.vistrails.basic:String,org.vistrails.vistrails.basic:Integer'
),
(0, 'output', 'o',
'org.vistrails.vistrails.basic:Float'),
]))
self.assertEqual(results, [[3, 8, 5]])
def do_default(self, val):
if val:
src = 'o = 42'
else:
src = ('from vistrails.core.modules.vistrails_module import '
'InvalidOutput\n'
'o = InvalidOutput')
src = urllib2.quote(src)
with intercept_result(Default, 'Result') as results:
self.assertFalse(
execute([
('Default', 'org.vistrails.vistrails.control_flow', [
('Default', [('Integer', '28')]),
]),
('PythonSource', 'org.vistrails.vistrails.basic', [
('source', [('String', src)]),
]),
], [
(1, 'o', 0, 'Input'),
],
add_port_specs=[
(1, 'output', 'o',
'org.vistrails.vistrails.basic:Integer'),
]))
if val:
self.assertEqual(results, [42])
else:
self.assertEqual(results, [28])
def test_multiple(self):
src = urllib2.quote('o = i + j')
with intercept_result(Map, 'Result') as results:
self.assertFalse(
execute([
('PythonSource', 'org.vistrails.vistrails.basic', [
('source', [('String', src)]),
]),
('Map', 'org.vistrails.vistrails.control_flow', [
('InputPort', [('List', "['i', 'j']")]),
('OutputPort', [('String', 'o')]),
('InputList', [('List', '[(1, 2), (3, 8), (-2, 3)]')]),
]),
], [
(0, 'self', 1, 'FunctionPort'),
],
add_port_specs=[
(0, 'input', 'i',
'org.vistrails.vistrails.basic:Integer'),
(0, 'input', 'j',
'org.vistrails.vistrails.basic:Integer'),
(0, 'output', 'o',
'org.vistrails.vistrails.basic:Integer'),
]))
self.assertEqual(results, [[3, 11, 1]])
def do_if(self, val):
with intercept_result(If, 'Result') as results:
interp_dict = execute([
('If', 'org.vistrails.vistrails.control_flow', [
('FalseOutputPorts', [('List', "['value']")]),
('TrueOutputPorts', [('List', "['value']")]),
('Condition', [('Boolean', str(val))]),
]),
('Integer', 'org.vistrails.vistrails.basic', [
('value', [('Integer', '42')]),
]),
('Integer', 'org.vistrails.vistrails.basic', [
('value', [('Integer', '28')]),
]),
], [
(1, 'self', 0, 'TruePort'),
(2, 'self', 0, 'FalsePort'),
],
full_results=True)
self.assertFalse(interp_dict.errors)
if val:
self.assertEqual(results, [42])
else:
self.assertEqual(results, [28])
self.assertEqual(interp_dict.executed, {0: True, 1: val, 2: not val})
def test_filter(self):
src = urllib2.quote('o = bool(i)')
with intercept_result(Filter, 'Result') as results:
self.assertFalse(
execute([
('PythonSource', 'org.vistrails.vistrails.basic', [
('source', [('String', src)]),
]),
('Filter', 'org.vistrails.vistrails.control_flow', [
('InputPort', [('List', "['i']")]),
('OutputPort', [('String', 'o')]),
('InputList', [
('List', "[0, 1, 2, 3, '', 'foo', True, False]")
]),
]),
], [
(0, 'self', 1, 'FunctionPort'),
],
add_port_specs=[
(0, 'input', 'i',
'org.vistrails.vistrails.basic:Module'),
(0, 'output', 'o',
'org.vistrails.vistrails.basic:Boolean'),
]))
self.assertEqual(results, [[1, 2, 3, 'foo', True]])
def test_npy_numpy(self):
"""Uses NumPyArray to load an array in .NPY format.
"""
from ..identifiers import identifier
from vistrails.tests.utils import execute, intercept_result
with intercept_result(NumPyArray, 'value') as results:
self.assertFalse(
execute([
('read|NumPyArray', identifier, [
('datatype', [('String', 'npy')]),
('file', [('File', self._test_dir + '/random.npy')]),
]),
('PythonSource', 'org.vistrails.vistrails.basic', [
('source', [('String', '')]),
]),
], [
(0, 'value', 1, 'l'),
],
add_port_specs=[
(1, 'input', 'l',
'org.vistrails.vistrails.basic:List'),
]))
self.assertEqual(len(results), 1)
self.assertEqual(list(results[0]), [1.0, 7.0, 5.0, 3.0, 6.0, 1.0])
def test_multiple(self):
src = urllib2.quote('o = i + j')
with intercept_result(Map, 'Result') as results:
self.assertFalse(execute([
('PythonSource', 'org.vistrails.vistrails.basic', [
('source', [('String', src)]),
]),
('Map', 'org.vistrails.vistrails.control_flow', [
('InputPort', [('List', "['i', 'j']")]),
('OutputPort', [('String', 'o')]),
('InputList', [('List',
'[(1, 2), (3, 8), (-2, 3)]')]),
]),
],
[
(0, 'self', 1, 'FunctionPort'),
],
add_port_specs=[
(0, 'input', 'i',
'org.vistrails.vistrails.basic:Integer'),
(0, 'input', 'j',
'org.vistrails.vistrails.basic:Integer'),
(0, 'output', 'o',
'org.vistrails.vistrails.basic:Integer'),
]))
self.assertEqual(results, [[3, 11, 1]])
def do_if(self, val):
with intercept_result(If, 'Result') as results:
interp_dict = execute([
('If', 'org.vistrails.vistrails.control_flow', [
('FalseOutputPorts', [('List', "['value']")]),
('TrueOutputPorts', [('List', "['value']")]),
('Condition', [('Boolean', str(val))]),
]),
('Integer', 'org.vistrails.vistrails.basic', [
('value', [('Integer', '42')]),
]),
('Integer', 'org.vistrails.vistrails.basic', [
('value', [('Integer', '28')]),
]),
],
[
(1, 'self', 0, 'TruePort'),
(2, 'self', 0, 'FalsePort'),
],
full_results=True)
self.assertFalse(interp_dict.errors)
if val:
self.assertEqual(results, [42])
else:
self.assertEqual(results, [28])
self.assertEqual(interp_dict.executed, {0: True, 1: val, 2: not val})
def do_aggregate(self, agg_functions):
with intercept_result(AggregateColumn, 'value') as results:
errors = execute([
('WriteFile', 'org.vistrails.vistrails.basic', [
('in_value', [('String', '22;a;T;100\n'
'43;b;F;3\n'
'-7;d;T;41\n'
'500;e;F;21\n'
'20;a;T;1\n'
'43;b;F;23\n'
'21;a;F;41\n')]),
]),
('read|CSVFile', identifier, [
('delimiter', [('String', ';')]),
('header_present', [('Boolean', 'False')]),
('sniff_header', [('Boolean', 'False')]),
]),
('AggregateColumn', identifier, agg_functions),
],
[
(0, 'out_value', 1, 'file'),
(1, 'value', 2, 'table'),
])
self.assertFalse(errors)
self.assertEqual(len(results), 1)
return results[0]
def do_project(self, project_functions, error=None):
with intercept_result(ProjectTable, 'value') as results:
errors = execute([
('BuildTable', identifier, [
('letters', [('List', repr(['a', 'b', 'c', 'd']))]),
('numbers', [('List', repr([1, 2, 3, '4']))]),
('cardinals', [('List', repr(['one', 'two',
'three', 'four']))]),
('ordinals', [('List', repr(['first', 'second',
'third', 'fourth']))])
]),
('ProjectTable', identifier, project_functions),
],
[
(0, 'value', 1, 'table'),
],
add_port_specs=[
(0, 'input', 'letters',
'org.vistrails.vistrails.basic:List'),
(0, 'input', 'numbers',
'org.vistrails.vistrails.basic:List'),
(0, 'input', 'cardinals',
'org.vistrails.vistrails.basic:List'),
(0, 'input', 'ordinals',
'org.vistrails.vistrails.basic:List'),
])
if error is not None:
self.assertEqual([1], errors.keys())
self.assertIn(error, errors[1].message)
return None
else:
self.assertFalse(errors)
self.assertEqual(len(results), 1)
return results[0]
def test_simple(self):
"""Test converting timestamps into matplotlib's format.
"""
try:
import matplotlib
except ImportError: # pragma: no cover
self.skipTest("matplotlib is not available")
from matplotlib.dates import date2num
with intercept_result(TimestampsToMatplotlib, 'dates') as results:
self.assertFalse(
execute([
('convert|dates|TimestampsToMatplotlib', identifier, [
('timestamps', [('List', '[1324842375, 1369842877]')]),
]),
]))
self.assertEqual(len(results), 1)
results = results[0]
self.assertEqual(
list(results),
list(
date2num([
datetime.datetime.utcfromtimestamp(1324842375),
datetime.datetime.utcfromtimestamp(1369842877)
])))
def test_timezone(self):
"""Test reading timezone-aware dates by supplying an offset.
"""
try:
import matplotlib
except ImportError: # pragma: no cover
self.skipTest("matplotlib is not available")
from matplotlib.dates import date2num
dates = ['2013-05-20 9:25', '2013-05-20 09:31', '2013-01-02 18:05']
in_fmt = '%Y-%m-%d %H:%M'
with intercept_result(StringsToMatplotlib, 'dates') as results:
self.assertFalse(
execute([
('convert|dates|StringsToMatplotlib', identifier,
[('strings', [('List', repr(dates))]),
('format', [('String', in_fmt)]),
('timezone', [('String', '-0500')])]),
]))
self.assertEqual(len(results), 1)
results = results[0]
self.assertEqual(
list(results),
list(
date2num([
datetime.datetime(2013, 5, 20, 14, 25, 0),
datetime.datetime(2013, 5, 20, 14, 31, 0),
datetime.datetime(2013, 1, 2, 23, 5, 0)
])))
def test_dateutil(self):
"""Test reading non-timezone-aware dates without providing the format.
dateutil is required for this one.
"""
try:
import dateutil
except ImportError: # pragma: no cover
self.skipTest("dateutil is not available")
dates = [
'2013-05-20 9:25', 'Thu Sep 25 10:36:26 2003',
'2003 10:36:28 CET 25 Sep Thu'
] # Timezone will be ignored
with intercept_result(StringsToDates, 'dates') as results:
self.assertFalse(
execute([
('convert|dates|StringsToDates', identifier, [
('strings', [('List', repr(dates))]),
]),
]))
self.assertEqual(len(results), 1)
results = results[0]
fmt = '%Y-%m-%d %H:%M:%S %Z %z'
self.assertEqual([d.strftime(fmt) for d in results], [
'2013-05-20 09:25:00 ', '2003-09-25 10:36:26 ',
'2003-09-25 10:36:28 '
])
def test_timezone_pytz(self):
"""Test reading timezone-aware dates through pytz.
"""
try:
import pytz
except ImportError: # pragma: no cover
self.skipTest("pytz is not available")
if LooseVersion(pytz.__version__) < PYTZ_MIN_VER: # pragma: no cover
self.skipTest("pytz version is known to cause issues (%s)" %
pytz.__version__)
dates = ['2013-01-20 9:25', '2013-01-20 09:31', '2013-06-02 19:05']
in_fmt = '%Y-%m-%d %H:%M'
with intercept_result(StringsToDates, 'dates') as results:
self.assertFalse(
execute([
('convert|dates|StringsToDates', identifier,
[('strings', [('List', repr(dates))]),
('format', [('String', in_fmt)]),
('timezone', [('String', 'America/New_York')])]),
]))
self.assertEqual(len(results), 1)
results = results[0]
self.assertTrue(all(d.tzinfo is not None for d in results))
fmt = '%Y-%m-%d %H:%M:%S %Z %z'
self.assertEqual([d.strftime(fmt) for d in results], [
'2013-01-20 09:25:00 EST -0500', '2013-01-20 09:31:00 EST -0500',
'2013-06-02 19:05:00 EDT -0400'
])
def do_andor(self, l):
with intercept_result(And, "Result") as and_results:
with intercept_result(Or, "Result") as or_results:
self.assertFalse(
execute(
[
("List", "org.vistrails.vistrails.basic", [("value", [("List", str(l))])]),
("And", "org.vistrails.vistrails.control_flow", []),
("Or", "org.vistrails.vistrails.control_flow", []),
],
[(0, "value", 1, "InputList"), (0, "value", 2, "InputList")],
)
)
self.assertEqual(len(and_results), 1)
self.assertEqual(len(or_results), 1)
return and_results[0], or_results[0]
def test_timestamps(self):
"""Test conversion to datetime objects.
"""
timestamps = [1369041900, 1369042260, 1357153500]
with intercept_result(TimestampsToDates, 'dates') as results:
self.assertFalse(
execute([
('convert|dates|TimestampsToDates', identifier, [
('timestamps', [('List', repr(timestamps))]),
]),
]))
self.assertEqual(len(results), 1)
results = results[0]
self.assertTrue(all(d.tzinfo is utc for d in results))
fmt = '%Y-%m-%d %H:%M:%S %Z %z'
self.assertEqual([d.strftime(fmt) for d in results], [
'2013-05-20 09:25:00 UTC +0000', '2013-05-20 09:31:00 UTC +0000',
'2013-01-02 19:05:00 UTC +0000'
])
try:
import pytz
except ImportError: # pragma: no cover
pass
else:
self.assertEqual([
d.astimezone(pytz.timezone('US/Eastern')).strftime(fmt)
for d in results
], [
'2013-05-20 05:25:00 EDT -0400',
'2013-05-20 05:31:00 EDT -0400',
'2013-01-02 14:05:00 EST -0500'
])
def test_timezone_pytz(self):
"""Test reading timezone-aware dates through pytz.
"""
try:
import pytz
except ImportError: # pragma: no cover
self.skipTest("pytz is not available")
if LooseVersion(pytz.__version__) < PYTZ_MIN_VER: # pragma: no cover
self.skipTest("pytz version is known to cause issues (%s)" %
pytz.__version__)
dates = ['2013-01-20 9:25', '2013-01-20 09:31', '2013-06-02 19:05']
in_fmt = '%Y-%m-%d %H:%M'
with intercept_result(StringsToDates, 'dates') as results:
self.assertFalse(execute([
('convert|dates|StringsToDates', identifier, [
('strings', [('List', repr(dates))]),
('format', [('String', in_fmt)]),
('timezone', [('String', 'America/New_York')])
]),
]))
self.assertEqual(len(results), 1)
results = results[0]
self.assertTrue(all(d.tzinfo is not None for d in results))
fmt = '%Y-%m-%d %H:%M:%S %Z %z'
self.assertEqual(
[d.strftime(fmt) for d in results],
['2013-01-20 09:25:00 EST -0500',
'2013-01-20 09:31:00 EST -0500',
'2013-06-02 19:05:00 EDT -0400'])
def test_reshape(self):
"""Uses Reshape to reshape arrays and lists.
"""
from vistrails.tests.utils import execute, intercept_result
from ..identifiers import identifier
with intercept_result(Reshape, 'value') as results:
self.assertFalse(
execute([
('convert|Reshape', identifier, [
('array', [('List', '[0, 1, 2, 3, 4, 5]')]),
('shape', [('List', '[2, 3]')]),
]),
]))
self.assertFalse(
execute([
('convert|Reshape', identifier, [
('array', [('List', '[[1, 2], [3, 4], [5, 6]]')]),
('shape', [('List', '[6]')]),
]),
]))
self.assertEqual(len(results), 2)
self.assertEqual(results[0].shape, (2, 3))
self.assertTrue((results[0] == [[0, 1, 2], [3, 4, 5]]).all())
self.assertEqual(results[1].shape, (6, ))
self.assertTrue((results[1] == [1, 2, 3, 4, 5, 6]).all())
def test_timestamps(self):
"""Test conversion to datetime objects.
"""
timestamps = [1369041900, 1369042260, 1357153500]
with intercept_result(TimestampsToDates, 'dates') as results:
self.assertFalse(execute([
('convert|dates|TimestampsToDates', identifier, [
('timestamps', [('List', repr(timestamps))]),
]),
]))
self.assertEqual(len(results), 1)
results = results[0]
self.assertTrue(all(d.tzinfo is utc for d in results))
fmt = '%Y-%m-%d %H:%M:%S %Z %z'
self.assertEqual(
[d.strftime(fmt) for d in results],
['2013-05-20 09:25:00 UTC +0000',
'2013-05-20 09:31:00 UTC +0000',
'2013-01-02 19:05:00 UTC +0000'])
try:
import pytz
except ImportError: # pragma: no cover
pass
else:
self.assertEqual(
[d.astimezone(pytz.timezone('US/Eastern')).strftime(fmt)
for d in results],
['2013-05-20 05:25:00 EDT -0400',
'2013-05-20 05:31:00 EDT -0400',
'2013-01-02 14:05:00 EST -0500'])
def do_default(self, val):
if val:
src = 'o = 42'
else:
src = ('from vistrails.core.modules.vistrails_module import '
'InvalidOutput\n'
'o = InvalidOutput')
src = urllib2.quote(src)
with intercept_result(Default, 'Result') as results:
self.assertFalse(execute([
('Default', 'org.vistrails.vistrails.control_flow', [
('Default', [('Integer', '28')]),
]),
('PythonSource', 'org.vistrails.vistrails.basic', [
('source', [('String', src)]),
]),
],
[
(1, 'o', 0, 'Input'),
],
add_port_specs=[
(1, 'output', 'o',
'org.vistrails.vistrails.basic:Integer'),
]))
if val:
self.assertEqual(results, [42])
else:
self.assertEqual(results, [28])
def test_strings(self):
from vistrails.tests.utils import execute, intercept_result
from ..common import ExtractColumn
from ..identifiers import identifier
with intercept_result(ExtractColumn, 'value') as results:
self.assertFalse(execute([
('BuildTable', identifier, [
('a', [('List', "['a', '2', 'c']")]),
('b', [('List', "[4, 5, 6]")]),
]),
(self.WRITER_MODULE, identifier, []),
(self.READER_MODULE, identifier, []),
('ExtractColumn', identifier, [
('column_index', [('Integer', '0')]),
('numeric', [('Boolean', 'False')]),
]),
], [
(0, 'value', 1, 'table'),
(1, 'file', 2, 'file'),
(2, 'value', 3, 'table'),
],
add_port_specs=[
(0, 'input', 'a',
'(org.vistrails.vistrails.basic:List)'),
(0, 'input', 'b',
'(org.vistrails.vistrails.basic:List)'),
]))
self.assertEqual(len(results), 1)
self.assertEqual(results[0], ['a', '2', 'c'])
def do_select(self, select_functions, error=None):
with intercept_result(SelectFromTable, 'value') as results:
errors = execute([
('WriteFile', 'org.vistrails.vistrails.basic', [
('in_value', [('String', '22;a;T;abaab\n'
'43;b;F;aabab\n'
'-7;d;T;abbababb\n'
'500;e;F;aba abacc')]),
]),
('read|CSVFile', identifier, [
('delimiter', [('String', ';')]),
('header_present', [('Boolean', 'False')]),
('sniff_header', [('Boolean', 'False')]),
]),
('SelectFromTable', identifier, select_functions),
],
[
(0, 'out_value', 1, 'file'),
(1, 'value', 2, 'table'),
])
if error is not None:
self.assertEqual([2], errors.keys())
self.assertIn(error, errors[2].message)
return None
else:
self.assertFalse(errors)
self.assertEqual(len(results), 1)
return results[0]
def test_timezone(self):
"""Test reading timezone-aware dates by supplying an offset.
"""
try:
import matplotlib
except ImportError: # pragma: no cover
self.skipTest("matplotlib is not available")
from matplotlib.dates import date2num
dates = ['2013-05-20 9:25', '2013-05-20 09:31', '2013-01-02 18:05']
in_fmt = '%Y-%m-%d %H:%M'
with intercept_result(StringsToMatplotlib, 'dates') as results:
self.assertFalse(execute([
('convert|dates|StringsToMatplotlib', identifier, [
('strings', [('List', repr(dates))]),
('format', [('String', in_fmt)]),
('timezone', [('String', '-0500')])
]),
]))
self.assertEqual(len(results), 1)
results = results[0]
self.assertEqual(list(results), list(date2num([
datetime.datetime(2013, 5, 20, 14, 25, 0),
datetime.datetime(2013, 5, 20, 14, 31, 0),
datetime.datetime(2013, 1, 2, 23, 5, 0)])))
def do_andor(self, l):
with intercept_result(And, 'Result') as and_results:
with intercept_result(Or, 'Result') as or_results:
self.assertFalse(execute([
('List', 'org.vistrails.vistrails.basic', [
('value', [('List', str(l))]),
]),
('And', 'org.vistrails.vistrails.control_flow', []),
('Or', 'org.vistrails.vistrails.control_flow', []),
],
[
(0, 'value', 1, 'InputList'),
(0, 'value', 2, 'InputList'),
]))
self.assertEqual(len(and_results), 1)
self.assertEqual(len(or_results), 1)
return and_results[0], or_results[0]
def test_sum(self):
with intercept_result(Sum, 'Result') as results:
self.assertFalse(execute([
('Sum', 'org.vistrails.vistrails.control_flow', [
('InputList', [('List', "[1, 2, 3, 8, 14.7]")]),
]),
]))
self.assertEqual(results, [28.7])
def do_andor(self, l):
with intercept_result(And, 'Result') as and_results:
with intercept_result(Or, 'Result') as or_results:
self.assertFalse(
execute([
('List', 'org.vistrails.vistrails.basic', [
('value', [('List', str(l))]),
]),
('And', 'org.vistrails.vistrails.control_flow', []),
('Or', 'org.vistrails.vistrails.control_flow', []),
], [
(0, 'value', 1, 'InputList'),
(0, 'value', 2, 'InputList'),
]))
self.assertEqual(len(and_results), 1)
self.assertEqual(len(or_results), 1)
return and_results[0], or_results[0]
def test_sum(self):
with intercept_result(Sum, 'Result') as results:
self.assertFalse(execute([
('Sum', 'org.vistrails.vistrails.control_flow', [
('InputList', [('List', "[1, 2, 3, 8, 14.7]")]),
]),
]))
self.assertEqual(results, [28.7])
def test_sum(self):
with intercept_result(Sum, "Result") as results:
self.assertFalse(
execute(
[("Sum", "org.vistrails.vistrails.control_flow", [("InputList", [("List", "[1, 2, 3, 8, 14.7]")])])]
)
)
self.assertEqual(results, [28.7])
def test_cartesian_combine(self):
with intercept_result(CartesianProduct, 'Result') as results:
self.assertFalse(execute([
('CartesianProduct', 'org.vistrails.vistrails.control_flow', [
('List1', [('List', "[(1, 2)]")]),
('List2', [('List', "[3, 4]")]),
]),
]))
self.assertEqual(results, [[(1, 2, 3), (1, 2, 4)]])
def test_elementwise(self):
with intercept_result(ElementwiseProduct, 'Result') as results:
self.assertFalse(execute([
('ElementwiseProduct', 'org.vistrails.vistrails.control_flow', [
('List1', [('List', "[1, 2, 0]")]),
('List2', [('List', "[4, -3, 7]")]),
]),
]))
self.assertEqual(results, [[4, -6, 0]])
def test_cartesian_combine(self):
with intercept_result(CartesianProduct, 'Result') as results:
self.assertFalse(execute([
('CartesianProduct', 'org.vistrails.vistrails.control_flow', [
('List1', [('List', "[(1, 2)]")]),
('List2', [('List', "[3, 4]")]),
]),
]))
self.assertEqual(results, [[(1, 2, 3), (1, 2, 4)]])
def test_dot(self):
with intercept_result(Dot, 'Result') as results:
self.assertFalse(execute([
('Dot', 'org.vistrails.vistrails.control_flow', [
('List1', [('List', "[1, 2, 0]")]),
('List2', [('List', "[4, -3, 7]")]),
]),
]))
self.assertEqual(results, [-2])
def test_dot(self):
with intercept_result(Dot, 'Result') as results:
self.assertFalse(execute([
('Dot', 'org.vistrails.vistrails.control_flow', [
('List1', [('List', "[1, 2, 0]")]),
('List2', [('List', "[4, -3, 7]")]),
]),
]))
self.assertEqual(results, [-2])
def test_join(self):
"""Test joining tables that have column names.
"""
import numpy
with intercept_result(JoinTables, 'value') as results:
self.assertFalse(execute([
('BuildTable', identifier, [
('id', [('List', repr([1, '2', 4, 5]))]),
('A_name', [('List',
repr(['one', 2, 'four', 'five'])),
]),
]),
('BuildTable', identifier, [
('B_age', [('List',
repr([14, 50, '12', 22])),
]),
('id', [('List', repr(['1', 2, 3, 5]))]),
]),
('JoinTables', identifier, [
('left_column_idx', [('Integer', '0')]),
('right_column_name', [('String', 'id')]),
('right_column_idx', [('Integer', '1')]),
]),
],
[
(0, 'value', 2, 'left_table'),
(1, 'value', 2, 'right_table'),
],
add_port_specs=[
(0, 'input', 'id',
'org.vistrails.vistrails.basic:List'),
(0, 'input', 'A_name',
'org.vistrails.vistrails.basic:List'),
(1, 'input', 'B_age',
'org.vistrails.vistrails.basic:List'),
(1, 'input', 'id',
'org.vistrails.vistrails.basic:List'),
]))
self.assertEqual(len(results), 1)
table, = results
self.assertEqual(table.names, ['left.id', 'A_name',
'B_age', 'right.id'])
self.assertEqual(table.get_column(0, False), [1, '2', 5])
l = table.get_column(0, True)
self.assertIsInstance(l, numpy.ndarray)
self.assertEqual(list(l), [1, 2, 5])
self.assertEqual(table.get_column(3, False), ['1', 2, 5])
l = table.get_column(3, True)
self.assertIsInstance(l, numpy.ndarray)
self.assertEqual(list(l), [1, 2, 5])
self.assertEqual(table.get_column(1, False), ['one', 2, 'five'])
self.assertEqual(list(table.get_column(2, True)), [14, 50, 22])
def test_elementwise_tuples(self):
with intercept_result(ElementwiseProduct, 'Result') as results:
self.assertFalse(execute([
('ElementwiseProduct', 'org.vistrails.vistrails.control_flow', [
('List1', [('List', "[1, 2, 0]")]),
('List2', [('List', "[4, -3, 7]")]),
('NumericalProduct', [('Boolean', "False")]),
]),
]))
self.assertEqual(results, [[(1, 4), (2, -3), (0, 7)]])
def test_cross(self):
with intercept_result(Cross, 'Result') as results:
self.assertFalse(execute([
('Cross', 'org.vistrails.vistrails.control_flow', [
('List1', [('List', "[1, 2, 0]")]),
('List2', [('List', "[4, -3, 7]")]),
]),
]))
# ( | 2 -3 |, | 0 7 |, | 1 4 | )
# | 0 7 | | 1 4 | | 2 -3 |
self.assertEqual(results, [[14, -7, -11]])
def test_xls_header_nonnumeric(self):
"""Uses ExcelSpreadsheet to load data.
"""
with intercept_result(ExtractColumn, 'value') as results:
with intercept_result(ExcelSpreadsheet, 'column_count') as cols:
self.assertFalse(
execute([
('read|ExcelSpreadsheet', identifier, [
('file', [('File', self._test_dir + '/xl.xls')]),
('sheet_name', [('String', 'Feuil1')]),
('header_present', [('Boolean', 'True')])
]),
('ExtractColumn', identifier, [
('column_index', [('Integer', '0')]),
('column_name', [('String', 'data1')]),
('numeric', [('Boolean', 'False')]),
]),
], [
(0, 'value', 1, 'table'),
]))
self.assertEqual(cols, [2])
self.assertEqual(len(results), 1)
self.assertEqual(list(results[0]), ['here', 'is', 'some', 'text'])
def test_xls_numeric(self):
"""Uses ExcelSpreadsheet to load a numeric array.
"""
with intercept_result(ExtractColumn, 'value') as results:
with intercept_result(ExcelSpreadsheet, 'column_count') as cols:
self.assertFalse(
execute([
('read|ExcelSpreadsheet', identifier, [
('file', [('File', self._test_dir + '/xl.xls')]),
('sheet_index', [('Integer', '1')]),
('sheet_name', [('String', 'Feuil2')]),
('header_present', [('Boolean', 'False')])
]),
('ExtractColumn', identifier, [
('column_index', [('Integer', '0')]),
('numeric', [('Boolean', 'True')]),
]),
], [
(0, 'value', 1, 'table'),
]))
self.assertEqual(cols, [1])
self.assertEqual(len(results), 1)
self.assertAlmostEqual_lists(list(results[0]), [1, 2, 2, 3, -7.6])
def test_npy_auto_numpy(self):
"""Uses NumPyArray to load an array in (autodetected) .NPY format.
"""
from ..identifiers import identifier
from vistrails.tests.utils import execute, intercept_result
with intercept_result(NumPyArray, 'value') as results:
self.assertFalse(
execute([
('read|NumPyArray', identifier, [
('file', [('File', self._test_dir + '/random.npy')]),
]),
]))
self.assertEqual(len(results), 1)
self.assertEqual(list(results[0]), [1.0, 7.0, 5.0, 3.0, 6.0, 1.0])