def test_reshape(self): """Uses Reshape to reshape arrays and lists. """ from vistrails.tests.utils import execute, intercept_result from ..identifiers import identifier with intercept_result(Reshape, 'value') as results: self.assertFalse( execute([ ('convert|Reshape', identifier, [ ('array', [('List', '[0, 1, 2, 3, 4, 5]')]), ('shape', [('List', '[2, 3]')]), ]), ])) self.assertFalse( execute([ ('convert|Reshape', identifier, [ ('array', [('List', '[[1, 2], [3, 4], [5, 6]]')]), ('shape', [('List', '[6]')]), ]), ])) self.assertEqual(len(results), 2) self.assertEqual(results[0].shape, (2, 3)) self.assertTrue((results[0] == [[0, 1, 2], [3, 4, 5]]).all()) self.assertEqual(results[1].shape, (6, )) self.assertTrue((results[1] == [1, 2, 3, 4, 5, 6]).all())
def test_pipeline(self): with intercept_results(Iris, 'target', Predict, 'prediction') as (y_true, y_pred): self.assertFalse( execute( [('datasets|Iris', identifier, []), ('preprocessing|StandardScaler', identifier, []), ('feature_selection|SelectKBest', identifier, [ ('k', [('Integer', '2')]) ]), ('classifiers|LinearSVC', identifier, []), ('Pipeline', identifier, []), ('Predict', identifier, [])], [ # feed data to pipeline (0, 'data', 4, 'training_data'), (0, 'target', 4, 'training_target'), # put models in pipeline (1, 'model', 4, 'model1'), (2, 'model', 4, 'model2'), (3, 'model', 4, 'model3'), # predict using pipeline (4, 'model', 5, 'model'), (0, 'data', 5, 'data') ])) y_true, y_pred = np.array(y_true[0]), np.array(y_pred[0]) self.assertEqual(y_true.shape, y_pred.shape) self.assertTrue(np.mean(y_true == y_pred) > .8)
def test_multiple(self): src = urllib2.quote('o = i + j') with intercept_result(Map, 'Result') as results: self.assertFalse(execute([ ('PythonSource', 'org.vistrails.vistrails.basic', [ ('source', [('String', src)]), ]), ('Map', 'org.vistrails.vistrails.control_flow', [ ('InputPort', [('List', "['i', 'j']")]), ('OutputPort', [('String', 'o')]), ('InputList', [('List', '[(1, 2), (3, 8), (-2, 3)]')]), ]), ], [ (0, 'self', 1, 'FunctionPort'), ], add_port_specs=[ (0, 'input', 'i', 'org.vistrails.vistrails.basic:Integer'), (0, 'input', 'j', 'org.vistrails.vistrails.basic:Integer'), (0, 'output', 'o', 'org.vistrails.vistrails.basic:Integer'), ])) self.assertEqual(results, [[3, 11, 1]])
def test_pythonsource(self): import urllib2 source = ('o = i * 2\n' "r = \"it's %d!!!\" % o\n" 'go_on = o < 100') source = urllib2.quote(source) from vistrails.tests.utils import execute, intercept_result with intercept_result(While, 'Result') as results: self.assertFalse(execute([ ('PythonSource', 'org.vistrails.vistrails.basic', [ ('source', [('String', source)]), ('i', [('Integer', '5')]), ]), ('While', 'org.vistrails.vistrails.control_flow', [ ('ConditionPort', [('String', 'go_on')]), ('OutputPort', [('String', 'r')]), ('StateInputPorts', [('List', "['i']")]), ('StateOutputPorts', [('List', "['o']")]), ]), ], [ (0, 'self', 1, 'FunctionPort'), ], add_port_specs=[ (0, 'input', 'i', 'org.vistrails.vistrails.basic:Integer'), (0, 'output', 'o', 'org.vistrails.vistrails.basic:Integer'), (0, 'output', 'r', 'org.vistrails.vistrails.basic:String'), (0, 'output', 'go_on', 'org.vistrails.vistrails.basic:Boolean'), ])) self.assertEqual(results, ["it's 160!!!"])
def test_multiple(self): src = urllib2.quote("o = i + j") with intercept_result(Map, "Result") as results: self.assertFalse( execute( [ ("PythonSource", "org.vistrails.vistrails.basic", [("source", [("String", src)])]), ( "Map", "org.vistrails.vistrails.control_flow", [ ("InputPort", [("List", "['i', 'j']")]), ("OutputPort", [("String", "o")]), ("InputList", [("List", "[(1, 2), (3, 8), (-2, 3)]")]), ], ), ], [(0, "self", 1, "FunctionPort")], add_port_specs=[ (0, "input", "i", "org.vistrails.vistrails.basic:Integer"), (0, "input", "j", "org.vistrails.vistrails.basic:Integer"), (0, "output", "o", "org.vistrails.vistrails.basic:Integer"), ], ) ) self.assertEqual(results, [[3, 11, 1]])
def test_simple(self): src = urllib2.quote("o = i + 1") with intercept_result(Map, "Result") as results: self.assertFalse( execute( [ ("PythonSource", "org.vistrails.vistrails.basic", [("source", [("String", src)])]), ( "Map", "org.vistrails.vistrails.control_flow", [ ("InputPort", [("List", "['i']")]), ("OutputPort", [("String", "o")]), ("InputList", [("List", "[1, 2, 8, 9.1]")]), ], ), ], [(0, "self", 1, "FunctionPort")], add_port_specs=[ (0, "input", "i", "org.vistrails.vistrails.basic:Float"), (0, "output", "o", "org.vistrails.vistrails.basic:Float"), ], ) ) self.assertEqual(results, [[2, 3, 9, 10.1]])
def test_tuple(self): src = urllib2.quote("o = len(i[0]) + i[1]") with intercept_result(Map, "Result") as results: self.assertFalse( execute( [ ("PythonSource", "org.vistrails.vistrails.basic", [("source", [("String", src)])]), ( "Map", "org.vistrails.vistrails.control_flow", [ ("InputPort", [("List", "['i']")]), ("OutputPort", [("String", "o")]), ("InputList", [("List", '[("aa", 1), ("", 8), ("a", 4)]')]), ], ), ], [(0, "self", 1, "FunctionPort")], add_port_specs=[ (0, "input", "i", "org.vistrails.vistrails.basic:String,org.vistrails.vistrails.basic:Integer"), (0, "output", "o", "org.vistrails.vistrails.basic:Float"), ], ) ) self.assertEqual(results, [[3, 8, 5]])
def test_pipeline(self): with intercept_results(Iris, 'target', Predict, 'prediction') as (y_true, y_pred): self.assertFalse(execute( [ ('datasets|Iris', identifier, []), ('preprocessing|StandardScaler', identifier, []), ('feature_selection|SelectKBest', identifier, [('k', [('Integer', '2')])]), ('classifiers|LinearSVC', identifier, []), ('Pipeline', identifier, []), ('Predict', identifier, []) ], [ # feed data to pipeline (0, 'data', 4, 'training_data'), (0, 'target', 4, 'training_target'), # put models in pipeline (1, 'model', 4, 'model1'), (2, 'model', 4, 'model2'), (3, 'model', 4, 'model3'), # predict using pipeline (4, 'model', 5, 'model'), (0, 'data', 5, 'data') ] )) y_true, y_pred = np.array(y_true[0]), np.array(y_pred[0]) self.assertEqual(y_true.shape, y_pred.shape) self.assertTrue(np.mean(y_true == y_pred) > .8)
def test_train_test_split(self): # check that we can split the iris dataset with intercept_results(TrainTestSplit, 'training_data', TrainTestSplit, 'training_target', TrainTestSplit, 'test_data', TrainTestSplit, 'test_target') as results: X_train, y_train, X_test, y_test = results self.assertFalse(execute( [ ('datasets|Iris', identifier, []), ('cross-validation|TrainTestSplit', identifier, [('test_size', [('Integer', '50')])]) ], [ (0, 'data', 1, 'data'), (0, 'target', 1, 'target') ] )) X_train = np.vstack(X_train) X_test = np.vstack(X_test) y_train = np.hstack(y_train) y_test = np.hstack(y_test) self.assertEqual(X_train.shape, (100, 4)) self.assertEqual(X_test.shape, (50, 4)) self.assertEqual(y_train.shape, (100,)) self.assertEqual(y_test.shape, (50,))
def test_tuple(self): src = urllib2.quote('o = len(i[0]) + i[1]') with intercept_result(Map, 'Result') as results: self.assertFalse( execute( [ ('PythonSource', 'org.vistrails.vistrails.basic', [ ('source', [('String', src)]), ]), ('Map', 'org.vistrails.vistrails.control_flow', [ ('InputPort', [('List', "['i']")]), ('OutputPort', [('String', 'o')]), ('InputList', [ ('List', '[("aa", 1), ("", 8), ("a", 4)]') ]), ]), ], [ (0, 'self', 1, 'FunctionPort'), ], add_port_specs=[ (0, 'input', 'i', 'org.vistrails.vistrails.basic:String,org.vistrails.vistrails.basic:Integer' ), (0, 'output', 'o', 'org.vistrails.vistrails.basic:Float'), ])) self.assertEqual(results, [[3, 8, 5]])
def test_csv_numeric(self): """Uses CSVFile and ExtractColumn to load a numeric array. """ with intercept_result(ExtractColumn, 'value') as results: with intercept_result(CSVFile, 'column_count') as columns: self.assertFalse(execute([ ('read|CSVFile', identifier, [ ('file', [('File', self._test_dir + '/test.csv')]), ]), ('ExtractColumn', identifier, [ ('column_index', [('Integer', '1')]), ('column_name', [('String', 'col 2')]), ('numeric', [('Boolean', 'True')]), ]), ('PythonSource', 'org.vistrails.vistrails.basic', [ ('source', [('String', '')]), ]), ], [ (0, 'value', 1, 'table'), (1, 'value', 2, 'l'), ], add_port_specs=[ (2, 'input', 'l', 'org.vistrails.vistrails.basic:List'), ])) # Here we use a PythonSource just to check that a numpy array # can be passed on a List port self.assertEqual(columns, [3]) self.assertEqual(len(results), 1) self.assertEqual(list(results[0]), [2.0, 3.0, 14.5])
def do_default(self, val): if val: src = 'o = 42' else: src = ('from vistrails.core.modules.vistrails_module import ' 'InvalidOutput\n' 'o = InvalidOutput') src = urllib2.quote(src) with intercept_result(Default, 'Result') as results: self.assertFalse(execute([ ('Default', 'org.vistrails.vistrails.control_flow', [ ('Default', [('Integer', '28')]), ]), ('PythonSource', 'org.vistrails.vistrails.basic', [ ('source', [('String', src)]), ]), ], [ (1, 'o', 0, 'Input'), ], add_port_specs=[ (1, 'output', 'o', 'org.vistrails.vistrails.basic:Integer'), ])) if val: self.assertEqual(results, [42]) else: self.assertEqual(results, [28])
def do_if(self, val): with intercept_result(If, 'Result') as results: interp_dict = execute([ ('If', 'org.vistrails.vistrails.control_flow', [ ('FalseOutputPorts', [('List', "['value']")]), ('TrueOutputPorts', [('List', "['value']")]), ('Condition', [('Boolean', str(val))]), ]), ('Integer', 'org.vistrails.vistrails.basic', [ ('value', [('Integer', '42')]), ]), ('Integer', 'org.vistrails.vistrails.basic', [ ('value', [('Integer', '28')]), ]), ], [ (1, 'self', 0, 'TruePort'), (2, 'self', 0, 'FalsePort'), ], full_results=True) self.assertFalse(interp_dict.errors) if val: self.assertEqual(results, [42]) else: self.assertEqual(results, [28]) self.assertEqual(interp_dict.executed, {0: True, 1: val, 2: not val})
def test_xls_header_nonnumeric(self): """Uses ExcelSpreadsheet to load data. """ with intercept_result(ExtractColumn, "value") as results: with intercept_result(ExcelSpreadsheet, "column_count") as cols: self.assertFalse( execute( [ ( "read|ExcelSpreadsheet", identifier, [ ("file", [("File", self._test_dir + "/xl.xls")]), ("sheet_name", [("String", "Feuil1")]), ("header_present", [("Boolean", "True")]), ], ), ( "ExtractColumn", identifier, [ ("column_index", [("Integer", "0")]), ("column_name", [("String", "data1")]), ("numeric", [("Boolean", "False")]), ], ), ], [(0, "self", 1, "table")], ) ) self.assertEqual(cols, [2]) self.assertEqual(len(results), 1) self.assertEqual(list(results[0]), ["here", "is", "some", "text"])
def test_xls_header_numeric(self): """Uses ExcelSpreadsheet to load a numeric array. """ with intercept_result(ExtractColumn, "value") as results: with intercept_result(ExcelSpreadsheet, "column_count") as cols: self.assertFalse( execute( [ ( "read|ExcelSpreadsheet", identifier, [ ("file", [("File", self._test_dir + "/xl.xls")]), # Will default to first sheet ("header_present", [("Boolean", "True")]), ], ), ( "ExtractColumn", identifier, [("column_name", [("String", "data2")]), ("numeric", [("Boolean", "True")])], ), ], [(0, "self", 1, "table")], ) ) self.assertEqual(cols, [2]) self.assertEqual(len(results), 1) self.assertAlmostEqual_lists(list(results[0]), [1, -2.8, 3.4, 3.3])
def test_xls_numeric(self): """Uses ExcelSpreadsheet to load a numeric array. """ with intercept_result(ExtractColumn, "value") as results: with intercept_result(ExcelSpreadsheet, "column_count") as cols: self.assertFalse( execute( [ ( "read|ExcelSpreadsheet", identifier, [ ("file", [("File", self._test_dir + "/xl.xls")]), ("sheet_index", [("Integer", "1")]), ("sheet_name", [("String", "Feuil2")]), ("header_present", [("Boolean", "False")]), ], ), ( "ExtractColumn", identifier, [("column_index", [("Integer", "0")]), ("numeric", [("Boolean", "True")])], ), ], [(0, "self", 1, "table")], ) ) self.assertEqual(cols, [1]) self.assertEqual(len(results), 1) self.assertAlmostEqual_lists(list(results[0]), [1, 2, 2, 3, -7.6])
def test_simple(self): """Test converting timestamps into matplotlib's format. """ try: import matplotlib except ImportError: # pragma: no cover self.skipTest("matplotlib is not available") from matplotlib.dates import date2num with intercept_result(TimestampsToMatplotlib, "dates") as results: self.assertFalse( execute( [ ( "convert|dates|TimestampsToMatplotlib", identifier, [("timestamps", [("List", "[1324842375, 1369842877]")])], ) ] ) ) self.assertEqual(len(results), 1) results = results[0] self.assertEqual( list(results), list( date2num( [datetime.datetime.utcfromtimestamp(1324842375), datetime.datetime.utcfromtimestamp(1369842877)] ) ), )
def test_timezone(self): """Test reading timezone-aware dates by supplying an offset. """ dates = ["2013-05-20 9:25", "2013-05-20 09:31", "2013-01-02 19:05"] in_fmt = "%Y-%m-%d %H:%M" with intercept_result(StringsToDates, "dates") as results: self.assertFalse( execute( [ ( "convert|dates|StringsToDates", identifier, [ ("strings", [("List", repr(dates))]), ("format", [("String", in_fmt)]), ("timezone", [("String", "-0500")]), ], ) ] ) ) self.assertEqual(len(results), 1) results = results[0] self.assertTrue(all(d.tzinfo is not None for d in results)) fmt = "%Y-%m-%d %H:%M:%S %z" self.assertEqual( [d.strftime(fmt) for d in results], ["2013-05-20 09:25:00 -0500", "2013-05-20 09:31:00 -0500", "2013-01-02 19:05:00 -0500"], )
def test_filter(self): src = urllib2.quote("o = bool(i)") with intercept_result(Filter, "Result") as results: self.assertFalse( execute( [ ("PythonSource", "org.vistrails.vistrails.basic", [("source", [("String", src)])]), ( "Filter", "org.vistrails.vistrails.control_flow", [ ("InputPort", [("List", "['i']")]), ("OutputPort", [("String", "o")]), ("InputList", [("List", "[0, 1, 2, 3, '', 'foo', True, False]")]), ], ), ], [(0, "self", 1, "FunctionPort")], add_port_specs=[ (0, "input", "i", "org.vistrails.vistrails.basic:Module"), (0, "output", "o", "org.vistrails.vistrails.basic:Boolean"), ], ) ) self.assertEqual(results, [[1, 2, 3, "foo", True]])
def do_default(self, val): if val: src = 'o = 42' else: src = ('from vistrails.core.modules.vistrails_module import ' 'InvalidOutput\n' 'o = InvalidOutput') src = urllib2.quote(src) with intercept_result(Default, 'Result') as results: self.assertFalse( execute([ ('Default', 'org.vistrails.vistrails.control_flow', [ ('Default', [('Integer', '28')]), ]), ('PythonSource', 'org.vistrails.vistrails.basic', [ ('source', [('String', src)]), ]), ], [ (1, 'o', 0, 'Input'), ], add_port_specs=[ (1, 'output', 'o', 'org.vistrails.vistrails.basic:Integer'), ])) if val: self.assertEqual(results, [42]) else: self.assertEqual(results, [28])
def do_select(self, select_functions, error=None): with intercept_result(SelectFromTable, 'value') as results: errors = execute([ ('WriteFile', 'org.vistrails.vistrails.basic', [ ('in_value', [('String', '22;a;T;abaab\n' '43;b;F;aabab\n' '-7;d;T;abbababb\n' '500;e;F;aba abacc')]), ]), ('read|CSVFile', identifier, [ ('delimiter', [('String', ';')]), ('header_present', [('Boolean', 'False')]), ('sniff_header', [('Boolean', 'False')]), ]), ('SelectFromTable', identifier, select_functions), ], [ (0, 'out_value', 1, 'file'), (1, 'value', 2, 'table'), ]) if error is not None: self.assertEqual([2], errors.keys()) self.assertIn(error, errors[2].message) return None else: self.assertFalse(errors) self.assertEqual(len(results), 1) return results[0]
def do_aggregate(self, agg_functions): with intercept_result(AggregateColumn, 'value') as results: errors = execute([ ('WriteFile', 'org.vistrails.vistrails.basic', [ ('in_value', [('String', '22;a;T;100\n' '43;b;F;3\n' '-7;d;T;41\n' '500;e;F;21\n' '20;a;T;1\n' '43;b;F;23\n' '21;a;F;41\n')]), ]), ('read|CSVFile', identifier, [ ('delimiter', [('String', ';')]), ('header_present', [('Boolean', 'False')]), ('sniff_header', [('Boolean', 'False')]), ]), ('AggregateColumn', identifier, agg_functions), ], [ (0, 'out_value', 1, 'file'), (1, 'value', 2, 'table'), ]) self.assertFalse(errors) self.assertEqual(len(results), 1) return results[0]
def test_dateutil(self): """Test reading non-timezone-aware dates without providing the format. dateutil is required for this one. """ try: import dateutil except ImportError: # pragma: no cover self.skipTest("dateutil is not available") dates = ['2013-05-20 9:25', 'Thu Sep 25 10:36:26 2003', '2003 10:36:28 CET 25 Sep Thu'] # Timezone will be ignored with intercept_result(StringsToDates, 'dates') as results: self.assertFalse(execute([ ('convert|dates|StringsToDates', identifier, [ ('strings', [('List', repr(dates))]), ]), ])) self.assertEqual(len(results), 1) results = results[0] fmt = '%Y-%m-%d %H:%M:%S %Z %z' self.assertEqual( [d.strftime(fmt) for d in results], ['2013-05-20 09:25:00 ', '2003-09-25 10:36:26 ', '2003-09-25 10:36:28 '])
def test_strings(self): from vistrails.tests.utils import execute, intercept_result from ..common import ExtractColumn from ..identifiers import identifier with intercept_result(ExtractColumn, 'value') as results: self.assertFalse(execute([ ('BuildTable', identifier, [ ('a', [('List', "['a', '2', 'c']")]), ('b', [('List', "[4, 5, 6]")]), ]), (self.WRITER_MODULE, identifier, []), (self.READER_MODULE, identifier, []), ('ExtractColumn', identifier, [ ('column_index', [('Integer', '0')]), ('numeric', [('Boolean', 'False')]), ]), ], [ (0, 'value', 1, 'table'), (1, 'file', 2, 'file'), (2, 'value', 3, 'table'), ], add_port_specs=[ (0, 'input', 'a', '(org.vistrails.vistrails.basic:List)'), (0, 'input', 'b', '(org.vistrails.vistrails.basic:List)'), ])) self.assertEqual(len(results), 1) self.assertEqual(results[0], ['a', '2', 'c'])
def test_npy_numpy(self): """Uses NumPyArray to load an array in .NPY format. """ from ..identifiers import identifier from vistrails.tests.utils import execute, intercept_result with intercept_result(NumPyArray, 'value') as results: self.assertFalse( execute([ ('read|NumPyArray', identifier, [ ('datatype', [('String', 'npy')]), ('file', [('File', self._test_dir + '/random.npy')]), ]), ('PythonSource', 'org.vistrails.vistrails.basic', [ ('source', [('String', '')]), ]), ], [ (0, 'value', 1, 'l'), ], add_port_specs=[ (1, 'input', 'l', 'org.vistrails.vistrails.basic:List'), ])) self.assertEqual(len(results), 1) self.assertEqual(list(results[0]), [1.0, 7.0, 5.0, 3.0, 6.0, 1.0])
def test_csv_numeric(self): """Uses CSVFile and ExtractColumn to load a numeric array. """ with intercept_result(ExtractColumn, 'value') as results: with intercept_result(CSVFile, 'column_count') as columns: self.assertFalse( execute([ ('read|CSVFile', identifier, [ ('file', [('File', self._test_dir + '/test.csv')]), ]), ('ExtractColumn', identifier, [ ('column_index', [('Integer', '1')]), ('column_name', [('String', 'col 2')]), ('numeric', [('Boolean', 'True')]), ]), ('PythonSource', 'org.vistrails.vistrails.basic', [ ('source', [('String', '')]), ]), ], [ (0, 'value', 1, 'table'), (1, 'value', 2, 'l'), ], add_port_specs=[ (2, 'input', 'l', 'org.vistrails.vistrails.basic:List'), ])) # Here we use a PythonSource just to check that a numpy array # can be passed on a List port self.assertEqual(columns, [3]) self.assertEqual(len(results), 1) self.assertEqual(list(results[0]), [2.0, 3.0, 14.5])
def test_timestamps(self): """Test conversion to datetime objects. """ timestamps = [1369041900, 1369042260, 1357153500] with intercept_result(TimestampsToDates, 'dates') as results: self.assertFalse(execute([ ('convert|dates|TimestampsToDates', identifier, [ ('timestamps', [('List', repr(timestamps))]), ]), ])) self.assertEqual(len(results), 1) results = results[0] self.assertTrue(all(d.tzinfo is utc for d in results)) fmt = '%Y-%m-%d %H:%M:%S %Z %z' self.assertEqual( [d.strftime(fmt) for d in results], ['2013-05-20 09:25:00 UTC +0000', '2013-05-20 09:31:00 UTC +0000', '2013-01-02 19:05:00 UTC +0000']) try: import pytz except ImportError: # pragma: no cover pass else: self.assertEqual( [d.astimezone(pytz.timezone('US/Eastern')).strftime(fmt) for d in results], ['2013-05-20 05:25:00 EDT -0400', '2013-05-20 05:31:00 EDT -0400', '2013-01-02 14:05:00 EST -0500'])
def test_npy_numpy(self): """Uses NumPyArray to load an array in .NPY format. """ from ..identifiers import identifier from vistrails.tests.utils import execute, intercept_result with intercept_result(NumPyArray, 'value') as results: self.assertFalse(execute([ ('read|numpy|NumPyArray', identifier, [ ('datatype', [('String', 'npy')]), ('file', [('File', self._test_dir + '/random.npy')]), ]), ('PythonSource', 'org.vistrails.vistrails.basic', [ ('source', [('String', '')]), ]), ], [ (0, 'value', 1, 'l'), ], add_port_specs=[ (1, 'input', 'l', 'org.vistrails.vistrails.basic:List'), ])) self.assertEqual(len(results), 1) self.assertEqual(list(results[0]), [1.0, 7.0, 5.0, 3.0, 6.0, 1.0])
def test_dateutil(self): """Test reading non-timezone-aware dates without providing the format. dateutil is required for this one. """ try: import dateutil except ImportError: # pragma: no cover self.skipTest("dateutil is not available") dates = [ "2013-05-20 9:25", "Thu Sep 25 10:36:26 2003", "2003 10:36:28 CET 25 Sep Thu", ] # Timezone will be ignored with intercept_result(StringsToDates, "dates") as results: self.assertFalse( execute([("convert|dates|StringsToDates", identifier, [("strings", [("List", repr(dates))])])]) ) self.assertEqual(len(results), 1) results = results[0] fmt = "%Y-%m-%d %H:%M:%S %Z %z" self.assertEqual( [d.strftime(fmt) for d in results], ["2013-05-20 09:25:00 ", "2003-09-25 10:36:26 ", "2003-09-25 10:36:28 "], )
def do_project(self, project_functions, error=None): with intercept_result(ProjectTable, 'value') as results: errors = execute([ ('BuildTable', identifier, [ ('letters', [('List', repr(['a', 'b', 'c', 'd']))]), ('numbers', [('List', repr([1, 2, 3, '4']))]), ('cardinals', [('List', repr(['one', 'two', 'three', 'four']))]), ('ordinals', [('List', repr(['first', 'second', 'third', 'fourth']))]) ]), ('ProjectTable', identifier, project_functions), ], [ (0, 'value', 1, 'table'), ], add_port_specs=[ (0, 'input', 'letters', 'org.vistrails.vistrails.basic:List'), (0, 'input', 'numbers', 'org.vistrails.vistrails.basic:List'), (0, 'input', 'cardinals', 'org.vistrails.vistrails.basic:List'), (0, 'input', 'ordinals', 'org.vistrails.vistrails.basic:List'), ]) if error is not None: self.assertEqual([1], errors.keys()) self.assertIn(error, errors[1].message) return None else: self.assertFalse(errors) self.assertEqual(len(results), 1) return results[0]
def test_timezone_pytz(self): """Test reading timezone-aware dates through pytz. """ try: import pytz except ImportError: # pragma: no cover self.skipTest("pytz is not available") if LooseVersion(pytz.__version__) < PYTZ_MIN_VER: # pragma: no cover self.skipTest("pytz version is known to cause issues (%s)" % pytz.__version__) dates = ['2013-01-20 9:25', '2013-01-20 09:31', '2013-06-02 19:05'] in_fmt = '%Y-%m-%d %H:%M' with intercept_result(StringsToDates, 'dates') as results: self.assertFalse(execute([ ('convert|dates|StringsToDates', identifier, [ ('strings', [('List', repr(dates))]), ('format', [('String', in_fmt)]), ('timezone', [('String', 'America/New_York')]) ]), ])) self.assertEqual(len(results), 1) results = results[0] self.assertTrue(all(d.tzinfo is not None for d in results)) fmt = '%Y-%m-%d %H:%M:%S %Z %z' self.assertEqual( [d.strftime(fmt) for d in results], ['2013-01-20 09:25:00 EST -0500', '2013-01-20 09:31:00 EST -0500', '2013-06-02 19:05:00 EDT -0400'])
def testIncorrectURL_2(self): from vistrails.tests.utils import execute self.assertTrue(execute([ ('DownloadFile', identifier, [ ('url', [('String', 'http://neitherodesthisohrly')]), ]), ]))
def test_timezone(self): """Test reading timezone-aware dates by supplying an offset. """ try: import matplotlib except ImportError: # pragma: no cover self.skipTest("matplotlib is not available") from matplotlib.dates import date2num dates = ['2013-05-20 9:25', '2013-05-20 09:31', '2013-01-02 18:05'] in_fmt = '%Y-%m-%d %H:%M' with intercept_result(StringsToMatplotlib, 'dates') as results: self.assertFalse(execute([ ('convert|dates|StringsToMatplotlib', identifier, [ ('strings', [('List', repr(dates))]), ('format', [('String', in_fmt)]), ('timezone', [('String', '-0500')]) ]), ])) self.assertEqual(len(results), 1) results = results[0] self.assertEqual(list(results), list(date2num([ datetime.datetime(2013, 5, 20, 14, 25, 0), datetime.datetime(2013, 5, 20, 14, 31, 0), datetime.datetime(2013, 1, 2, 23, 5, 0)])))
def test_timezone(self): """Test reading timezone-aware dates by supplying an offset. """ try: import matplotlib except ImportError: # pragma: no cover self.skipTest("matplotlib is not available") from matplotlib.dates import date2num dates = ['2013-05-20 9:25', '2013-05-20 09:31', '2013-01-02 18:05'] in_fmt = '%Y-%m-%d %H:%M' with intercept_result(StringsToMatplotlib, 'dates') as results: self.assertFalse( execute([ ('convert|dates|StringsToMatplotlib', identifier, [('strings', [('List', repr(dates))]), ('format', [('String', in_fmt)]), ('timezone', [('String', '-0500')])]), ])) self.assertEqual(len(results), 1) results = results[0] self.assertEqual( list(results), list( date2num([ datetime.datetime(2013, 5, 20, 14, 25, 0), datetime.datetime(2013, 5, 20, 14, 31, 0), datetime.datetime(2013, 1, 2, 23, 5, 0) ])))
def testIncorrectURL(self): from vistrails.tests.utils import execute self.assertTrue(execute([ ('DownloadFile', identifier, [ ('url', [('String', 'http://idbetthisdoesnotexistohrly')]), ]), ]))
def test_simple(self): """Test converting timestamps into matplotlib's format. """ try: import matplotlib except ImportError: # pragma: no cover self.skipTest("matplotlib is not available") from matplotlib.dates import date2num with intercept_result(TimestampsToMatplotlib, 'dates') as results: self.assertFalse( execute([ ('convert|dates|TimestampsToMatplotlib', identifier, [ ('timestamps', [('List', '[1324842375, 1369842877]')]), ]), ])) self.assertEqual(len(results), 1) results = results[0] self.assertEqual( list(results), list( date2num([ datetime.datetime.utcfromtimestamp(1324842375), datetime.datetime.utcfromtimestamp(1369842877) ])))
def test_timezone_pytz(self): """Test reading timezone-aware dates through pytz. """ try: import pytz except ImportError: # pragma: no cover self.skipTest("pytz is not available") if LooseVersion(pytz.__version__) < PYTZ_MIN_VER: # pragma: no cover self.skipTest("pytz version is known to cause issues (%s)" % pytz.__version__) dates = ['2013-01-20 9:25', '2013-01-20 09:31', '2013-06-02 19:05'] in_fmt = '%Y-%m-%d %H:%M' with intercept_result(StringsToDates, 'dates') as results: self.assertFalse( execute([ ('convert|dates|StringsToDates', identifier, [('strings', [('List', repr(dates))]), ('format', [('String', in_fmt)]), ('timezone', [('String', 'America/New_York')])]), ])) self.assertEqual(len(results), 1) results = results[0] self.assertTrue(all(d.tzinfo is not None for d in results)) fmt = '%Y-%m-%d %H:%M:%S %Z %z' self.assertEqual([d.strftime(fmt) for d in results], [ '2013-01-20 09:25:00 EST -0500', '2013-01-20 09:31:00 EST -0500', '2013-06-02 19:05:00 EDT -0400' ])
def test_dateutil(self): """Test reading non-timezone-aware dates without providing the format. dateutil is required for this one. """ try: import dateutil except ImportError: # pragma: no cover self.skipTest("dateutil is not available") dates = [ '2013-05-20 9:25', 'Thu Sep 25 10:36:26 2003', '2003 10:36:28 CET 25 Sep Thu' ] # Timezone will be ignored with intercept_result(StringsToDates, 'dates') as results: self.assertFalse( execute([ ('convert|dates|StringsToDates', identifier, [ ('strings', [('List', repr(dates))]), ]), ])) self.assertEqual(len(results), 1) results = results[0] fmt = '%Y-%m-%d %H:%M:%S %Z %z' self.assertEqual([d.strftime(fmt) for d in results], [ '2013-05-20 09:25:00 ', '2003-09-25 10:36:26 ', '2003-09-25 10:36:28 ' ])
def test_timestamps(self): """Test conversion to datetime objects. """ timestamps = [1369041900, 1369042260, 1357153500] with intercept_result(TimestampsToDates, 'dates') as results: self.assertFalse( execute([ ('convert|dates|TimestampsToDates', identifier, [ ('timestamps', [('List', repr(timestamps))]), ]), ])) self.assertEqual(len(results), 1) results = results[0] self.assertTrue(all(d.tzinfo is utc for d in results)) fmt = '%Y-%m-%d %H:%M:%S %Z %z' self.assertEqual([d.strftime(fmt) for d in results], [ '2013-05-20 09:25:00 UTC +0000', '2013-05-20 09:31:00 UTC +0000', '2013-01-02 19:05:00 UTC +0000' ]) try: import pytz except ImportError: # pragma: no cover pass else: self.assertEqual([ d.astimezone(pytz.timezone('US/Eastern')).strftime(fmt) for d in results ], [ '2013-05-20 05:25:00 EDT -0400', '2013-05-20 05:31:00 EDT -0400', '2013-01-02 14:05:00 EST -0500' ])
def test_sum(self): with intercept_result(Sum, 'Result') as results: self.assertFalse(execute([ ('Sum', 'org.vistrails.vistrails.control_flow', [ ('InputList', [('List', "[1, 2, 3, 8, 14.7]")]), ]), ])) self.assertEqual(results, [28.7])
def test_sum(self): with intercept_result(Sum, "Result") as results: self.assertFalse( execute( [("Sum", "org.vistrails.vistrails.control_flow", [("InputList", [("List", "[1, 2, 3, 8, 14.7]")])])] ) ) self.assertEqual(results, [28.7])
def test_iris(self): # check that the iris dataset can be loaded with intercept_results(Iris, 'data', Iris, 'target') as (data, target): self.assertFalse(execute([('datasets|Iris', identifier, [])])) data = np.vstack(data) target = np.hstack(target) self.assertEqual(data.shape, (150, 4)) self.assertEqual(target.shape, (150, ))
def test_elementwise(self): with intercept_result(ElementwiseProduct, 'Result') as results: self.assertFalse(execute([ ('ElementwiseProduct', 'org.vistrails.vistrails.control_flow', [ ('List1', [('List', "[1, 2, 0]")]), ('List2', [('List', "[4, -3, 7]")]), ]), ])) self.assertEqual(results, [[4, -6, 0]])
def test_dot(self): with intercept_result(Dot, 'Result') as results: self.assertFalse(execute([ ('Dot', 'org.vistrails.vistrails.control_flow', [ ('List1', [('List', "[1, 2, 0]")]), ('List2', [('List', "[4, -3, 7]")]), ]), ])) self.assertEqual(results, [-2])
def test_digits(self): # check that the digits dataset can be loaded with intercept_results(Digits, 'data', Digits, 'target') as (data, target): self.assertFalse(execute([('datasets|Digits', identifier, [])])) data = np.vstack(data) target = np.hstack(target) self.assertEqual(data.shape, (1797, 64)) self.assertEqual(target.shape, (1797, ))
def test_cartesian_combine(self): with intercept_result(CartesianProduct, 'Result') as results: self.assertFalse(execute([ ('CartesianProduct', 'org.vistrails.vistrails.control_flow', [ ('List1', [('List', "[(1, 2)]")]), ('List2', [('List', "[3, 4]")]), ]), ])) self.assertEqual(results, [[(1, 2, 3), (1, 2, 4)]])
def test_join(self): """Test joining tables that have column names. """ import numpy with intercept_result(JoinTables, 'value') as results: self.assertFalse(execute([ ('BuildTable', identifier, [ ('id', [('List', repr([1, '2', 4, 5]))]), ('A_name', [('List', repr(['one', 2, 'four', 'five'])), ]), ]), ('BuildTable', identifier, [ ('B_age', [('List', repr([14, 50, '12', 22])), ]), ('id', [('List', repr(['1', 2, 3, 5]))]), ]), ('JoinTables', identifier, [ ('left_column_idx', [('Integer', '0')]), ('right_column_name', [('String', 'id')]), ('right_column_idx', [('Integer', '1')]), ]), ], [ (0, 'value', 2, 'left_table'), (1, 'value', 2, 'right_table'), ], add_port_specs=[ (0, 'input', 'id', 'org.vistrails.vistrails.basic:List'), (0, 'input', 'A_name', 'org.vistrails.vistrails.basic:List'), (1, 'input', 'B_age', 'org.vistrails.vistrails.basic:List'), (1, 'input', 'id', 'org.vistrails.vistrails.basic:List'), ])) self.assertEqual(len(results), 1) table, = results self.assertEqual(table.names, ['left.id', 'A_name', 'B_age', 'right.id']) self.assertEqual(table.get_column(0, False), [1, '2', 5]) l = table.get_column(0, True) self.assertIsInstance(l, numpy.ndarray) self.assertEqual(list(l), [1, 2, 5]) self.assertEqual(table.get_column(3, False), ['1', 2, 5]) l = table.get_column(3, True) self.assertIsInstance(l, numpy.ndarray) self.assertEqual(list(l), [1, 2, 5]) self.assertEqual(table.get_column(1, False), ['one', 2, 'five']) self.assertEqual(list(table.get_column(2, True)), [14, 50, 22])
def test_elementwise_tuples(self): with intercept_result(ElementwiseProduct, 'Result') as results: self.assertFalse(execute([ ('ElementwiseProduct', 'org.vistrails.vistrails.control_flow', [ ('List1', [('List', "[1, 2, 0]")]), ('List2', [('List', "[4, -3, 7]")]), ('NumericalProduct', [('Boolean', "False")]), ]), ])) self.assertEqual(results, [[(1, 4), (2, -3), (0, 7)]])