def test_reshape(self):
     """Uses Reshape to reshape arrays and lists.
     """
     from vistrails.tests.utils import execute, intercept_result
     from ..identifiers import identifier
     with intercept_result(Reshape, 'value') as results:
         self.assertFalse(
             execute([
                 ('convert|Reshape', identifier, [
                     ('array', [('List', '[0, 1, 2, 3, 4, 5]')]),
                     ('shape', [('List', '[2, 3]')]),
                 ]),
             ]))
         self.assertFalse(
             execute([
                 ('convert|Reshape', identifier, [
                     ('array', [('List', '[[1, 2], [3, 4], [5, 6]]')]),
                     ('shape', [('List', '[6]')]),
                 ]),
             ]))
     self.assertEqual(len(results), 2)
     self.assertEqual(results[0].shape, (2, 3))
     self.assertTrue((results[0] == [[0, 1, 2], [3, 4, 5]]).all())
     self.assertEqual(results[1].shape, (6, ))
     self.assertTrue((results[1] == [1, 2, 3, 4, 5, 6]).all())
Example #2
0
 def test_pipeline(self):
     with intercept_results(Iris, 'target', Predict,
                            'prediction') as (y_true, y_pred):
         self.assertFalse(
             execute(
                 [('datasets|Iris', identifier, []),
                  ('preprocessing|StandardScaler', identifier, []),
                  ('feature_selection|SelectKBest', identifier, [
                      ('k', [('Integer', '2')])
                  ]), ('classifiers|LinearSVC', identifier, []),
                  ('Pipeline', identifier, []),
                  ('Predict', identifier, [])],
                 [
                     # feed data to pipeline
                     (0, 'data', 4, 'training_data'),
                     (0, 'target', 4, 'training_target'),
                     # put models in pipeline
                     (1, 'model', 4, 'model1'),
                     (2, 'model', 4, 'model2'),
                     (3, 'model', 4, 'model3'),
                     # predict using pipeline
                     (4, 'model', 5, 'model'),
                     (0, 'data', 5, 'data')
                 ]))
         y_true, y_pred = np.array(y_true[0]), np.array(y_pred[0])
         self.assertEqual(y_true.shape, y_pred.shape)
         self.assertTrue(np.mean(y_true == y_pred) > .8)
Example #3
0
 def test_multiple(self):
     src = urllib2.quote('o = i + j')
     with intercept_result(Map, 'Result') as results:
         self.assertFalse(execute([
                 ('PythonSource', 'org.vistrails.vistrails.basic', [
                     ('source', [('String', src)]),
                 ]),
                 ('Map', 'org.vistrails.vistrails.control_flow', [
                     ('InputPort', [('List', "['i', 'j']")]),
                     ('OutputPort', [('String', 'o')]),
                     ('InputList', [('List',
                         '[(1, 2), (3, 8), (-2, 3)]')]),
                 ]),
             ],
             [
                 (0, 'self', 1, 'FunctionPort'),
             ],
             add_port_specs=[
                 (0, 'input', 'i',
                  'org.vistrails.vistrails.basic:Integer'),
                 (0, 'input', 'j',
                  'org.vistrails.vistrails.basic:Integer'),
                 (0, 'output', 'o',
                  'org.vistrails.vistrails.basic:Integer'),
             ]))
     self.assertEqual(results, [[3, 11, 1]])
Example #4
0
 def test_pythonsource(self):
     import urllib2
     source = ('o = i * 2\n'
               "r = \"it's %d!!!\" % o\n"
               'go_on = o < 100')
     source = urllib2.quote(source)
     from vistrails.tests.utils import execute, intercept_result
     with intercept_result(While, 'Result') as results:
         self.assertFalse(execute([
                 ('PythonSource', 'org.vistrails.vistrails.basic', [
                     ('source', [('String', source)]),
                     ('i', [('Integer', '5')]),
                 ]),
                 ('While', 'org.vistrails.vistrails.control_flow', [
                     ('ConditionPort', [('String', 'go_on')]),
                     ('OutputPort', [('String', 'r')]),
                     ('StateInputPorts', [('List', "['i']")]),
                     ('StateOutputPorts', [('List', "['o']")]),
                 ]),
             ],
             [
                 (0, 'self', 1, 'FunctionPort'),
             ],
             add_port_specs=[
                 (0, 'input', 'i',
                  'org.vistrails.vistrails.basic:Integer'),
                 (0, 'output', 'o',
                  'org.vistrails.vistrails.basic:Integer'),
                 (0, 'output', 'r',
                  'org.vistrails.vistrails.basic:String'),
                 (0, 'output', 'go_on',
                  'org.vistrails.vistrails.basic:Boolean'),
             ]))
     self.assertEqual(results, ["it's 160!!!"])
Example #5
0
 def test_multiple(self):
     src = urllib2.quote("o = i + j")
     with intercept_result(Map, "Result") as results:
         self.assertFalse(
             execute(
                 [
                     ("PythonSource", "org.vistrails.vistrails.basic", [("source", [("String", src)])]),
                     (
                         "Map",
                         "org.vistrails.vistrails.control_flow",
                         [
                             ("InputPort", [("List", "['i', 'j']")]),
                             ("OutputPort", [("String", "o")]),
                             ("InputList", [("List", "[(1, 2), (3, 8), (-2, 3)]")]),
                         ],
                     ),
                 ],
                 [(0, "self", 1, "FunctionPort")],
                 add_port_specs=[
                     (0, "input", "i", "org.vistrails.vistrails.basic:Integer"),
                     (0, "input", "j", "org.vistrails.vistrails.basic:Integer"),
                     (0, "output", "o", "org.vistrails.vistrails.basic:Integer"),
                 ],
             )
         )
     self.assertEqual(results, [[3, 11, 1]])
Example #6
0
 def test_simple(self):
     src = urllib2.quote("o = i + 1")
     with intercept_result(Map, "Result") as results:
         self.assertFalse(
             execute(
                 [
                     ("PythonSource", "org.vistrails.vistrails.basic", [("source", [("String", src)])]),
                     (
                         "Map",
                         "org.vistrails.vistrails.control_flow",
                         [
                             ("InputPort", [("List", "['i']")]),
                             ("OutputPort", [("String", "o")]),
                             ("InputList", [("List", "[1, 2, 8, 9.1]")]),
                         ],
                     ),
                 ],
                 [(0, "self", 1, "FunctionPort")],
                 add_port_specs=[
                     (0, "input", "i", "org.vistrails.vistrails.basic:Float"),
                     (0, "output", "o", "org.vistrails.vistrails.basic:Float"),
                 ],
             )
         )
     self.assertEqual(results, [[2, 3, 9, 10.1]])
Example #7
0
 def test_tuple(self):
     src = urllib2.quote("o = len(i[0]) + i[1]")
     with intercept_result(Map, "Result") as results:
         self.assertFalse(
             execute(
                 [
                     ("PythonSource", "org.vistrails.vistrails.basic", [("source", [("String", src)])]),
                     (
                         "Map",
                         "org.vistrails.vistrails.control_flow",
                         [
                             ("InputPort", [("List", "['i']")]),
                             ("OutputPort", [("String", "o")]),
                             ("InputList", [("List", '[("aa", 1), ("", 8), ("a", 4)]')]),
                         ],
                     ),
                 ],
                 [(0, "self", 1, "FunctionPort")],
                 add_port_specs=[
                     (0, "input", "i", "org.vistrails.vistrails.basic:String,org.vistrails.vistrails.basic:Integer"),
                     (0, "output", "o", "org.vistrails.vistrails.basic:Float"),
                 ],
             )
         )
     self.assertEqual(results, [[3, 8, 5]])
Example #8
0
 def test_pipeline(self):
     with intercept_results(Iris, 'target', Predict, 'prediction') as (y_true, y_pred):
         self.assertFalse(execute(
             [
                 ('datasets|Iris', identifier, []),
                 ('preprocessing|StandardScaler', identifier, []),
                 ('feature_selection|SelectKBest', identifier,
                     [('k', [('Integer', '2')])]),
                 ('classifiers|LinearSVC', identifier, []),
                 ('Pipeline', identifier, []),
                 ('Predict', identifier, [])
             ],
             [
                 # feed data to pipeline
                 (0, 'data', 4, 'training_data'),
                 (0, 'target', 4, 'training_target'),
                 # put models in pipeline
                 (1, 'model', 4, 'model1'),
                 (2, 'model', 4, 'model2'),
                 (3, 'model', 4, 'model3'),
                 # predict using pipeline
                 (4, 'model', 5, 'model'),
                 (0, 'data', 5, 'data')
             ]
         ))
         y_true, y_pred = np.array(y_true[0]), np.array(y_pred[0])
         self.assertEqual(y_true.shape, y_pred.shape)
         self.assertTrue(np.mean(y_true == y_pred) > .8)
Example #9
0
 def test_train_test_split(self):
     # check that we can split the iris dataset
     with intercept_results(TrainTestSplit, 'training_data', TrainTestSplit,
                            'training_target', TrainTestSplit, 'test_data',
                            TrainTestSplit, 'test_target') as results:
         X_train, y_train, X_test, y_test = results
         self.assertFalse(execute(
             [
                 ('datasets|Iris', identifier, []),
                 ('cross-validation|TrainTestSplit', identifier,
                  [('test_size', [('Integer', '50')])])
             ],
             [
                 (0, 'data', 1, 'data'),
                 (0, 'target', 1, 'target')
             ]
         ))
     X_train = np.vstack(X_train)
     X_test = np.vstack(X_test)
     y_train = np.hstack(y_train)
     y_test = np.hstack(y_test)
     self.assertEqual(X_train.shape, (100, 4))
     self.assertEqual(X_test.shape, (50, 4))
     self.assertEqual(y_train.shape, (100,))
     self.assertEqual(y_test.shape, (50,))
Example #10
0
 def test_tuple(self):
     src = urllib2.quote('o = len(i[0]) + i[1]')
     with intercept_result(Map, 'Result') as results:
         self.assertFalse(
             execute(
                 [
                     ('PythonSource', 'org.vistrails.vistrails.basic', [
                         ('source', [('String', src)]),
                     ]),
                     ('Map', 'org.vistrails.vistrails.control_flow', [
                         ('InputPort', [('List', "['i']")]),
                         ('OutputPort', [('String', 'o')]),
                         ('InputList', [
                             ('List', '[("aa", 1), ("", 8), ("a", 4)]')
                         ]),
                     ]),
                 ], [
                     (0, 'self', 1, 'FunctionPort'),
                 ],
                 add_port_specs=[
                     (0, 'input', 'i',
                      'org.vistrails.vistrails.basic:String,org.vistrails.vistrails.basic:Integer'
                      ),
                     (0, 'output', 'o',
                      'org.vistrails.vistrails.basic:Float'),
                 ]))
     self.assertEqual(results, [[3, 8, 5]])
Example #11
0
 def test_csv_numeric(self):
     """Uses CSVFile and ExtractColumn to load a numeric array.
     """
     with intercept_result(ExtractColumn, 'value') as results:
         with intercept_result(CSVFile, 'column_count') as columns:
             self.assertFalse(execute([
                     ('read|CSVFile', identifier, [
                         ('file', [('File', self._test_dir + '/test.csv')]),
                     ]),
                     ('ExtractColumn', identifier, [
                         ('column_index', [('Integer', '1')]),
                         ('column_name', [('String', 'col 2')]),
                         ('numeric', [('Boolean', 'True')]),
                     ]),
                     ('PythonSource', 'org.vistrails.vistrails.basic', [
                         ('source', [('String', '')]),
                     ]),
                 ],
                 [
                     (0, 'value', 1, 'table'),
                     (1, 'value', 2, 'l'),
                 ],
                 add_port_specs=[
                     (2, 'input', 'l',
                      'org.vistrails.vistrails.basic:List'),
                 ]))
             # Here we use a PythonSource just to check that a numpy array
             # can be passed on a List port
     self.assertEqual(columns, [3])
     self.assertEqual(len(results), 1)
     self.assertEqual(list(results[0]), [2.0, 3.0, 14.5])
Example #12
0
 def do_default(self, val):
     if val:
         src = 'o = 42'
     else:
         src = ('from vistrails.core.modules.vistrails_module import '
                'InvalidOutput\n'
                'o = InvalidOutput')
     src = urllib2.quote(src)
     with intercept_result(Default, 'Result') as results:
         self.assertFalse(execute([
                 ('Default', 'org.vistrails.vistrails.control_flow', [
                     ('Default', [('Integer', '28')]),
                 ]),
                 ('PythonSource', 'org.vistrails.vistrails.basic', [
                     ('source', [('String', src)]),
                 ]),
             ],
             [
                 (1, 'o', 0, 'Input'),
             ],
             add_port_specs=[
                 (1, 'output', 'o',
                  'org.vistrails.vistrails.basic:Integer'),
             ]))
     if val:
         self.assertEqual(results, [42])
     else:
         self.assertEqual(results, [28])
Example #13
0
 def do_if(self, val):
     with intercept_result(If, 'Result') as results:
         interp_dict = execute([
                 ('If', 'org.vistrails.vistrails.control_flow', [
                     ('FalseOutputPorts', [('List', "['value']")]),
                     ('TrueOutputPorts', [('List', "['value']")]),
                     ('Condition', [('Boolean', str(val))]),
                 ]),
                 ('Integer', 'org.vistrails.vistrails.basic', [
                     ('value', [('Integer', '42')]),
                 ]),
                 ('Integer', 'org.vistrails.vistrails.basic', [
                     ('value', [('Integer', '28')]),
                 ]),
             ],
             [
                 (1, 'self', 0, 'TruePort'),
                 (2, 'self', 0, 'FalsePort'),
             ],
             full_results=True)
         self.assertFalse(interp_dict.errors)
     if val:
         self.assertEqual(results, [42])
     else:
         self.assertEqual(results, [28])
     self.assertEqual(interp_dict.executed, {0: True, 1: val, 2: not val})
Example #14
0
 def test_xls_header_nonnumeric(self):
     """Uses ExcelSpreadsheet to load data.
     """
     with intercept_result(ExtractColumn, "value") as results:
         with intercept_result(ExcelSpreadsheet, "column_count") as cols:
             self.assertFalse(
                 execute(
                     [
                         (
                             "read|ExcelSpreadsheet",
                             identifier,
                             [
                                 ("file", [("File", self._test_dir + "/xl.xls")]),
                                 ("sheet_name", [("String", "Feuil1")]),
                                 ("header_present", [("Boolean", "True")]),
                             ],
                         ),
                         (
                             "ExtractColumn",
                             identifier,
                             [
                                 ("column_index", [("Integer", "0")]),
                                 ("column_name", [("String", "data1")]),
                                 ("numeric", [("Boolean", "False")]),
                             ],
                         ),
                     ],
                     [(0, "self", 1, "table")],
                 )
             )
     self.assertEqual(cols, [2])
     self.assertEqual(len(results), 1)
     self.assertEqual(list(results[0]), ["here", "is", "some", "text"])
Example #15
0
 def test_xls_header_numeric(self):
     """Uses ExcelSpreadsheet to load a numeric array.
     """
     with intercept_result(ExtractColumn, "value") as results:
         with intercept_result(ExcelSpreadsheet, "column_count") as cols:
             self.assertFalse(
                 execute(
                     [
                         (
                             "read|ExcelSpreadsheet",
                             identifier,
                             [
                                 ("file", [("File", self._test_dir + "/xl.xls")]),
                                 # Will default to first sheet
                                 ("header_present", [("Boolean", "True")]),
                             ],
                         ),
                         (
                             "ExtractColumn",
                             identifier,
                             [("column_name", [("String", "data2")]), ("numeric", [("Boolean", "True")])],
                         ),
                     ],
                     [(0, "self", 1, "table")],
                 )
             )
     self.assertEqual(cols, [2])
     self.assertEqual(len(results), 1)
     self.assertAlmostEqual_lists(list(results[0]), [1, -2.8, 3.4, 3.3])
Example #16
0
 def test_xls_numeric(self):
     """Uses ExcelSpreadsheet to load a numeric array.
     """
     with intercept_result(ExtractColumn, "value") as results:
         with intercept_result(ExcelSpreadsheet, "column_count") as cols:
             self.assertFalse(
                 execute(
                     [
                         (
                             "read|ExcelSpreadsheet",
                             identifier,
                             [
                                 ("file", [("File", self._test_dir + "/xl.xls")]),
                                 ("sheet_index", [("Integer", "1")]),
                                 ("sheet_name", [("String", "Feuil2")]),
                                 ("header_present", [("Boolean", "False")]),
                             ],
                         ),
                         (
                             "ExtractColumn",
                             identifier,
                             [("column_index", [("Integer", "0")]), ("numeric", [("Boolean", "True")])],
                         ),
                     ],
                     [(0, "self", 1, "table")],
                 )
             )
     self.assertEqual(cols, [1])
     self.assertEqual(len(results), 1)
     self.assertAlmostEqual_lists(list(results[0]), [1, 2, 2, 3, -7.6])
    def test_simple(self):
        """Test converting timestamps into matplotlib's format.
        """
        try:
            import matplotlib
        except ImportError:  # pragma: no cover
            self.skipTest("matplotlib is not available")

        from matplotlib.dates import date2num

        with intercept_result(TimestampsToMatplotlib, "dates") as results:
            self.assertFalse(
                execute(
                    [
                        (
                            "convert|dates|TimestampsToMatplotlib",
                            identifier,
                            [("timestamps", [("List", "[1324842375, 1369842877]")])],
                        )
                    ]
                )
            )
        self.assertEqual(len(results), 1)
        results = results[0]
        self.assertEqual(
            list(results),
            list(
                date2num(
                    [datetime.datetime.utcfromtimestamp(1324842375), datetime.datetime.utcfromtimestamp(1369842877)]
                )
            ),
        )
 def test_timezone(self):
     """Test reading timezone-aware dates by supplying an offset.
     """
     dates = ["2013-05-20 9:25", "2013-05-20 09:31", "2013-01-02 19:05"]
     in_fmt = "%Y-%m-%d %H:%M"
     with intercept_result(StringsToDates, "dates") as results:
         self.assertFalse(
             execute(
                 [
                     (
                         "convert|dates|StringsToDates",
                         identifier,
                         [
                             ("strings", [("List", repr(dates))]),
                             ("format", [("String", in_fmt)]),
                             ("timezone", [("String", "-0500")]),
                         ],
                     )
                 ]
             )
         )
     self.assertEqual(len(results), 1)
     results = results[0]
     self.assertTrue(all(d.tzinfo is not None for d in results))
     fmt = "%Y-%m-%d %H:%M:%S %z"
     self.assertEqual(
         [d.strftime(fmt) for d in results],
         ["2013-05-20 09:25:00 -0500", "2013-05-20 09:31:00 -0500", "2013-01-02 19:05:00 -0500"],
     )
Example #19
0
 def test_filter(self):
     src = urllib2.quote("o = bool(i)")
     with intercept_result(Filter, "Result") as results:
         self.assertFalse(
             execute(
                 [
                     ("PythonSource", "org.vistrails.vistrails.basic", [("source", [("String", src)])]),
                     (
                         "Filter",
                         "org.vistrails.vistrails.control_flow",
                         [
                             ("InputPort", [("List", "['i']")]),
                             ("OutputPort", [("String", "o")]),
                             ("InputList", [("List", "[0, 1, 2, 3, '', 'foo', True, False]")]),
                         ],
                     ),
                 ],
                 [(0, "self", 1, "FunctionPort")],
                 add_port_specs=[
                     (0, "input", "i", "org.vistrails.vistrails.basic:Module"),
                     (0, "output", "o", "org.vistrails.vistrails.basic:Boolean"),
                 ],
             )
         )
     self.assertEqual(results, [[1, 2, 3, "foo", True]])
Example #20
0
 def do_default(self, val):
     if val:
         src = 'o = 42'
     else:
         src = ('from vistrails.core.modules.vistrails_module import '
                'InvalidOutput\n'
                'o = InvalidOutput')
     src = urllib2.quote(src)
     with intercept_result(Default, 'Result') as results:
         self.assertFalse(
             execute([
                 ('Default', 'org.vistrails.vistrails.control_flow', [
                     ('Default', [('Integer', '28')]),
                 ]),
                 ('PythonSource', 'org.vistrails.vistrails.basic', [
                     ('source', [('String', src)]),
                 ]),
             ], [
                 (1, 'o', 0, 'Input'),
             ],
                     add_port_specs=[
                         (1, 'output', 'o',
                          'org.vistrails.vistrails.basic:Integer'),
                     ]))
     if val:
         self.assertEqual(results, [42])
     else:
         self.assertEqual(results, [28])
Example #21
0
 def do_select(self, select_functions, error=None):
     with intercept_result(SelectFromTable, 'value') as results:
         errors = execute([
                 ('WriteFile', 'org.vistrails.vistrails.basic', [
                     ('in_value', [('String', '22;a;T;abaab\n'
                                              '43;b;F;aabab\n'
                                              '-7;d;T;abbababb\n'
                                              '500;e;F;aba abacc')]),
                 ]),
                 ('read|CSVFile', identifier, [
                     ('delimiter', [('String', ';')]),
                     ('header_present', [('Boolean', 'False')]),
                     ('sniff_header', [('Boolean', 'False')]),
                 ]),
                 ('SelectFromTable', identifier, select_functions),
             ],
             [
                 (0, 'out_value', 1, 'file'),
                 (1, 'value', 2, 'table'),
             ])
     if error is not None:
         self.assertEqual([2], errors.keys())
         self.assertIn(error, errors[2].message)
         return None
     else:
         self.assertFalse(errors)
         self.assertEqual(len(results), 1)
         return results[0]
Example #22
0
 def do_aggregate(self, agg_functions):
     with intercept_result(AggregateColumn, 'value') as results:
         errors = execute([
                 ('WriteFile', 'org.vistrails.vistrails.basic', [
                     ('in_value', [('String', '22;a;T;100\n'
                                              '43;b;F;3\n'
                                              '-7;d;T;41\n'
                                              '500;e;F;21\n'
                                              '20;a;T;1\n'
                                              '43;b;F;23\n'
                                              '21;a;F;41\n')]),
                 ]),
                 ('read|CSVFile', identifier, [
                     ('delimiter', [('String', ';')]),
                     ('header_present', [('Boolean', 'False')]),
                     ('sniff_header', [('Boolean', 'False')]),
                 ]),
                 ('AggregateColumn', identifier, agg_functions),
             ],
             [
                 (0, 'out_value', 1, 'file'),
                 (1, 'value', 2, 'table'),
             ])
     self.assertFalse(errors)
     self.assertEqual(len(results), 1)
     return results[0]
Example #23
0
 def do_if(self, val):
     with intercept_result(If, 'Result') as results:
         interp_dict = execute([
             ('If', 'org.vistrails.vistrails.control_flow', [
                 ('FalseOutputPorts', [('List', "['value']")]),
                 ('TrueOutputPorts', [('List', "['value']")]),
                 ('Condition', [('Boolean', str(val))]),
             ]),
             ('Integer', 'org.vistrails.vistrails.basic', [
                 ('value', [('Integer', '42')]),
             ]),
             ('Integer', 'org.vistrails.vistrails.basic', [
                 ('value', [('Integer', '28')]),
             ]),
         ], [
             (1, 'self', 0, 'TruePort'),
             (2, 'self', 0, 'FalsePort'),
         ],
                               full_results=True)
         self.assertFalse(interp_dict.errors)
     if val:
         self.assertEqual(results, [42])
     else:
         self.assertEqual(results, [28])
     self.assertEqual(interp_dict.executed, {0: True, 1: val, 2: not val})
Example #24
0
    def test_dateutil(self):
        """Test reading non-timezone-aware dates without providing the format.

        dateutil is required for this one.
        """
        try:
            import dateutil
        except ImportError: # pragma: no cover
            self.skipTest("dateutil is not available")

        dates = ['2013-05-20 9:25',
                 'Thu Sep 25 10:36:26 2003',
                 '2003 10:36:28 CET 25 Sep Thu'] # Timezone will be ignored
        with intercept_result(StringsToDates, 'dates') as results:
            self.assertFalse(execute([
                    ('convert|dates|StringsToDates', identifier, [
                        ('strings', [('List', repr(dates))]),
                    ]),
                ]))
        self.assertEqual(len(results), 1)
        results = results[0]
        fmt = '%Y-%m-%d %H:%M:%S %Z %z'
        self.assertEqual(
                [d.strftime(fmt) for d in results],
                ['2013-05-20 09:25:00  ',
                 '2003-09-25 10:36:26  ',
                 '2003-09-25 10:36:28  '])
Example #25
0
 def test_strings(self):
     from vistrails.tests.utils import execute, intercept_result
     from ..common import ExtractColumn
     from ..identifiers import identifier
     with intercept_result(ExtractColumn, 'value') as results:
         self.assertFalse(execute([
                 ('BuildTable', identifier, [
                     ('a', [('List', "['a', '2', 'c']")]),
                     ('b', [('List', "[4, 5, 6]")]),
                 ]),
                 (self.WRITER_MODULE, identifier, []),
                 (self.READER_MODULE, identifier, []),
                 ('ExtractColumn', identifier, [
                     ('column_index', [('Integer', '0')]),
                     ('numeric', [('Boolean', 'False')]),
                 ]),
             ], [
                 (0, 'value', 1, 'table'),
                 (1, 'file', 2, 'file'),
                 (2, 'value', 3, 'table'),
             ],
             add_port_specs=[
                 (0, 'input', 'a',
                  '(org.vistrails.vistrails.basic:List)'),
                 (0, 'input', 'b',
                  '(org.vistrails.vistrails.basic:List)'),
             ]))
     self.assertEqual(len(results), 1)
     self.assertEqual(results[0], ['a', '2', 'c'])
Example #26
0
    def test_npy_numpy(self):
        """Uses NumPyArray to load an array in .NPY format.
        """
        from ..identifiers import identifier
        from vistrails.tests.utils import execute, intercept_result

        with intercept_result(NumPyArray, 'value') as results:
            self.assertFalse(
                execute([
                    ('read|NumPyArray', identifier, [
                        ('datatype', [('String', 'npy')]),
                        ('file', [('File', self._test_dir + '/random.npy')]),
                    ]),
                    ('PythonSource', 'org.vistrails.vistrails.basic', [
                        ('source', [('String', '')]),
                    ]),
                ], [
                    (0, 'value', 1, 'l'),
                ],
                        add_port_specs=[
                            (1, 'input', 'l',
                             'org.vistrails.vistrails.basic:List'),
                        ]))
        self.assertEqual(len(results), 1)
        self.assertEqual(list(results[0]), [1.0, 7.0, 5.0, 3.0, 6.0, 1.0])
Example #27
0
 def test_csv_numeric(self):
     """Uses CSVFile and ExtractColumn to load a numeric array.
     """
     with intercept_result(ExtractColumn, 'value') as results:
         with intercept_result(CSVFile, 'column_count') as columns:
             self.assertFalse(
                 execute([
                     ('read|CSVFile', identifier, [
                         ('file', [('File', self._test_dir + '/test.csv')]),
                     ]),
                     ('ExtractColumn', identifier, [
                         ('column_index', [('Integer', '1')]),
                         ('column_name', [('String', 'col 2')]),
                         ('numeric', [('Boolean', 'True')]),
                     ]),
                     ('PythonSource', 'org.vistrails.vistrails.basic', [
                         ('source', [('String', '')]),
                     ]),
                 ], [
                     (0, 'value', 1, 'table'),
                     (1, 'value', 2, 'l'),
                 ],
                         add_port_specs=[
                             (2, 'input', 'l',
                              'org.vistrails.vistrails.basic:List'),
                         ]))
             # Here we use a PythonSource just to check that a numpy array
             # can be passed on a List port
     self.assertEqual(columns, [3])
     self.assertEqual(len(results), 1)
     self.assertEqual(list(results[0]), [2.0, 3.0, 14.5])
Example #28
0
 def test_timestamps(self):
     """Test conversion to datetime objects.
     """
     timestamps = [1369041900, 1369042260, 1357153500]
     with intercept_result(TimestampsToDates, 'dates') as results:
         self.assertFalse(execute([
                 ('convert|dates|TimestampsToDates', identifier, [
                     ('timestamps', [('List', repr(timestamps))]),
                 ]),
             ]))
     self.assertEqual(len(results), 1)
     results = results[0]
     self.assertTrue(all(d.tzinfo is utc for d in results))
     fmt = '%Y-%m-%d %H:%M:%S %Z %z'
     self.assertEqual(
             [d.strftime(fmt) for d in results],
             ['2013-05-20 09:25:00 UTC +0000',
              '2013-05-20 09:31:00 UTC +0000',
              '2013-01-02 19:05:00 UTC +0000'])
     try:
         import pytz
     except ImportError: # pragma: no cover
         pass
     else:
         self.assertEqual(
                 [d.astimezone(pytz.timezone('US/Eastern')).strftime(fmt)
                  for d in results],
                 ['2013-05-20 05:25:00 EDT -0400',
                  '2013-05-20 05:31:00 EDT -0400',
                  '2013-01-02 14:05:00 EST -0500'])
    def test_npy_numpy(self):
        """Uses NumPyArray to load an array in .NPY format.
        """
        from ..identifiers import identifier
        from vistrails.tests.utils import execute, intercept_result

        with intercept_result(NumPyArray, 'value') as results:
            self.assertFalse(execute([
                    ('read|numpy|NumPyArray', identifier, [
                        ('datatype', [('String', 'npy')]),
                        ('file', [('File', self._test_dir + '/random.npy')]),
                    ]),
                    ('PythonSource', 'org.vistrails.vistrails.basic', [
                        ('source', [('String', '')]),
                    ]),
                ],
                [
                    (0, 'value', 1, 'l'),
                ],
                add_port_specs=[
                    (1, 'input', 'l',
                     'org.vistrails.vistrails.basic:List'),
                ]))
        self.assertEqual(len(results), 1)
        self.assertEqual(list(results[0]), [1.0, 7.0, 5.0, 3.0, 6.0, 1.0])
Example #30
0
 def test_strings(self):
     from vistrails.tests.utils import execute, intercept_result
     from ..common import ExtractColumn
     from ..identifiers import identifier
     with intercept_result(ExtractColumn, 'value') as results:
         self.assertFalse(execute([
                 ('BuildTable', identifier, [
                     ('a', [('List', "['a', '2', 'c']")]),
                     ('b', [('List', "[4, 5, 6]")]),
                 ]),
                 (self.WRITER_MODULE, identifier, []),
                 (self.READER_MODULE, identifier, []),
                 ('ExtractColumn', identifier, [
                     ('column_index', [('Integer', '0')]),
                     ('numeric', [('Boolean', 'False')]),
                 ]),
             ], [
                 (0, 'value', 1, 'table'),
                 (1, 'file', 2, 'file'),
                 (2, 'value', 3, 'table'),
             ],
             add_port_specs=[
                 (0, 'input', 'a',
                  '(org.vistrails.vistrails.basic:List)'),
                 (0, 'input', 'b',
                  '(org.vistrails.vistrails.basic:List)'),
             ]))
     self.assertEqual(len(results), 1)
     self.assertEqual(results[0], ['a', '2', 'c'])
    def test_dateutil(self):
        """Test reading non-timezone-aware dates without providing the format.

        dateutil is required for this one.
        """
        try:
            import dateutil
        except ImportError:  # pragma: no cover
            self.skipTest("dateutil is not available")

        dates = [
            "2013-05-20 9:25",
            "Thu Sep 25 10:36:26 2003",
            "2003 10:36:28 CET 25 Sep Thu",
        ]  # Timezone will be ignored
        with intercept_result(StringsToDates, "dates") as results:
            self.assertFalse(
                execute([("convert|dates|StringsToDates", identifier, [("strings", [("List", repr(dates))])])])
            )
        self.assertEqual(len(results), 1)
        results = results[0]
        fmt = "%Y-%m-%d %H:%M:%S %Z %z"
        self.assertEqual(
            [d.strftime(fmt) for d in results],
            ["2013-05-20 09:25:00  ", "2003-09-25 10:36:26  ", "2003-09-25 10:36:28  "],
        )
Example #32
0
 def do_project(self, project_functions, error=None):
     with intercept_result(ProjectTable, 'value') as results:
         errors = execute([
                 ('BuildTable', identifier, [
                     ('letters', [('List', repr(['a', 'b', 'c', 'd']))]),
                     ('numbers', [('List', repr([1, 2, 3, '4']))]),
                     ('cardinals', [('List', repr(['one', 'two',
                                                   'three', 'four']))]),
                     ('ordinals', [('List', repr(['first', 'second',
                                                  'third', 'fourth']))])
                 ]),
                 ('ProjectTable', identifier, project_functions),
             ],
             [
                 (0, 'value', 1, 'table'),
             ],
             add_port_specs=[
                 (0, 'input', 'letters',
                  'org.vistrails.vistrails.basic:List'),
                 (0, 'input', 'numbers',
                  'org.vistrails.vistrails.basic:List'),
                 (0, 'input', 'cardinals',
                  'org.vistrails.vistrails.basic:List'),
                 (0, 'input', 'ordinals',
                  'org.vistrails.vistrails.basic:List'),
             ])
     if error is not None:
         self.assertEqual([1], errors.keys())
         self.assertIn(error, errors[1].message)
         return None
     else:
         self.assertFalse(errors)
         self.assertEqual(len(results), 1)
         return results[0]
Example #33
0
    def test_timezone_pytz(self):
        """Test reading timezone-aware dates through pytz.
        """
        try:
            import pytz
        except ImportError: # pragma: no cover
            self.skipTest("pytz is not available")
        if LooseVersion(pytz.__version__) < PYTZ_MIN_VER: # pragma: no cover
            self.skipTest("pytz version is known to cause issues (%s)" %
                          pytz.__version__)

        dates = ['2013-01-20 9:25', '2013-01-20 09:31', '2013-06-02 19:05']
        in_fmt = '%Y-%m-%d %H:%M'
        with intercept_result(StringsToDates, 'dates') as results:
            self.assertFalse(execute([
                    ('convert|dates|StringsToDates', identifier, [
                        ('strings', [('List', repr(dates))]),
                        ('format', [('String', in_fmt)]),
                        ('timezone', [('String', 'America/New_York')])
                    ]),
                ]))
        self.assertEqual(len(results), 1)
        results = results[0]
        self.assertTrue(all(d.tzinfo is not None for d in results))
        fmt = '%Y-%m-%d %H:%M:%S %Z %z'
        self.assertEqual(
                [d.strftime(fmt) for d in results],
                ['2013-01-20 09:25:00 EST -0500',
                 '2013-01-20 09:31:00 EST -0500',
                 '2013-06-02 19:05:00 EDT -0400'])
Example #34
0
 def testIncorrectURL_2(self):
     from vistrails.tests.utils import execute
     self.assertTrue(execute([
             ('DownloadFile', identifier, [
                 ('url', [('String', 'http://neitherodesthisohrly')]),
             ]),
         ]))
Example #35
0
    def test_timezone(self):
        """Test reading timezone-aware dates by supplying an offset.
        """
        try:
            import matplotlib
        except ImportError: # pragma: no cover
            self.skipTest("matplotlib is not available")

        from matplotlib.dates import date2num

        dates = ['2013-05-20 9:25', '2013-05-20 09:31', '2013-01-02 18:05']
        in_fmt = '%Y-%m-%d %H:%M'
        with intercept_result(StringsToMatplotlib, 'dates') as results:
            self.assertFalse(execute([
                    ('convert|dates|StringsToMatplotlib', identifier, [
                        ('strings', [('List', repr(dates))]),
                        ('format', [('String', in_fmt)]),
                        ('timezone', [('String', '-0500')])
                    ]),
                ]))
        self.assertEqual(len(results), 1)
        results = results[0]
        self.assertEqual(list(results), list(date2num([
                datetime.datetime(2013, 5, 20, 14, 25, 0),
                datetime.datetime(2013, 5, 20, 14, 31, 0),
                datetime.datetime(2013, 1, 2, 23, 5, 0)])))
Example #36
0
    def test_timezone(self):
        """Test reading timezone-aware dates by supplying an offset.
        """
        try:
            import matplotlib
        except ImportError:  # pragma: no cover
            self.skipTest("matplotlib is not available")

        from matplotlib.dates import date2num

        dates = ['2013-05-20 9:25', '2013-05-20 09:31', '2013-01-02 18:05']
        in_fmt = '%Y-%m-%d %H:%M'
        with intercept_result(StringsToMatplotlib, 'dates') as results:
            self.assertFalse(
                execute([
                    ('convert|dates|StringsToMatplotlib', identifier,
                     [('strings', [('List', repr(dates))]),
                      ('format', [('String', in_fmt)]),
                      ('timezone', [('String', '-0500')])]),
                ]))
        self.assertEqual(len(results), 1)
        results = results[0]
        self.assertEqual(
            list(results),
            list(
                date2num([
                    datetime.datetime(2013, 5, 20, 14, 25, 0),
                    datetime.datetime(2013, 5, 20, 14, 31, 0),
                    datetime.datetime(2013, 1, 2, 23, 5, 0)
                ])))
Example #37
0
 def testIncorrectURL(self):
     from vistrails.tests.utils import execute
     self.assertTrue(execute([
             ('DownloadFile', identifier, [
                 ('url', [('String', 'http://idbetthisdoesnotexistohrly')]),
             ]),
         ]))
Example #38
0
    def test_simple(self):
        """Test converting timestamps into matplotlib's format.
        """
        try:
            import matplotlib
        except ImportError:  # pragma: no cover
            self.skipTest("matplotlib is not available")

        from matplotlib.dates import date2num

        with intercept_result(TimestampsToMatplotlib, 'dates') as results:
            self.assertFalse(
                execute([
                    ('convert|dates|TimestampsToMatplotlib', identifier, [
                        ('timestamps', [('List', '[1324842375, 1369842877]')]),
                    ]),
                ]))
        self.assertEqual(len(results), 1)
        results = results[0]
        self.assertEqual(
            list(results),
            list(
                date2num([
                    datetime.datetime.utcfromtimestamp(1324842375),
                    datetime.datetime.utcfromtimestamp(1369842877)
                ])))
Example #39
0
    def test_timezone_pytz(self):
        """Test reading timezone-aware dates through pytz.
        """
        try:
            import pytz
        except ImportError:  # pragma: no cover
            self.skipTest("pytz is not available")
        if LooseVersion(pytz.__version__) < PYTZ_MIN_VER:  # pragma: no cover
            self.skipTest("pytz version is known to cause issues (%s)" %
                          pytz.__version__)

        dates = ['2013-01-20 9:25', '2013-01-20 09:31', '2013-06-02 19:05']
        in_fmt = '%Y-%m-%d %H:%M'
        with intercept_result(StringsToDates, 'dates') as results:
            self.assertFalse(
                execute([
                    ('convert|dates|StringsToDates', identifier,
                     [('strings', [('List', repr(dates))]),
                      ('format', [('String', in_fmt)]),
                      ('timezone', [('String', 'America/New_York')])]),
                ]))
        self.assertEqual(len(results), 1)
        results = results[0]
        self.assertTrue(all(d.tzinfo is not None for d in results))
        fmt = '%Y-%m-%d %H:%M:%S %Z %z'
        self.assertEqual([d.strftime(fmt) for d in results], [
            '2013-01-20 09:25:00 EST -0500', '2013-01-20 09:31:00 EST -0500',
            '2013-06-02 19:05:00 EDT -0400'
        ])
Example #40
0
    def test_dateutil(self):
        """Test reading non-timezone-aware dates without providing the format.

        dateutil is required for this one.
        """
        try:
            import dateutil
        except ImportError:  # pragma: no cover
            self.skipTest("dateutil is not available")

        dates = [
            '2013-05-20 9:25', 'Thu Sep 25 10:36:26 2003',
            '2003 10:36:28 CET 25 Sep Thu'
        ]  # Timezone will be ignored
        with intercept_result(StringsToDates, 'dates') as results:
            self.assertFalse(
                execute([
                    ('convert|dates|StringsToDates', identifier, [
                        ('strings', [('List', repr(dates))]),
                    ]),
                ]))
        self.assertEqual(len(results), 1)
        results = results[0]
        fmt = '%Y-%m-%d %H:%M:%S %Z %z'
        self.assertEqual([d.strftime(fmt) for d in results], [
            '2013-05-20 09:25:00  ', '2003-09-25 10:36:26  ',
            '2003-09-25 10:36:28  '
        ])
Example #41
0
 def test_timestamps(self):
     """Test conversion to datetime objects.
     """
     timestamps = [1369041900, 1369042260, 1357153500]
     with intercept_result(TimestampsToDates, 'dates') as results:
         self.assertFalse(
             execute([
                 ('convert|dates|TimestampsToDates', identifier, [
                     ('timestamps', [('List', repr(timestamps))]),
                 ]),
             ]))
     self.assertEqual(len(results), 1)
     results = results[0]
     self.assertTrue(all(d.tzinfo is utc for d in results))
     fmt = '%Y-%m-%d %H:%M:%S %Z %z'
     self.assertEqual([d.strftime(fmt) for d in results], [
         '2013-05-20 09:25:00 UTC +0000', '2013-05-20 09:31:00 UTC +0000',
         '2013-01-02 19:05:00 UTC +0000'
     ])
     try:
         import pytz
     except ImportError:  # pragma: no cover
         pass
     else:
         self.assertEqual([
             d.astimezone(pytz.timezone('US/Eastern')).strftime(fmt)
             for d in results
         ], [
             '2013-05-20 05:25:00 EDT -0400',
             '2013-05-20 05:31:00 EDT -0400',
             '2013-01-02 14:05:00 EST -0500'
         ])
Example #42
0
 def test_sum(self):
     with intercept_result(Sum, 'Result') as results:
         self.assertFalse(execute([
                 ('Sum', 'org.vistrails.vistrails.control_flow', [
                     ('InputList', [('List', "[1, 2, 3, 8, 14.7]")]),
                 ]),
             ]))
     self.assertEqual(results, [28.7])
Example #43
0
 def test_sum(self):
     with intercept_result(Sum, "Result") as results:
         self.assertFalse(
             execute(
                 [("Sum", "org.vistrails.vistrails.control_flow", [("InputList", [("List", "[1, 2, 3, 8, 14.7]")])])]
             )
         )
     self.assertEqual(results, [28.7])
Example #44
0
 def test_sum(self):
     with intercept_result(Sum, 'Result') as results:
         self.assertFalse(execute([
                 ('Sum', 'org.vistrails.vistrails.control_flow', [
                     ('InputList', [('List', "[1, 2, 3, 8, 14.7]")]),
                 ]),
             ]))
     self.assertEqual(results, [28.7])
Example #45
0
 def test_iris(self):
     # check that the iris dataset can be loaded
     with intercept_results(Iris, 'data', Iris, 'target') as (data, target):
         self.assertFalse(execute([('datasets|Iris', identifier, [])]))
     data = np.vstack(data)
     target = np.hstack(target)
     self.assertEqual(data.shape, (150, 4))
     self.assertEqual(target.shape, (150, ))
Example #46
0
 def test_elementwise(self):
     with intercept_result(ElementwiseProduct, 'Result') as results:
         self.assertFalse(execute([
                 ('ElementwiseProduct', 'org.vistrails.vistrails.control_flow', [
                     ('List1', [('List', "[1, 2, 0]")]),
                     ('List2', [('List', "[4, -3, 7]")]),
                 ]),
             ]))
     self.assertEqual(results, [[4, -6, 0]])
Example #47
0
 def test_dot(self):
     with intercept_result(Dot, 'Result') as results:
         self.assertFalse(execute([
                 ('Dot', 'org.vistrails.vistrails.control_flow', [
                     ('List1', [('List', "[1, 2, 0]")]),
                     ('List2', [('List', "[4, -3, 7]")]),
                 ]),
             ]))
     self.assertEqual(results, [-2])
Example #48
0
 def test_digits(self):
     # check that the digits dataset can be loaded
     with intercept_results(Digits, 'data', Digits,
                            'target') as (data, target):
         self.assertFalse(execute([('datasets|Digits', identifier, [])]))
     data = np.vstack(data)
     target = np.hstack(target)
     self.assertEqual(data.shape, (1797, 64))
     self.assertEqual(target.shape, (1797, ))
Example #49
0
 def test_cartesian_combine(self):
     with intercept_result(CartesianProduct, 'Result') as results:
         self.assertFalse(execute([
                 ('CartesianProduct', 'org.vistrails.vistrails.control_flow', [
                     ('List1', [('List', "[(1, 2)]")]),
                     ('List2', [('List', "[3, 4]")]),
                 ]),
             ]))
     self.assertEqual(results, [[(1, 2, 3), (1, 2, 4)]])
Example #50
0
    def test_join(self):
        """Test joining tables that have column names.
        """
        import numpy

        with intercept_result(JoinTables, 'value') as results:
            self.assertFalse(execute([
                    ('BuildTable', identifier, [
                        ('id', [('List', repr([1, '2', 4, 5]))]),
                        ('A_name', [('List',
                            repr(['one', 2, 'four', 'five'])),
                        ]),
                    ]),
                    ('BuildTable', identifier, [
                        ('B_age', [('List',
                            repr([14, 50, '12', 22])),
                        ]),
                        ('id', [('List', repr(['1', 2, 3, 5]))]),
                    ]),
                    ('JoinTables', identifier, [
                        ('left_column_idx', [('Integer', '0')]),
                        ('right_column_name', [('String', 'id')]),
                        ('right_column_idx', [('Integer', '1')]),
                    ]),
                ],
                [
                    (0, 'value', 2, 'left_table'),
                    (1, 'value', 2, 'right_table'),
                ],
                add_port_specs=[
                    (0, 'input', 'id',
                     'org.vistrails.vistrails.basic:List'),
                    (0, 'input', 'A_name',
                     'org.vistrails.vistrails.basic:List'),
                    (1, 'input', 'B_age',
                     'org.vistrails.vistrails.basic:List'),
                    (1, 'input', 'id',
                     'org.vistrails.vistrails.basic:List'),
                ]))
        self.assertEqual(len(results), 1)
        table, = results

        self.assertEqual(table.names, ['left.id', 'A_name',
                                       'B_age', 'right.id'])

        self.assertEqual(table.get_column(0, False), [1, '2', 5])
        l = table.get_column(0, True)
        self.assertIsInstance(l, numpy.ndarray)
        self.assertEqual(list(l), [1, 2, 5])
        self.assertEqual(table.get_column(3, False), ['1', 2, 5])
        l = table.get_column(3, True)
        self.assertIsInstance(l, numpy.ndarray)
        self.assertEqual(list(l), [1, 2, 5])

        self.assertEqual(table.get_column(1, False), ['one', 2, 'five'])
        self.assertEqual(list(table.get_column(2, True)), [14, 50, 22])
Example #51
0
 def test_elementwise_tuples(self):
     with intercept_result(ElementwiseProduct, 'Result') as results:
         self.assertFalse(execute([
                 ('ElementwiseProduct', 'org.vistrails.vistrails.control_flow', [
                     ('List1', [('List', "[1, 2, 0]")]),
                     ('List2', [('List', "[4, -3, 7]")]),
                     ('NumericalProduct', [('Boolean', "False")]),
                 ]),
             ]))
     self.assertEqual(results, [[(1, 4), (2, -3), (0, 7)]])