def make_pipeline(self):
        """Creates an example pipeline that is used to conduct tests.
        """
        vistrail = Vistrail()
        controller = VistrailController(vistrail)
        controller.change_selected_version(0)

        # 0     1    2   7   8
        # |    / \        \ /
        # 3   4   5        9
        #         |       / \
        #         6     10   11
        modules = [controller.add_module('org.vistrails.vistrails.basic',
                                         'String')
                   for i in xrange(12)]

        def connect(outmod, inmod):
            controller.add_connection(
                modules[outmod].id,
                'value',
                modules[inmod].id,
                'value')

        for (outmod, inmod) in [(0, 3),
                                (1, 4), (1, 5), (5, 6),
                                (7, 9), (8, 9), (9, 10), (9, 11)]:
            connect(outmod, inmod)

        return controller, modules
Beispiel #2
0
 def __init__(self, arg=None):
     initialize()
     if arg is None:
         # Copied from VistrailsApplicationInterface#open_vistrail()
         locator = UntitledLocator()
         loaded_objs = vistrails.core.db.io.load_vistrail(locator)
         self.controller = VistrailController(loaded_objs[0], locator,
                                              *loaded_objs[1:])
     elif isinstance(arg, (_Pipeline, Pipeline)):
         if isinstance(arg, Pipeline):
             pipeline = arg.pipeline
         else:
             pipeline = arg
         # Copied from VistrailsApplicationInterface#open_workflow()
         vistrail = _Vistrail()
         ops = []
         for module in pipeline.module_list:
             ops.append(('add', module))
         for connection in pipeline.connection_list:
             ops.append(('add', connection))
         action = vistrails.core.db.action.create_action(ops)
         vistrail.add_action(action, 0L)
         vistrail.update_id_scope()
         vistrail.change_description("Imported pipeline", 0L)
         self.controller = VistrailController(vistrail, UntitledLocator())
     elif isinstance(arg, VistrailController):
         self.controller = arg
     elif isinstance(arg, basestring):
         raise TypeError("Vistrail was constructed from %r.\n"
                         "Use load_vistrail() to get a Vistrail from a "
                         "file." % type(arg).__name__)
     else:
         raise TypeError("Vistrail was constructed from unexpected "
                         "argument type %r" % type(arg).__name__)
Beispiel #3
0
 def test_vtl_files(self):
     from vistrails.tests.utils import run_file
     for root, dirs, file_names in os.walk(self.vtl_path):
         for file_name in file_names:
             if file_name.endswith('.vtl'):
                 f = os.path.join(root, file_name)
                 locator = FileLocator(f)
                 version = locator._vnode
                 # if there is a version specified try to execute it,
                 # else just load the pipeline
                 if version:
                     errors = run_file(f, lambda x: x == version)
                     self.assertEqual(errors, [], 'Errors processing %s: %s' % (f, str(errors)))
                 else:
                     import vistrails.core.db.io
                     from vistrails.core.vistrail.controller import \
                         VistrailController
                     loaded_objs = vistrails.core.db.io.load_vistrail(locator)
                     controller = VistrailController(loaded_objs[0],
                                                     locator,
                                                     *loaded_objs[1:])
                     controller.change_selected_version(
                         controller.vistrail.get_latest_version())
                     self.assertTrue(controller.current_pipeline.is_valid,
                                     "Latest pipeline is invalid: %s" % f)
Beispiel #4
0
 def __init__(self, arg=None):
     initialize()
     if arg is None:
         # Copied from VistrailsApplicationInterface#open_vistrail()
         locator = UntitledLocator()
         loaded_objs = vistrails.core.db.io.load_vistrail(locator)
         self.controller = VistrailController(loaded_objs[0], locator,
                                              *loaded_objs[1:])
     elif isinstance(arg, (_Pipeline, Pipeline)):
         if isinstance(arg, Pipeline):
             pipeline = arg.pipeline
         else:
             pipeline = arg
         # Copied from VistrailsApplicationInterface#open_workflow()
         vistrail = _Vistrail()
         ops = []
         for module in pipeline.module_list:
             ops.append(('add', module))
         for connection in pipeline.connection_list:
             ops.append(('add', connection))
         action = vistrails.core.db.action.create_action(ops)
         vistrail.add_action(action, 0L)
         vistrail.update_id_scope()
         vistrail.change_description("Imported pipeline", 0L)
         self.controller = VistrailController(vistrail, UntitledLocator())
     elif isinstance(arg, VistrailController):
         self.controller = arg
     elif isinstance(arg, basestring):
         raise TypeError("Vistrail was constructed from %r.\n"
                         "Use load_vistrail() to get a Vistrail from a "
                         "file." % type(arg).__name__)
     else:
         raise TypeError("Vistrail was constructed from unexpected "
                         "argument type %r" % type(arg).__name__)
 def test_vtl_files(self):
     from vistrails.tests.utils import run_file
     for root, dirs, file_names in os.walk(self.vtl_path):
         for file_name in sorted(file_names):
             if file_name.endswith('.vtl'):
                 # update available packages
                 from vistrails.core.packagemanager import get_package_manager
                 get_package_manager().build_available_package_names_list()
                 f = os.path.join(root, file_name)
                 locator = FileLocator(f)
                 version = locator._vnode
                 # if there is a version specified try to execute it,
                 # else just load the pipeline
                 if version:
                     errors = run_file(f, lambda x: x == version)
                     self.assertEqual(
                         errors, [],
                         'Errors processing %s: %s' % (f, str(errors)))
                 else:
                     import vistrails.core.db.io
                     from vistrails.core.vistrail.controller import \
                         VistrailController
                     loaded_objs = vistrails.core.db.io.load_vistrail(
                         locator)
                     controller = VistrailController(
                         loaded_objs[0], locator, *loaded_objs[1:])
                     controller.change_selected_version(
                         controller.vistrail.get_latest_version())
                     self.assertTrue(controller.current_pipeline.is_valid,
                                     "Latest pipeline is invalid: %s" % f)
Beispiel #6
0
    def make_pipeline(self):
        """Creates an example pipeline that is used to conduct tests.
        """
        vistrail = Vistrail()
        controller = VistrailController(vistrail)
        controller.change_selected_version(0)

        # 0     1    2   7   8
        # |    / \        \ /
        # 3   4   5        9
        #         |       / \
        #         6     10   11
        modules = [
            controller.add_module('org.vistrails.vistrails.basic', 'String')
            for i in xrange(12)
        ]

        def connect(outmod, inmod):
            controller.add_connection(modules[outmod].id, 'value',
                                      modules[inmod].id, 'value')

        for (outmod, inmod) in [(0, 3), (1, 4), (1, 5), (5, 6), (7, 9), (8, 9),
                                (9, 10), (9, 11)]:
            connect(outmod, inmod)

        return controller, modules
Beispiel #7
0
 def copyBaseVistrailController(vt_controller):
     vistrail = vt_controller.vistrail
     newvt_controller = BaseVistrailController(vistrail, None)
     current_log = vt_controller.log
     newvt_controller.log = current_log
     # DAK: why does the base version copy the _mashups but the non-base
     # version doesn't??
     for m in vt_controller._mashups:
         newvt_controller._mashups.append(copy.copy(m))
     return newvt_controller
 def copyBaseVistrailController(vt_controller):
     vistrail = vt_controller.vistrail
     newvt_controller = BaseVistrailController(vistrail, None)
     current_log = vt_controller.log
     newvt_controller.log = current_log
     # DAK: why does the base version copy the _mashups but the non-base
     # version doesn't??
     for m in vt_controller._mashups:
         newvt_controller._mashups.append(copy.copy(m))
     return newvt_controller
Beispiel #9
0
    def pipeline_from_generator(variable_gen):
        # Get the original OutputPort module
        orig_controller = variable_gen._generator.controller
        base_pipeline = orig_controller.vistrail.getPipeline('dat-vars')
        if len(base_pipeline.module_list) != 1:
            raise ValueError("dat-vars version is invalid")
        output_port = base_pipeline.module_list[0]

        controller = VistrailController(Vistrail())
        # OutputPort
        operations = [('add', output_port)]
        # Rest of the pipeline
        operations += variable_gen._generator.operations
        # Connection
        connection = controller.create_connection(
            variable_gen._output_module, variable_gen._outputport_name,
            output_port, 'InternalPipe')
        operations.append(('add', connection))
        # Materialize this
        action = create_action(operations)
        controller.add_new_action(action)
        version = controller.perform_action(action)
        controller.change_selected_version(version)
        assert version == controller.current_version == 1
        return controller.current_pipeline, 1
Beispiel #10
0
    def test_find_modules_by_type(self):
        """Tests the find_modules_by_type() function.
        """
        vistrail = Vistrail()
        controller = VistrailController(vistrail)
        controller.change_selected_version(0)

        mod1 = controller.add_module('org.vistrails.vistrails.basic', 'String')
        mod2 = controller.add_module('org.vistrails.vistrails.basic', 'Float')
        mod3 = controller.add_module('org.vistrails.vistrails.basic', 'String')
        mod4 = controller.add_module('org.vistrails.vistrails.basic',
                                     'Integer')

        from dat.vistrails_interface import find_modules_by_type
        from vistrails.core.modules.basic_modules import Boolean, Float, String

        self.assertEqual(
            set(m.id for m in find_modules_by_type(controller.current_pipeline,
                                                   [String])),
            set([mod1.id, mod3.id]))

        self.assertEqual([
            m.id
            for m in find_modules_by_type(controller.current_pipeline, [Float])
        ], [mod2.id, mod4.id])

        self.assertEqual(
            find_modules_by_type(controller.current_pipeline, [Boolean]), [])
Beispiel #11
0
 def add_vistrail(self, vistrail, locator,
         abstraction_files=None,  thumbnail_files=None, mashups=None):
     objs = vistrail, locator, abstraction_files, thumbnail_files, mashups
     controller = VistrailController(*objs)
     self._controllers[locator] = controller
     self._cur_controller = controller
     return self._cur_controller
Beispiel #12
0
    def test_cache(self):
        from vistrails.core.modules.basic_modules import StandardOutput
        old_compute = StandardOutput.compute
        StandardOutput.compute = lambda s: None

        try:
            from vistrails.core.db.locator import XMLFileLocator
            from vistrails.core.vistrail.controller import VistrailController
            from vistrails.core.db.io import load_vistrail
            """Test if basic caching is working."""
            locator = XMLFileLocator(
                vistrails.core.system.vistrails_root_directory() +
                '/tests/resources/dummy.xml')
            (v, abstractions, thumbnails, mashups) = load_vistrail(locator)

            # the controller will take care of upgrades
            controller = VistrailController(v, locator, abstractions,
                                            thumbnails, mashups)
            p1 = v.getPipeline('int chain')
            n = v.get_version_number('int chain')
            controller.change_selected_version(n)
            controller.flush_delayed_actions()
            p1 = controller.current_pipeline

            view = DummyView()
            interpreter = CachedInterpreter.get()
            result = interpreter.execute(
                p1,
                locator=v,
                current_version=n,
                view=view,
            )
            # to force fresh params
            p2 = v.getPipeline('int chain')
            controller.change_selected_version(n)
            controller.flush_delayed_actions()
            p2 = controller.current_pipeline
            result = interpreter.execute(
                p2,
                locator=v,
                current_version=n,
                view=view,
            )
            self.assertEqual(len(result.modules_added), 1)
        finally:
            StandardOutput.compute = old_compute
Beispiel #13
0
    def pipeline_from_generator(variable_gen):
        # Get the original OutputPort module
        orig_controller = variable_gen._generator.controller
        base_pipeline = orig_controller.vistrail.getPipeline('dat-vars')
        if len(base_pipeline.module_list) != 1:
            raise ValueError("dat-vars version is invalid")
        output_port = base_pipeline.module_list[0]

        controller = VistrailController(Vistrail())
        # OutputPort
        operations = [('add', output_port)]
        # Rest of the pipeline
        operations += variable_gen._generator.operations
        # Connection
        connection = controller.create_connection(
            variable_gen._output_module,
            variable_gen._outputport_name,
            output_port,
            'InternalPipe')
        operations.append(('add', connection))
        # Materialize this
        action = create_action(operations)
        controller.add_new_action(action)
        version = controller.perform_action(action)
        controller.change_selected_version(version)
        assert version == controller.current_version == 1
        return controller.current_pipeline, 1
Beispiel #14
0
    def execute(self):
        """ execute() -> None
        Execute the selected pipeline with the edited aliases

        """
        aliases = {}
        for r in xrange(self.aliasTable.rowCount()):
            name = str(self.aliasTable.item(r, 0).text())
            value = str(self.aliasTable.item(r, 1).text())
            aliases[name] = value

        versionNumber = self.versions[
            self.pipelineList.currentIndex().row()][0]
        pipeline = self.vistrail.getPipelineVersionNumber(versionNumber)
        controller = VistrailController(self.vistrail)
        controller.execute_workflow_list([
            (self.vistrail.locator, versionNumber, pipeline, DummyView(),
             aliases, None)
        ])
Beispiel #15
0
    def execute(self):
        """ execute() -> None
        Execute the selected pipeline with the edited aliases

        """
        aliases = {}
        for r in xrange(self.aliasTable.rowCount()):
            name = str(self.aliasTable.item(r, 0).text())
            value = str(self.aliasTable.item(r, 1).text())
            aliases[name] = value

        versionNumber = self.versions[self.pipelineList.currentIndex().row()][0]
        pipeline = self.vistrail.getPipelineVersionNumber(versionNumber)
        controller = VistrailController(self.vistrail)
        controller.execute_workflow_list([(self.vistrail.locator,
                                           versionNumber,
                                           pipeline,
                                           DummyView(),
                                           aliases,
                                           None)])
Beispiel #16
0
def run_file(filename, tag_filter=lambda x: True):
    """Loads a .vt file and runs all the tagged versions in it.
    """
    import vistrails.core.db.io
    from vistrails.core.db.locator import FileLocator
    from vistrails.core.system import vistrails_root_directory
    from vistrails.core.vistrail.controller import VistrailController

    filename = os.path.join(vistrails_root_directory(), '..', filename)
    locator = FileLocator(filename)
    loaded_objs = vistrails.core.db.io.load_vistrail(locator)
    controller = VistrailController(loaded_objs[0], locator, *loaded_objs[1:])

    errors = []
    for version, name in controller.vistrail.get_tagMap().iteritems():
        if tag_filter(name):
            controller.change_selected_version(version)
            (result,), _ = controller.execute_current_workflow()
            if result.errors:
                errors.append(("%d: %s" % (version, name), result.errors))

    return errors
Beispiel #17
0
    def test_cache(self):
        from vistrails.core.modules.basic_modules import StandardOutput
        old_compute = StandardOutput.compute
        StandardOutput.compute = lambda s: None

        try:
            from vistrails.core.db.locator import XMLFileLocator
            from vistrails.core.vistrail.controller import VistrailController
            from vistrails.core.db.io import load_vistrail

            """Test if basic caching is working."""
            locator = XMLFileLocator(vistrails.core.system.vistrails_root_directory() +
                                '/tests/resources/dummy.xml')
            (v, abstractions, thumbnails, mashups) = load_vistrail(locator)

            # the controller will take care of upgrades
            controller = VistrailController(v, locator, abstractions,
                                            thumbnails,  mashups)
            p1 = v.getPipeline('int chain')
            n = v.get_version_number('int chain')
            controller.change_selected_version(n)
            controller.flush_delayed_actions()
            p1 = controller.current_pipeline

            view = DummyView()
            interpreter = CachedInterpreter.get()
            result = interpreter.execute(p1,
                                         locator=v,
                                         current_version=n,
                                         view=view,
                                         )
            # to force fresh params
            p2 = v.getPipeline('int chain')
            controller.change_selected_version(n)
            controller.flush_delayed_actions()
            p2 = controller.current_pipeline
            result = interpreter.execute(p2,
                                         locator=v,
                                         current_version=n,
                                         view=view,
                                         )
            self.assertEqual(len(result.modules_added), 1)
        finally:
            StandardOutput.compute = old_compute
Beispiel #18
0
 def previewChanges(self, aliases):
     print "previewChanges", aliases
     # we will just execute the pipeline with the given alias dictionary
     controller = VistrailController()
     controller.set_vistrail(self.plot_vistrail, self.locator)
     version = self.plot_vistrail.get_version_number(self.workflow_tag) if self.workflow_tag else controller.get_latest_version_in_graph()
     controller.change_selected_version(version)
     (results, _) = controller.execute_current_workflow(aliases)
Beispiel #19
0
def load_vistrail(filename, version=None):
    """Loads a Vistrail from a filename.
    """
    initialize()
    if not isinstance(filename, basestring):
        raise TypeError("load_vistrails() expects a filename, got %r" %
                        type(filename).__name__)

    locator = FileLocator(filename)
    # Copied from VistrailsApplicationInterface#open_vistrail()
    loaded_objs = vistrails.core.db.io.load_vistrail(locator)
    controller = VistrailController(loaded_objs[0], locator, *loaded_objs[1:])

    return Vistrail(controller)
Beispiel #20
0
def get_upgraded_pipeline(vistrail, version=None):
    """This is similar to Vistrail#getPipeline() but performs upgrades.

    getPipeline() can fail if the original pipeline has a different version.
    In contrast, this function will update the pipeline first using a
    controller.
    """
    if version is None:
        version = vistrail.get_latest_version()
    elif isinstance(version, (int, long)):
        pass
    elif isinstance(version, basestring):
        version = vistrail.get_tag_str(version).action_id
    else:
        raise TypeError

    controller = VistrailController(vistrail)
    controller.recompute_terse_graph()  # FIXME : this shouldn't be needed...
    controller.do_version_switch(version)
    return controller.current_pipeline
    def test_find_modules_by_type(self):
        """Tests the find_modules_by_type() function.
        """
        vistrail = Vistrail()
        controller = VistrailController(vistrail)
        controller.change_selected_version(0)

        mod1 = controller.add_module('org.vistrails.vistrails.basic',
                                     'String')
        mod2 = controller.add_module('org.vistrails.vistrails.basic',
                                     'Float')
        mod3 = controller.add_module('org.vistrails.vistrails.basic',
                                     'String')
        mod4 = controller.add_module('org.vistrails.vistrails.basic',
                                     'Integer')

        from dat.vistrails_interface import find_modules_by_type
        from vistrails.core.modules.basic_modules import Boolean, Float, String

        self.assertEqual(
            set(m.id for m in find_modules_by_type(
                controller.current_pipeline,
                [String])),
            set([mod1.id, mod3.id]))

        self.assertEqual(
            [m.id for m in find_modules_by_type(
                controller.current_pipeline,
                [Float])],
            [mod2.id, mod4.id])

        self.assertEqual(
            find_modules_by_type(
                controller.current_pipeline,
                [Boolean]),
            [])
Beispiel #22
0
    def load(self, loadworkflow=True):
        config = ConfigParser.ConfigParser()
        if config.read(self.config_file):
            if config.has_section('global'):
                if config.has_option('global', 'cellnum'):
                    self.cellnum = config.getint('global', 'cellnum')
                if config.has_option('global', 'filenum'):
                    self.filenum = config.getint('global', 'filenum')
                if config.has_option('global', 'varnum'):
                    self.varnum = config.getint('global', 'varnum')
                    print "   ------ Loaded plot %s, varnum = %d ------ " % ( self.name, self.varnum )
                if config.has_option('global', 'workflow_tag'):
                    self.workflow_tag = config.get('global', 'workflow_tag')
#                else:
#                    debug.warning("CDAT Package: file %s does not contain a required option 'workflow_tag'. Widget will not be loaded."%self.config_file)
#                    self.loaded = False
#                    return
                if config.has_option('global', 'filetypes'):
                    types = config.get('global', 'filetypes')
                    tlist = [t.strip() for t in types.split(";")]
                    for t in tlist:
                        kv = t.split(":")
                        self.filetypes[kv[0].strip()] = [v.strip() 
                                                         for v in kv[1].split(",")]
                if config.has_option('global', 'qt_filter'):
                    self.qt_filter = config.get('global', 'qt_filter')
                if config.has_option('global', 'dependencies'):
                    deps = config.get('global', 'dependencies')
                    self.dependencies = [d.strip() for d in deps.split(",")]
            
                if config.has_option('global', 'serialized_config_alias'):
                    self.serializedConfigAlias = config.get('global', 'serialized_config_alias')

                    for y in range(self.filenum):
                        self.files.append( 'Filename' + str(y+1) )
                            
                    for v in range(self.varnum):
                        self.vars.append( 'VariableName' + str(v+1) )
                        self.axes.append( 'Axes' + str(v+1) )

                    for x in range(self.cellnum):
                        section_name = 'cell' + str(x+1)
                        if config.has_section(section_name):
                            cellType = config.get(section_name, 'celltype')
                            if config.has_option(section_name, 'address_alias'):
                                self.cells.append( Cell( cellType, None, None,
                                                     config.get(section_name, 'address_alias') ) )
                            else:
                                self.cells.append(Cell( cellType,"Row"+str(x+1), "Column"+str(x+1) ) )                                                              
                else:
                    
                    for y in range(self.filenum):
                        option_name = 'filename_alias' + str(y+1)
                        if config.has_option('global', option_name):
                            self.files.append(config.get('global', option_name))
                            
                    for v in range(self.varnum):
                        option_name = 'varname_alias' + str(v+1)
                        if config.has_option('global', option_name):
                            self.vars.append(config.get('global', option_name))
                        axes_name = 'axes_alias' + str(v+1)
                        if config.has_option('global', axes_name):
                            self.axes.append(config.get('global', axes_name))
                        
                    for x in range(self.cellnum):
                        section_name = 'cell' + str(x+1)
                        if (config.has_section(section_name) and
                            config.has_option(section_name, 'celltype') and
                            config.has_option(section_name, 'row_alias') and
                            config.has_option(section_name, 'col_alias')):
                            self.cells.append(Cell(config.get(section_name, 'celltype'),
                                                   config.get(section_name, 'row_alias'),
                                                   config.get(section_name, 'col_alias')))
                
                if loadworkflow:
                    #load workflow in vistrail
                    #only if dependencies are enabled
                    manager = get_package_manager()
                    self.unsatisfied_deps = []
                    for dep in self.dependencies:
                        if not manager.has_package(dep):
                            self.unsatisfied_deps.append(dep)
                    if len(self.unsatisfied_deps) == 0:
                        try:
                            (self.plot_vistrail, abstractions , thumbnails, mashups) = load_vistrail(self.locator)
                            controller = VistrailController()
                            controller.set_vistrail(self.plot_vistrail, self.locator, 
                                                    abstractions, thumbnails,
                                                    mashups) 
    
                            self.workflow_version = self.plot_vistrail.get_version_number(self.workflow_tag) if self.workflow_tag else controller.get_latest_version_in_graph()
                            print " Loaded %s version: %s" % (  self.name, str( self.workflow_version ) )
                            controller.change_selected_version(self.workflow_version)
                            self.workflow = controller.current_pipeline
                            self.loaded = True
                        except Exception, err:
                            debug.warning( "Error loading workflow %s: %s" % ( self.name, err ) )
                            self.loaded = False
                    else:
                        debug.warning("UV-CDAT: %s widget could not be loaded \
    because it depends on packages that are not loaded:"%self.name)
                        debug.warning("  %s"%", ".join(self.unsatisfied_deps))
                        self.loaded = False
            else:
                debug.warning("UV-CDAT: file %s does not contain a 'global'\
 section. Widget will not be loaded."%self.config_file)
                self.loaded = False
 def exportMashup(filename, vtcontroller, mashuptrail, mashupversion, etype):
     """exportMashup(filename: str, vtcontroller: VistrailController, 
                     mashuptrail: Mashuptrail, type: int) -> bool 
         where etype is 
           0: include full tree 
           1: include only workflow and mashup identified by version
           2: as a link, it will point to a local file.
     """
     result = False
     if vtcontroller is not None and mashuptrail is not None:
         locator = vtcontroller.locator
         version = mashuptrail.vtVersion
         
         node = ElementTree.Element('vtlink')
     
         if isinstance(locator, DBLocator):
             node.set('host', str(locator.host))
             node.set('port', str(locator.port))
             node.set('database', str(locator.db))
             node.set('vtid', str(locator.obj_id))
         else:
             node.set('filename', str(locator.name))
             
         node.set('version', str(version))    
         node.set('execute', "True")
         node.set('forceDB', "False")
         node.set('showSpreadsheetOnly', "True")
         node.set('mashuptrail', str(mashuptrail.id))
         node.set('mashupVersion', str(mashupversion))
             
         if etype in [0,1]:
             if etype == 1: #minimal
                 pip = vtcontroller.vistrail.getPipeline(version)
                 vistrail = Vistrail()
                 id_remap = {}
                 action = vistrails.core.db.action.create_paste_action(pip, 
                                                     vistrail.idScope,
                                                     id_remap)
                 vistrail.add_action(action, 0L, 0)
                
                 tag = vtcontroller.vistrail.get_tag(version)
                 if tag is None:
                     tag = "Imported workflow"
                 vistrail.addTag(tag, action.id)
                 node.set('version', str(action.id))
                 id_scope = IdScope(1L)
                 newmashuptrail = Mashuptrail(
                                  MashupsManager.getNewMashuptrailId(), 
                                  action.id, 
                                  id_scope)
                 
                 maction = mashuptrail.actionMap[mashupversion]
                 mtag = mashuptrail.getTagForActionId(mashupversion)
                 newmashup = copy.copy(maction.mashup)
                 newmashup.remapPipelineObjects(id_remap)
                 currVersion = newmashuptrail.addVersion(
                                         newmashuptrail.getLatestVersion(),
                                         newmashup, maction.user, 
                                         maction.date)
                 newmashuptrail.currentVersion = currVersion
                 newmashuptrail.changeTag(currVersion, mtag, maction.user,
                                          maction.date)
                 newvtcontroller = BaseVistrailController(vistrail, None)
                 MashupsManager.addMashuptrailtoVistrailController(newvtcontroller,
                                                                   newmashuptrail)
                 node.set('mashuptrail', str(newmashuptrail.id))
                 node.set('mashupVersion', str(newmashuptrail.currentVersion))
             else:
                 vistrail = vtcontroller.vistrail
                 newvtcontroller = MashupsManager.copyBaseVistrailController(vtcontroller)
             
             #create temporary file
             (fd, name) = tempfile.mkstemp(prefix='vt_tmp',
                                       suffix='.vt')
             os.close(fd)
             try:
                 fileLocator = FileLocator(name)
                 newvtcontroller.write_vistrail(fileLocator)
                 contents = open(name).read()
                 vtcontent = base64.b64encode(contents)
             finally:
                 os.unlink(name)
             #if not vistrail.db_version:
             #    vistrail.db_version = currentVersion
             node.set('vtcontent',vtcontent)
             
         xmlstring = ElementTree.tostring(node)
         file_ = open(filename,'w')
         file_.write(xmlstring)
         file_.close()
         result = True
     return result
Beispiel #24
0
def execute_wf(wf, output_port):
    # Save the workflow in a temporary file
    temp_wf_fd, temp_wf = tempfile.mkstemp()

    try:
        f = open(temp_wf, 'w')
        f.write(wf)
        f.close()
        os.close(temp_wf_fd)

        # Clean the cache
        interpreter = get_default_interpreter()
        interpreter.flush()

        # Load the Pipeline from the temporary file
        vistrail = Vistrail()
        locator = XMLFileLocator(temp_wf)
        workflow = locator.load(Pipeline)

        # Build a Vistrail from this single Pipeline
        action_list = []
        for module in workflow.module_list:
            action_list.append(('add', module))
        for connection in workflow.connection_list:
            action_list.append(('add', connection))
        action = vistrails.core.db.action.create_action(action_list)

        vistrail.add_action(action, 0L)
        vistrail.update_id_scope()
        tag = 'parallel flow'
        vistrail.addTag(tag, action.id)

        # Build a controller and execute
        controller = VistrailController()
        controller.set_vistrail(vistrail, None)
        controller.change_selected_version(vistrail.get_version_number(tag))
        execution = controller.execute_current_workflow(
            custom_aliases=None,
            custom_params=None,
            extra_info=None,
            reason='API Pipeline Execution')

        # Build a list of errors
        errors = []
        pipeline = vistrail.getPipeline(tag)
        execution_errors = execution[0][0].errors
        if execution_errors:
            for key in execution_errors:
                module = pipeline.modules[key]
                msg = '%s: %s' % (module.name, execution_errors[key])
                errors.append(msg)

        # Get the execution log from the controller
        try:
            module_log = controller.log.workflow_execs[0].item_execs[0]
        except IndexError:
            errors.append("Module log not found")
            return dict(errors=errors)
        else:
            machine = controller.log.workflow_execs[0].machines[
                module_log.machine_id]
            xml_log = serialize(module_log)
            machine_log = serialize(machine)

        # Get the output value
        output = None
        serializable = None
        if not execution_errors:
            executed_module, = execution[0][0].executed
            executed_module = execution[0][0].objects[executed_module]
            try:
                output = executed_module.get_output(output_port)
            except ModuleError:
                errors.append("Output port not found: %s" % output_port)
                return dict(errors=errors)
            reg = vistrails.core.modules.module_registry.get_module_registry()
            base_classes = inspect.getmro(type(output))
            if Module in base_classes:
                serializable = reg.get_descriptor(type(output)).sigstring
                output = output.serialize()

        # Return the dictionary, that will be sent back to the client
        return dict(errors=errors,
                    output=output,
                    serializable=serializable,
                    xml_log=xml_log,
                    machine_log=machine_log)
    finally:
        os.unlink(temp_wf)
Beispiel #25
0
def execute_wf(wf, output_port):
    # Save the workflow in a temporary file
    temp_wf_fd, temp_wf = tempfile.mkstemp()

    try:
        f = open(temp_wf, 'w')
        f.write(wf)
        f.close()
        os.close(temp_wf_fd)

        # Clean the cache
        interpreter = get_default_interpreter()
        interpreter.flush()

        # Load the Pipeline from the temporary file
        vistrail = Vistrail()
        locator = XMLFileLocator(temp_wf)
        workflow = locator.load(Pipeline)

        # Build a Vistrail from this single Pipeline
        action_list = []
        for module in workflow.module_list:
            action_list.append(('add', module))
        for connection in workflow.connection_list:
            action_list.append(('add', connection))
        action = vistrails.core.db.action.create_action(action_list)

        vistrail.add_action(action, 0L)
        vistrail.update_id_scope()
        tag = 'parallel flow'
        vistrail.addTag(tag, action.id)

        # Build a controller and execute
        controller = VistrailController()
        controller.set_vistrail(vistrail, None)
        controller.change_selected_version(vistrail.get_version_number(tag))
        execution = controller.execute_current_workflow(
                custom_aliases=None,
                custom_params=None,
                extra_info=None,
                reason='API Pipeline Execution')

        # Build a list of errors
        errors = []
        pipeline = vistrail.getPipeline(tag)
        execution_errors = execution[0][0].errors
        if execution_errors:
            for key in execution_errors:
                module = pipeline.modules[key]
                msg = '%s: %s' %(module.name, execution_errors[key])
                errors.append(msg)

        # Get the execution log from the controller
        try:
            module_log = controller.log.workflow_execs[0].item_execs[0]
        except IndexError:
            errors.append("Module log not found")
            return dict(errors=errors)
        else:
            machine = controller.log.workflow_execs[0].machines[
                    module_log.machine_id]
            xml_log = serialize(module_log)
            machine_log = serialize(machine)

        # Get the output value
        output = None
        serializable = None
        if not execution_errors:
            executed_module, = execution[0][0].executed
            executed_module = execution[0][0].objects[executed_module]
            try:
                output = executed_module.get_output(output_port)
            except ModuleError:
                errors.append("Output port not found: %s" % output_port)
                return dict(errors=errors)
            reg = vistrails.core.modules.module_registry.get_module_registry()
            base_classes = inspect.getmro(type(output))
            if Module in base_classes:
                serializable = reg.get_descriptor(type(output)).sigstring
                output = output.serialize()

        # Return the dictionary, that will be sent back to the client
        return dict(errors=errors,
                    output=output,
                    serializable=serializable,
                    xml_log=xml_log,
                    machine_log=machine_log)
    finally:
        os.unlink(temp_wf)
Beispiel #26
0
    def test_infinite_looping_upgrade(self):
        """Test that circular upgrades fail gracefully"""
        # Expected actions are as follow:
        #  - loads workflow2.xml
        #  * pipeline is missing looping_fix.x version 0.1
        #  - enables looping_fix.x (version 0.2)
        #  * pipeline is still missing looping_fix.x version 0.1
        #  - runs upgrade for looping_fix.x, 0.1 -> 0.2
        #  - upgrade changes modules to package looping_fix.y version 0.1
        #  * pipeline is missing looping_fix.y version 0.1
        #  - enables looping_fix.y (version 0.2)
        #  * pipeline is still missing looping_fix.y version 0.1
        # Loop 50 times:
        #  - runs upgrade for looping_fix.y, 0.1 -> 0.2
        #  - upgrade changes modules to package looping_fix.x version 0.1
        #  * pipeline is missing looping_fix.x version 0.1
        #  - runs upgrade for looping_fix.x, 0.1 -> 0.2
        #  - upgrade changes modules to package looping_fix.y version 0.1
        #  * pipeline is missing looping_fix.y version 0.1
        # 50 calls to handle_invalid_pipeline()

        # Pre-adds packages so that the package manager can find them
        packages = ["pkg_x", "pkg_y"]
        prefix = "vistrails.tests.resources.looping_upgrades."
        pm = get_package_manager()
        for pkg in packages:
            pm.get_available_package(pkg, prefix=prefix)

        # Hooks handle_invalid_pipeline()
        from vistrails.core.vistrail.controller import VistrailController

        orig_hip = VistrailController.handle_invalid_pipeline
        count = [0]

        def new_hip(*args, **kwargs):
            count[0] += 1
            return orig_hip(*args, **kwargs)

        VistrailController.handle_invalid_pipeline = new_hip
        try:

            # Loads workflow.xml
            from vistrails.core.db.io import load_vistrail
            from vistrails.core.db.locator import FileLocator
            from vistrails.core.system import vistrails_root_directory

            locator = FileLocator(
                os.path.join(vistrails_root_directory(), "tests", "resources", "looping_upgrades", "workflow2.xml")
            )
            loaded_objs = load_vistrail(locator)
            controller = VistrailController(loaded_objs[0], locator, *loaded_objs[1:])

            # Select version (triggers all the validation/upgrade/loading)
            self.assertEqual(controller.get_latest_version_in_graph(), 1)
            try:
                controller.do_version_switch(1)
            except InvalidPipeline:
                pass
            else:
                self.fail("No InvalidPipeline exception raised!")

        # Restores handle_invalid_pipeline()
        finally:
            VistrailController.handle_invalid_pipeline = orig_hip
            # disable packages
            for pkg in reversed(packages):
                try:
                    pm.late_disable_package(pkg)
                except MissingPackage:
                    pass
        # make sure it looped 50 times before failing
        max_loops = getattr(get_vistrails_configuration(), "maxPipelineFixAttempts", 50)
        self.assertEqual(count[0], max_loops)
        # Check that original version gets selected
        self.assertEqual(1, controller.current_version)
Beispiel #27
0
def run_and_get_results(w_list,
                        parameters='',
                        update_vistrail=True,
                        extra_info=None,
                        reason='Console Mode Execution'):
    """run_and_get_results(w_list: list of (locator, version), parameters: str,
                           output_dir:str, update_vistrail: boolean,
                           extra_info:dict)
    Run all workflows in w_list, and returns an interpreter result object.
    version can be a tag name or a version id.
    
    """
    elements = parameters.split("$&$")
    aliases = {}
    params = []
    result = []
    for locator, workflow in w_list:
        (v, abstractions, thumbnails, mashups) = load_vistrail(locator)
        controller = VistrailController(v,
                                        locator,
                                        abstractions,
                                        thumbnails,
                                        mashups,
                                        auto_save=update_vistrail)
        if isinstance(workflow, basestring):
            version = v.get_version_number(workflow)
        elif isinstance(workflow, (int, long)):
            version = workflow
        elif workflow is None:
            version = controller.get_latest_version_in_graph()
        else:
            msg = "Invalid version tag or number: %s" % workflow
            raise VistrailsInternalError(msg)
        controller.change_selected_version(version)

        for e in elements:
            pos = e.find("=")
            if pos != -1:
                key = e[:pos].strip()
                value = e[pos + 1:].strip()

                if controller.current_pipeline.has_alias(key):
                    aliases[key] = value
                elif 'mashup_id' in extra_info:
                    # new-style mashups can have aliases not existing in pipeline
                    for mashuptrail in mashups:
                        if mashuptrail.vtVersion == version:
                            mashup = mashuptrail.getMashup(
                                extra_info['mashup_id'])
                            c = mashup.getAliasByName(key).component
                            params.append((c.vttype, c.vtid, value))

        if not update_vistrail:
            conf = get_vistrails_configuration()
            if conf.has('thumbs'):
                conf.thumbs.autoSave = False

        jobMonitor = controller.jobMonitor
        current_workflow = jobMonitor.currentWorkflow()
        if not current_workflow:
            for job in jobMonitor.workflows.itervalues():
                try:
                    job_version = int(job.version)
                except ValueError:
                    try:
                        job_version = v.get_version_number(job.version)
                    except KeyError:
                        # this is a PE or mashup
                        continue
                if version == job_version:
                    current_workflow = job
                    jobMonitor.startWorkflow(job)
            if not current_workflow:
                current_workflow = JobWorkflow(version)
                jobMonitor.startWorkflow(current_workflow)

        try:
            (results, _) = \
            controller.execute_current_workflow(custom_aliases=aliases,
                                                custom_params=params,
                                                extra_info=extra_info,
                                                reason=reason)
        finally:
            jobMonitor.finishWorkflow()
        new_version = controller.current_version
        if new_version != version:
            debug.log("Version '%s' (%s) was upgraded. The actual "
                      "version executed was %s" %
                      (workflow, version, new_version))
        run = results[0]
        run.workflow_info = (locator.name, new_version)
        run.pipeline = controller.current_pipeline

        if update_vistrail:
            controller.write_vistrail(locator)
        result.append(run)
        if current_workflow.jobs:
            if current_workflow.completed():
                run.job = "COMPLETED"
            else:
                run.job = "RUNNING: %s" % current_workflow.id
                for job in current_workflow.jobs.itervalues():
                    if not job.finished:
                        run.job += "\n  %s %s %s" % (job.start, job.name,
                                                     job.description())
            print run.job
    return result
Beispiel #28
0
def run_and_get_results(w_list, parameters='', workflow_info=None, 
                        update_vistrail=True, extra_info=None, 
                        reason='Console Mode Execution'):
    """run_and_get_results(w_list: list of (locator, version), parameters: str,
                           workflow_info:str, update_vistrail: boolean,
                           extra_info:dict)
    Run all workflows in w_list, and returns an interpreter result object.
    version can be a tag name or a version id.
    
    """
    elements = parameters.split("$&$")
    aliases = {}
    result = []
    for locator, workflow in w_list:
        (v, abstractions , thumbnails, mashups)  = load_vistrail(locator)
        controller = VistrailController(v, locator, abstractions, thumbnails, 
                                        mashups, auto_save=update_vistrail)
        if isinstance(workflow, basestring):
            version = v.get_version_number(workflow)
        elif isinstance(workflow, (int, long)):
            version = workflow
        elif workflow is None:
            version = controller.get_latest_version_in_graph()
        else:
            msg = "Invalid version tag or number: %s" % workflow
            raise VistrailsInternalError(msg)
        controller.change_selected_version(version)
        
        for e in elements:
            pos = e.find("=")
            if pos != -1:
                key = e[:pos].strip()
                value = e[pos+1:].strip()
            
                if controller.current_pipeline.has_alias(key):
                    aliases[key] = value
                    
        if workflow_info is not None and controller.current_pipeline is not None:
            # FIXME DAK: why is this always done?!? there is a flag for it...
            if is_running_gui():
                controller.updatePipelineScene()
                base_fname = "%s_%s_pipeline.pdf" % (locator.short_filename, version)
                filename = os.path.join(workflow_info, base_fname)
                controller.current_pipeline_scene.saveToPDF(filename)
            else:
                debug.critical("Cannot save pipeline figure when not "
                               "running in gui mode")
            base_fname = "%s_%s_pipeline.xml" % (locator.short_filename, version)
            filename = os.path.join(workflow_info, base_fname)
            vistrails.core.db.io.save_workflow(controller.current_pipeline, filename)
        if not update_vistrail:
            conf = get_vistrails_configuration()
            if conf.has('thumbs'):
                conf.thumbs.autoSave = False
        
        (results, _) = \
            controller.execute_current_workflow(custom_aliases=aliases,
                                                extra_info=extra_info,
                                                reason=reason)
        new_version = controller.current_version
        if new_version != version:
            debug.warning("Version '%s' (%s) was upgraded. The actual "
                          "version executed was %s" % \
                              (workflow, version, new_version))
        run = results[0]
        run.workflow_info = (locator.name, new_version)
        run.pipeline = controller.current_pipeline

        if update_vistrail:
            controller.write_vistrail(locator)
        result.append(run)
    return result
Beispiel #29
0
    def test_looping_pipeline_fix(self):
        """Chains upgrades and automatic package initialization."""
        # Expected actions are as follow:
        #  - loads workflow.xml
        #  * pipeline is missing looping_fix.a version 0.1
        #  - enables looping_fix.a (version 0.2)
        #  * pipeline is still missing looping_fix.a version 0.1
        #  - runs upgrade for looping_fix.a, 0.1 -> 0.2
        #  - upgrade changes modules to package looping_fix.b version 0.1
        #  * pipeline is missing looping_fix.b version 0.1
        #  - enables looping_fix.b (version 0.2)
        #  * pipeline is still missing looping_fix.b version 0.1
        #  - runs upgrade for looping_fix.b, 0.1 -> 0.2
        #  - upgrade changes modules to package looping_fix.c version 1.0
        #  * pipeline is missing looping_fix.c version 1.0
        #  - enables looping_fix.c (version 1.0)
        #  * pipeline is valid
        # 5 calls to handle_invalid_pipeline()

        # Pre-adds packages so that the package manager can find them
        packages = ["pkg_a", "pkg_b", "pkg_c"]
        prefix = "vistrails.tests.resources.looping_upgrades."
        pm = get_package_manager()
        for pkg in packages:
            pm.get_available_package(pkg, prefix=prefix)
        self.assertFalse(set(pkg.codepath for pkg in pm.enabled_package_list()).intersection(packages))

        # Hooks handle_invalid_pipeline()
        from vistrails.core.vistrail.controller import VistrailController

        orig_hip = VistrailController.handle_invalid_pipeline
        count = [0]

        def new_hip(*args, **kwargs):
            count[0] += 1
            return orig_hip(*args, **kwargs)

        VistrailController.handle_invalid_pipeline = new_hip
        try:

            # Loads workflow.xml
            from vistrails.core.db.io import load_vistrail
            from vistrails.core.db.locator import FileLocator
            from vistrails.core.system import vistrails_root_directory

            locator = FileLocator(
                os.path.join(vistrails_root_directory(), "tests", "resources", "looping_upgrades", "workflow.xml")
            )
            loaded_objs = load_vistrail(locator)
            controller = VistrailController(loaded_objs[0], locator, *loaded_objs[1:])

            # Select version (triggers all the validation/upgrade/loading)
            self.assertEqual(controller.get_latest_version_in_graph(), 1)
            controller.do_version_switch(1)

            self.assertEqual(count[0], 5)
        # Restores handle_invalid_pipeline()
        finally:
            VistrailController.handle_invalid_pipeline = orig_hip
            # disable packages
            for pkg in reversed(packages):
                try:
                    pm.late_disable_package(pkg)
                except MissingPackage:
                    pass
Beispiel #30
0
def run_and_get_results(w_list, parameters='', output_dir=None, 
                        update_vistrail=True, extra_info=None, 
                        reason='Console Mode Execution'):
    """run_and_get_results(w_list: list of (locator, version), parameters: str,
                           output_dir:str, update_vistrail: boolean,
                           extra_info:dict)
    Run all workflows in w_list, and returns an interpreter result object.
    version can be a tag name or a version id.
    
    """
    elements = parameters.split("$&$")
    aliases = {}
    params = []
    result = []
    for locator, workflow in w_list:
        (v, abstractions , thumbnails, mashups)  = load_vistrail(locator)
        controller = VistrailController(v, locator, abstractions, thumbnails,
                                        mashups, auto_save=update_vistrail)
        if isinstance(workflow, basestring):
            version = v.get_version_number(workflow)
        elif isinstance(workflow, (int, long)):
            version = workflow
        elif workflow is None:
            version = controller.get_latest_version_in_graph()
        else:
            msg = "Invalid version tag or number: %s" % workflow
            raise VistrailsInternalError(msg)
        controller.change_selected_version(version)
        
        for e in elements:
            pos = e.find("=")
            if pos != -1:
                key = e[:pos].strip()
                value = e[pos+1:].strip()
            
                if controller.current_pipeline.has_alias(key):
                    aliases[key] = value
                elif 'mashup_id' in extra_info:
                    # new-style mashups can have aliases not existing in pipeline
                    for mashuptrail in mashups:
                        if mashuptrail.vtVersion == version:
                            mashup = mashuptrail.getMashup(extra_info['mashup_id'])
                            c = mashup.getAliasByName(key).component
                            params.append((c.vttype, c.vtid, value))

        if output_dir is not None and controller.current_pipeline is not None:
            # FIXME DAK: why is this always done?!? there is a flag for it...
            if is_running_gui():
                controller.updatePipelineScene()
                base_fname = "%s_%s_pipeline.pdf" % (locator.short_filename, version)
                filename = os.path.join(output_dir, base_fname)
                controller.current_pipeline_scene.saveToPDF(filename)
            else:
                debug.critical("Cannot save pipeline figure when not "
                               "running in gui mode")
            base_fname = "%s_%s_pipeline.xml" % (locator.short_filename, version)
            filename = os.path.join(output_dir, base_fname)
            vistrails.core.db.io.save_workflow(controller.current_pipeline, filename)
        if not update_vistrail:
            conf = get_vistrails_configuration()
            if conf.has('thumbs'):
                conf.thumbs.autoSave = False
        
        jobMonitor = controller.jobMonitor
        current_workflow = jobMonitor.currentWorkflow()
        if not current_workflow:
            for job in jobMonitor.workflows.itervalues():
                try:
                    job_version = int(job.version)
                except ValueError:
                    job_version =  v.get_version_number(job.version)
                if version == job_version:
                    current_workflow = job
                    jobMonitor.startWorkflow(job)
            if not current_workflow:
                current_workflow = JobWorkflow(version)
                jobMonitor.startWorkflow(current_workflow)

        try:
            (results, _) = \
            controller.execute_current_workflow(custom_aliases=aliases,
                                                custom_params=params,
                                                extra_info=extra_info,
                                                reason=reason)
        finally:
            jobMonitor.finishWorkflow()
        new_version = controller.current_version
        if new_version != version:
            debug.log("Version '%s' (%s) was upgraded. The actual "
                      "version executed was %s" % (
                      workflow, version, new_version))
        run = results[0]
        run.workflow_info = (locator.name, new_version)
        run.pipeline = controller.current_pipeline

        if update_vistrail:
            controller.write_vistrail(locator)
        result.append(run)
        if current_workflow.jobs:
            if current_workflow.completed():
                run.job = "COMPLETED"
            else:
                run.job = "RUNNING: %s" % current_workflow.id
                for job in current_workflow.jobs.itervalues():
                    if not job.finished:
                        run.job += "\n  %s %s %s" % (job.start, job.name, job.description())
            print run.job
    return result
Beispiel #31
0
    def push_vistrail_to_repository(self, branching=False):
        """ uploads current VisTrail to web repository """

        self._repository_status['details'] = "Pushing to repository..."
        self._push_button.setEnabled(False)
        self._branch_button.setEnabled(False)
        self.update_push_information()
        try:
            # create temp file
            (fd, filename) = tempfile.mkstemp(suffix='.vt', prefix='vt_tmp')
            os.close(fd)

            # writing tmp vt and switching back to orginal vt
            locator = ZIPFileLocator(filename)
            controller = vistrails.api.get_current_controller()
            tmp_controller = VistrailController(controller.vistrail.do_copy(), 
                                                locator)
            tmp_controller.changed = True
            tmp_controller.write_vistrail(locator)

            # check if this vt is from the repository
            if controller.vistrail.get_annotation('repository_vt_id'):
                repository_vt_id = controller.vistrail.get_annotation('repository_vt_id').value
            else:
                repository_vt_id = -1

            # upload vistrail temp file to repository
            register_openers(cookiejar=self.dialog.cookiejar)
            project = self.serverCombo.itemData(self.serverCombo.currentIndex())[0]
            if project == "Default": project = ""

            params = {'vistrail_file': open(filename, 'rb'),
                      'action': 'upload',
                      'name': controller.locator.short_name,
                      'repository_vt_id': repository_vt_id if not branching else -1,
                      'is_runnable': not bool(len(self.unsupported_packages)+ \
                                              len(self.local_data_modules)),
                      'vt_id': 0,
                      'branched_from': "" if not branching else repository_vt_id,
                      'project': project,
                      'everyone_can_view': self.perm_view.checkState(),
                      'everyone_can_edit': self.perm_edit.checkState(),
                      'everyone_can_download': self.perm_download.checkState()
                     }

            upload_url = "%s/vistrails/remote_upload/" % \
                    self.config.webRepositoryURL

            datagen, headers = multipart_encode(params)
            request = urllib2.Request(upload_url, datagen, headers)
            result = urllib2.urlopen(request)
            updated_response = result.read()

            os.unlink(filename)

            if updated_response[:6] == "upload":
                # No update, just upload
                if result.code != 200:
                    self._repository_status['details'] = \
                            "Push to repository failed"
                    debug.critical("Push to repository failed (Please contact an administrator)")
                else:
                    repository_vt_id = int(updated_response[8:])
                    controller.vistrail.set_annotation('repository_vt_id',
                                                       repository_vt_id)
                    controller.vistrail.set_annotation('repository_creator',
                                                       self.dialog.loginUser)
                    # ensure that the annotations get saved
                    controller.set_changed(True)
                    self._repository_status['details'] = \
                            "Push to repository was successful"
            else:
                # update, load updated vistrail
                if result.code != 200:
                    self._repository_status['details'] = "Update Failed"
                    debug.critical("Update vistrail in web repository failed (Please contact an administrator)")
                else:
                    debug.log("getting version from web")
                    # request file to download
                    download_url = "%s/vistrails/download/%s/" % \
                            (self.config.webRepositoryURL, updated_response)

                    request = urllib2.Request(download_url)
                    result = urllib2.urlopen(request)
                    updated_file = result.read()

                    # create temp file of updated vistrail
                    (fd, updated_filename) = tempfile.mkstemp(suffix='.vtl',
                                                              prefix='vtl_tmp')
                    os.close(fd)
                    updated_vt = open(updated_filename, 'w')
                    updated_vt.write(updated_file)
                    updated_vt.close()

                    # switch vistrails to updated one
                    controller = vistrails.api.get_current_controller()

                    updated_locator = FileLocator(updated_filename)

                    (up_vistrail, abstractions, thumbnails, mashups) = \
                            load_vistrail(updated_locator)

                    # FIXME need to figure out what to do with this !!!
                    controller.set_vistrail(up_vistrail,
                                            controller.vistrail.locator,
                                            abstractions, thumbnails, mashups)

                    # update version tree drawing
                    controller.recompute_terse_graph()
                    controller.invalidate_version_tree()

                    os.remove(updated_filename)
                    os.remove(updated_filename[:-1])

                    self._repository_status['details'] = \
                            "Update to repository was successful"

        except Exception, e:
            debug.critical("An error occurred", str(e))
            self._repository_status['details'] = "An error occurred"
Beispiel #32
0
    def execute(workflowJSON):
        ''' Execute a workflow from it's JSON representation
        '''

        debug('convert json to xml')
        workflowXML = json2xml(workflowJSON)

        #temp_wf_fd, temp_wf = tempfile.mkstemp('.xml')

        debug('create temporary file')
        temp_wf_fd, temp_wf = tempfile.mkstemp()
        try:
            f = open(temp_wf, 'w')
            f.write(workflowXML)
            f.close()
            os.close(temp_wf_fd)

            #load workflow temp file into vistrails
            #vt.load_workflow(temp_wf)

            #execute workflow
            #execution = vt.execute()

            debug('Load the Pipeline from the temporary file')
            vistrail = Vistrail()
            locator = XMLFileLocator(temp_wf)
            workflow = locator.load(Pipeline)

            debug('Build a Vistrail from this single Pipeline')
            action_list = []
            for module in workflow.module_list:
                action_list.append(('add', module))
            for connection in workflow.connection_list:
                action_list.append(('add', connection))
            action = vistrails.core.db.action.create_action(action_list)

            debug('add actions')
            vistrail.add_action(action, 0L)
            vistrail.update_id_scope()
            tag = 'climatepipes'
            vistrail.addTag(tag, action.id)

            debug('Build a controller and execute')
            controller = VistrailController()
            controller.set_vistrail(vistrail, None)
            controller.change_selected_version(vistrail.get_version_number(tag))
            execution = controller.execute_current_workflow(
                    custom_aliases=None,
                    custom_params=None,
                    extra_info=None,
                    reason='API Pipeline Execution')

            debug('get result')
            execution_pipeline = execution[0][0]

            if len(execution_pipeline.errors) > 0:
                error("Executing workflow")
                for key in execution_pipeline.errors:
                    error(execution_pipeline.errors[key])
                    print execution_pipeline.errors[key]
                return None

            modules = execution_pipeline.objects

            for id, module in modules.iteritems():
                if isinstance(module, ToGeoJSON):
                    return json.dumps({'result': module.JSON, 'error': None })

        finally:
            os.unlink(temp_wf)
Beispiel #33
0
class Vistrail(object):
    """This class wraps both Vistrail and VistrailController.

    From it, you can get any pipeline from a tag name or version number.

    It has a concept of "current version", from which you can create new
    versions by performing actions.
    """
    _current_pipeline = None
    _html = None

    def __init__(self, arg=None):
        initialize()
        if arg is None:
            # Copied from VistrailsApplicationInterface#open_vistrail()
            locator = UntitledLocator()
            loaded_objs = vistrails.core.db.io.load_vistrail(locator)
            self.controller = VistrailController(loaded_objs[0], locator,
                                                 *loaded_objs[1:])
        elif isinstance(arg, (_Pipeline, Pipeline)):
            if isinstance(arg, Pipeline):
                pipeline = arg.pipeline
            else:
                pipeline = arg
            # Copied from VistrailsApplicationInterface#open_workflow()
            vistrail = _Vistrail()
            ops = []
            for module in pipeline.module_list:
                ops.append(('add', module))
            for connection in pipeline.connection_list:
                ops.append(('add', connection))
            action = vistrails.core.db.action.create_action(ops)
            vistrail.add_action(action, 0L)
            vistrail.update_id_scope()
            vistrail.change_description("Imported pipeline", 0L)
            self.controller = VistrailController(vistrail, UntitledLocator())
        elif isinstance(arg, VistrailController):
            self.controller = arg
        elif isinstance(arg, basestring):
            raise TypeError("Vistrail was constructed from %r.\n"
                            "Use load_vistrail() to get a Vistrail from a "
                            "file." % type(arg).__name__)
        else:
            raise TypeError("Vistrail was constructed from unexpected "
                            "argument type %r" % type(arg).__name__)

    def get_pipeline(self, version):
        """Returns a pipeline from a version number of tag.

        This does not change the currently selected version in this Vistrail.
        """
        vistrail = self.controller.vistrail
        if isinstance(version, (int, long)):
            if not vistrail.db_has_action_with_id(version):
                raise NoSuchVersion("Vistrail doesn't have a version %r" %
                                    version)
            return Pipeline(vistrail.getPipelineVersionNumber(version))
        elif isinstance(version, basestring):
            if not vistrail.has_tag_str(version):
                raise NoSuchVersion("Vistrail doesn't have a tag %r" % version)
            return Pipeline(vistrail.getPipelineVersionName(version))
        else:
            raise TypeError("get_pipeline() argument must be a string or "
                            "integer, not %r" % type(version).__name__)

    def select_version(self, version):
        """Sets a different version as current.

        The current workflow is accessible via current_workflow; it is the one
        that gets executed when calling execute(), and the version from which
        new versions are created if you perform actions.
        """
        vistrail = self.controller.vistrail
        if isinstance(version, (int, long)):
            if not vistrail.db_has_action_with_id(version):
                raise NoSuchVersion("Vistrail doesn't have a version %r" %
                                    version)
        elif (isinstance(version, basestring)):
            if not vistrail.has_tag_str(version):
                raise NoSuchVersion("Vistrail doesn't have a tag %r" % version)
            version = vistrail.get_tag_str(version).action_id
        else:
            raise TypeError("select_version() argument must be a string "
                            "or integer, not %r" % type(version).__name__)
        self.controller.do_version_switch(version)
        self._current_pipeline = None
        self._html = None

    def select_latest_version(self):
        """Sets the most recent version in the vistrail as current.
        """
        self.controller.do_version_switch(
                self.controller.get_latest_version_in_graph())
        self._current_pipeline = None
        self._html = None

    @property
    def current_pipeline(self):
        if self._current_pipeline is None:
            self._current_pipeline = Pipeline(
                    self.controller.current_pipeline,
                    vistrail=(self, self.current_version))
        return self._current_pipeline

    @property
    def current_version(self):
        return self.controller.current_version

    def set_tag(self, *args):
        """Sets a tag for the current or specified version.
        """
        if len(args) == 1:
            version, (tag,) = self.controller.current_version, args
        elif len(args) == 2:
            version, tag = args
        else:
            raise TypeError("set_tag() takes 1 or 2 arguments (%r given)" %
                            len(args))
        if isinstance(version, (int, long)):
            if not self.controller.vistrail.db_has_action_with_id(version):
                raise NoSuchVersion("Vistrail doesn't have a version %r" %
                                    version)
        elif isinstance(version, basestring):
            if not self.controller.vistrail.has_tag_str(version):
                raise NoSuchVersion("Vistrail doesn't have a tag %r" % version)
        else:
            raise TypeError("set_tag() expects the version to be a string or "
                            "integer, not %r" % type(version).__name__)
        self.controller.vistrail.set_tag(version, tag)

    def tag(self, tag):
        """Sets a tag for the current version.
        """
        self.set_tag(tag)

    def execute(self, *args, **kwargs):
        """Executes the current workflow.
        """
        return self.current_pipeline.execute(*args, **kwargs)

    @property
    def changed(self):
        return self.controller.changed

    # TODO : vistrail modification methods

    def __repr__(self):
        version_nb = self.controller.current_version
        if self.controller.vistrail.has_tag(version_nb):
            version = "%s (tag %s)" % (
                    version_nb,
                    self.controller.vistrail.get_tag(version_nb))
        else:
            version = version_nb
        return "<%s: %s, version %s, %s>" % (
                self.__class__.__name__,
                self.controller.name,
                version,
                ('not changed', 'changed')[self.controller.changed])

    def _repr_html_(self):
        if self._html is None:
            import cgi
            try:
                from cStringIO import StringIO
            except ImportError:
                from StringIO import StringIO

            self._html = ''
            stream = StringIO()
            self.controller.recompute_terse_graph()
            self.controller.save_version_graph(
                    stream,
                    highlight=self.controller.current_version)
            stream.seek(0)
            dot = stream.read()

            try:
                proc = subprocess.Popen(['dot', '-Tsvg'],
                                        stdin=subprocess.PIPE,
                                        stdout=subprocess.PIPE)
                svg, _ = proc.communicate(dot)
                if proc.wait() == 0:
                    self._html += svg
            except OSError:
                pass
            self._html += '<pre>' + cgi.escape(repr(self)) + '</pre>'
        return self._html
Beispiel #34
0
class Vistrail(object):
    """This class wraps both Vistrail and VistrailController.

    From it, you can get any pipeline from a tag name or version number.

    It has a concept of "current version", from which you can create new
    versions by performing actions.
    """
    _current_pipeline = None
    _html = None

    def __init__(self, arg=None):
        initialize()
        if arg is None:
            # Copied from VistrailsApplicationInterface#open_vistrail()
            locator = UntitledLocator()
            loaded_objs = vistrails.core.db.io.load_vistrail(locator)
            self.controller = VistrailController(loaded_objs[0], locator,
                                                 *loaded_objs[1:])
        elif isinstance(arg, (_Pipeline, Pipeline)):
            if isinstance(arg, Pipeline):
                pipeline = arg.pipeline
            else:
                pipeline = arg
            # Copied from VistrailsApplicationInterface#open_workflow()
            vistrail = _Vistrail()
            ops = []
            for module in pipeline.module_list:
                ops.append(('add', module))
            for connection in pipeline.connection_list:
                ops.append(('add', connection))
            action = vistrails.core.db.action.create_action(ops)
            vistrail.add_action(action, 0L)
            vistrail.update_id_scope()
            vistrail.change_description("Imported pipeline", 0L)
            self.controller = VistrailController(vistrail, UntitledLocator())
        elif isinstance(arg, VistrailController):
            self.controller = arg
        elif isinstance(arg, basestring):
            raise TypeError("Vistrail was constructed from %r.\n"
                            "Use load_vistrail() to get a Vistrail from a "
                            "file." % type(arg).__name__)
        else:
            raise TypeError("Vistrail was constructed from unexpected "
                            "argument type %r" % type(arg).__name__)

    def get_pipeline(self, version):
        """Returns a pipeline from a version number of tag.

        This does not change the currently selected version in this Vistrail.
        """
        vistrail = self.controller.vistrail
        if isinstance(version, (int, long)):
            if not vistrail.db_has_action_with_id(version):
                raise NoSuchVersion("Vistrail doesn't have a version %r" %
                                    version)
            return Pipeline(vistrail.getPipelineVersionNumber(version))
        elif isinstance(version, basestring):
            if not vistrail.has_tag_str(version):
                raise NoSuchVersion("Vistrail doesn't have a tag %r" % version)
            return Pipeline(vistrail.getPipelineVersionName(version))
        else:
            raise TypeError("get_pipeline() argument must be a string or "
                            "integer, not %r" % type(version).__name__)

    def select_version(self, version):
        """Sets a different version as current.

        The current workflow is accessible via current_workflow; it is the one
        that gets executed when calling execute(), and the version from which
        new versions are created if you perform actions.
        """
        vistrail = self.controller.vistrail
        if isinstance(version, (int, long)):
            if not vistrail.db_has_action_with_id(version):
                raise NoSuchVersion("Vistrail doesn't have a version %r" %
                                    version)
        elif (isinstance(version, basestring)):
            if not vistrail.has_tag_str(version):
                raise NoSuchVersion("Vistrail doesn't have a tag %r" % version)
            version = vistrail.get_tag_str(version).action_id
        else:
            raise TypeError("select_version() argument must be a string "
                            "or integer, not %r" % type(version).__name__)
        self.controller.do_version_switch(version)
        self._current_pipeline = None
        self._html = None

    def select_latest_version(self):
        """Sets the most recent version in the vistrail as current.
        """
        self.controller.do_version_switch(
            self.controller.get_latest_version_in_graph())
        self._current_pipeline = None
        self._html = None

    @property
    def current_pipeline(self):
        if self._current_pipeline is None:
            self._current_pipeline = Pipeline(self.controller.current_pipeline,
                                              vistrail=(self,
                                                        self.current_version))
        return self._current_pipeline

    @property
    def current_version(self):
        return self.controller.current_version

    def set_tag(self, *args):
        """Sets a tag for the current or specified version.
        """
        if len(args) == 1:
            version, (tag, ) = self.controller.current_version, args
        elif len(args) == 2:
            version, tag = args
        else:
            raise TypeError("set_tag() takes 1 or 2 arguments (%r given)" %
                            len(args))
        if isinstance(version, (int, long)):
            if not self.controller.vistrail.db_has_action_with_id(version):
                raise NoSuchVersion("Vistrail doesn't have a version %r" %
                                    version)
        elif isinstance(version, basestring):
            if not self.controller.vistrail.has_tag_str(version):
                raise NoSuchVersion("Vistrail doesn't have a tag %r" % version)
        else:
            raise TypeError("set_tag() expects the version to be a string or "
                            "integer, not %r" % type(version).__name__)
        self.controller.vistrail.set_tag(version, tag)

    def tag(self, tag):
        """Sets a tag for the current version.
        """
        self.set_tag(tag)

    def execute(self, *args, **kwargs):
        """Executes the current workflow.
        """
        return self.current_pipeline.execute(*args, **kwargs)

    @property
    def changed(self):
        return self.controller.changed

    # TODO : vistrail modification methods

    def __repr__(self):
        version_nb = self.controller.current_version
        if self.controller.vistrail.has_tag(version_nb):
            version = "%s (tag %s)" % (
                version_nb, self.controller.vistrail.get_tag(version_nb))
        else:
            version = version_nb
        return "<%s: %s, version %s, %s>" % (
            self.__class__.__name__, self.controller.name, version,
            ('not changed', 'changed')[self.controller.changed])

    def _repr_html_(self):
        if self._html is None:
            import cgi
            try:
                from cStringIO import StringIO
            except ImportError:
                from StringIO import StringIO

            self._html = ''
            stream = StringIO()
            self.controller.recompute_terse_graph()
            self.controller.save_version_graph(
                stream, highlight=self.controller.current_version)
            stream.seek(0)
            dot = stream.read()

            try:
                proc = subprocess.Popen(['dot', '-Tsvg'],
                                        stdin=subprocess.PIPE,
                                        stdout=subprocess.PIPE)
                svg, _ = proc.communicate(dot)
                if proc.wait() == 0:
                    self._html += svg
            except OSError:
                pass
            self._html += '<pre>' + cgi.escape(repr(self)) + '</pre>'
        return self._html
Beispiel #35
0
    def exportMashup(filename, vtcontroller, mashuptrail, mashupversion,
                     etype):
        """exportMashup(filename: str, vtcontroller: VistrailController, 
                        mashuptrail: Mashuptrail, type: int) -> bool 
            where etype is 
              0: include full tree 
              1: include only workflow and mashup identified by version
              2: as a link, it will point to a local file.
        """
        result = False
        if vtcontroller is not None and mashuptrail is not None:
            locator = vtcontroller.locator
            version = mashuptrail.vtVersion

            node = ElementTree.Element('vtlink')

            if isinstance(locator, DBLocator):
                node.set('host', str(locator.host))
                node.set('port', str(locator.port))
                node.set('database', str(locator.db))
                node.set('vtid', str(locator.obj_id))
            else:
                node.set('filename', str(locator.name))

            node.set('version', str(version))
            node.set('execute', "True")
            node.set('forceDB', "False")
            node.set('showSpreadsheetOnly', "True")
            node.set('mashuptrail', str(mashuptrail.id))
            node.set('mashupVersion', str(mashupversion))

            if etype in [0, 1]:
                if etype == 1:  #minimal
                    pip = vtcontroller.vistrail.getPipeline(version)
                    vistrail = Vistrail()
                    id_remap = {}
                    action = vistrails.core.db.action.create_paste_action(
                        pip, vistrail.idScope, id_remap)
                    vistrail.add_action(action, 0L, 0)

                    tag = vtcontroller.vistrail.get_tag(version)
                    if tag is None:
                        tag = "Imported workflow"
                    vistrail.addTag(tag, action.id)
                    node.set('version', str(action.id))
                    id_scope = IdScope(1L)
                    newmashuptrail = Mashuptrail(
                        MashupsManager.getNewMashuptrailId(), action.id,
                        id_scope)

                    maction = mashuptrail.actionMap[mashupversion]
                    mtag = mashuptrail.getTagForActionId(mashupversion)
                    newmashup = copy.copy(maction.mashup)
                    newmashup.remapPipelineObjects(id_remap)
                    currVersion = newmashuptrail.addVersion(
                        newmashuptrail.getLatestVersion(), newmashup,
                        maction.user, maction.date)
                    newmashuptrail.currentVersion = currVersion
                    newmashuptrail.changeTag(currVersion, mtag, maction.user,
                                             maction.date)
                    newvtcontroller = BaseVistrailController(vistrail, None)
                    MashupsManager.addMashuptrailtoVistrailController(
                        newvtcontroller, newmashuptrail)
                    node.set('mashuptrail', str(newmashuptrail.id))
                    node.set('mashupVersion',
                             str(newmashuptrail.currentVersion))
                else:
                    vistrail = vtcontroller.vistrail
                    newvtcontroller = MashupsManager.copyBaseVistrailController(
                        vtcontroller)

                #create temporary file
                (fd, name) = tempfile.mkstemp(prefix='vt_tmp', suffix='.vt')
                os.close(fd)
                try:
                    fileLocator = FileLocator(name)
                    newvtcontroller.write_vistrail(fileLocator)
                    contents = open(name).read()
                    vtcontent = base64.b64encode(contents)
                finally:
                    os.unlink(name)
                #if not vistrail.db_version:
                #    vistrail.db_version = currentVersion
                node.set('vtcontent', vtcontent)

            xmlstring = ElementTree.tostring(node)
            file_ = open(filename, 'w')
            file_.write(xmlstring)
            file_.close()
            result = True
        return result
Beispiel #36
0
sender = context.socket(zmq.PUSH)
sender.connect("tcp://{0}:{1}".format(HOST, SEND))

while True:
    # Receiving pipeline instance configuration
    data = receiver.recv()
    logging.debug('Receiving: ' + data)
    fields = data.split("|")
    filename = fields[0]
    parameter_list = ast.literal_eval(fields[1])
    inputs = ast.literal_eval(fields[2])
    outputs = ast.literal_eval(fields[3])

    locator = FileLocator(filename)
    loaded_objs = vistrails.core.db.io.load_vistrail(locator)
    controller = VistrailController(loaded_objs[0], locator, *loaded_objs[1:])
    controller.do_version_switch(controller.get_latest_version_in_graph())
    pipeline = Pipeline(controller)

    kwargs = {}
    for i in range(len(parameter_list)):
        kwargs[inputs[i]] = parameter_list[i]
    try:
        #Executing pipeline instance and retieving the result
        result = pipeline.execute(**kwargs)
        for output in outputs:
            parameter_list.append(str(result.output_port(output)))
    except:
        traceback.print_exc(file=sys.stdout)
        parameter_list.append(str(False))
        kwargs['result'] = parameter_list[-1]