def read_vistrail_from_db(db_connection, abs_id, version): import vistrails.db.services.io from vistrails.core.vistrail.vistrail import Vistrail vistrail = vistrails.db.services.io.open_vistrail_from_db( db_connection, abs_id, version) Vistrail.convert(vistrail) return vistrail
def generate_vtl(locator, version, pipeline, execute=False, forceDB=False, showSpreadsheetOnly=False, embedWorkflow=False): """generate_vtl(locator:DBLocator or XMLLocator, version: str, pipeline:Pipeline, execute:boolean, forceDB:boolean, showspreadsheetOnly:boolean, embedWorkflow: boolean) -> str It generates the contents of a .vtl file with the information given. """ node = ElementTree.Element('vtlink') if isinstance(locator, DBLocator): node.set('host', str(locator.host)) node.set('port', str(locator.port)) node.set('database', str(locator.db)) node.set('vtid', str(locator.obj_id)) elif locator is not None: node.set('filename', str(locator.name)) node.set('version', str(version)) node.set('execute', str(execute)) node.set('forceDB', str(forceDB)) node.set('showSpreadsheetOnly', str(showSpreadsheetOnly)) if embedWorkflow == True: vistrail = Vistrail() action_list = [] for module in pipeline.module_list: action_list.append(('add', module)) for connection in pipeline.connection_list: action_list.append(('add', connection)) action = vistrails.core.db.action.create_action(action_list) vistrail.add_action(action, 0L) vistrail.addTag("Imported workflow", action.id) if not forceDB: node.set('version', str(action.id)) if not vistrail.db_version: vistrail.db_version = currentVersion pipxmlstr = io.serialize(vistrail) vtcontent = base64.b64encode(pipxmlstr) node.set('vtcontent', vtcontent) return ElementTree.tostring(node)
def make_pipeline(self): """Creates an example pipeline that is used to conduct tests. """ vistrail = Vistrail() controller = VistrailController(vistrail) controller.change_selected_version(0) # 0 1 2 7 8 # | / \ \ / # 3 4 5 9 # | / \ # 6 10 11 modules = [ controller.add_module('org.vistrails.vistrails.basic', 'String') for i in xrange(12) ] def connect(outmod, inmod): controller.add_connection(modules[outmod].id, 'value', modules[inmod].id, 'value') for (outmod, inmod) in [(0, 3), (1, 4), (1, 5), (5, 6), (7, 9), (8, 9), (9, 10), (9, 11)]: connect(outmod, inmod) return controller, modules
def pipeline_from_generator(variable_gen): # Get the original OutputPort module orig_controller = variable_gen._generator.controller base_pipeline = orig_controller.vistrail.getPipeline('dat-vars') if len(base_pipeline.module_list) != 1: raise ValueError("dat-vars version is invalid") output_port = base_pipeline.module_list[0] controller = VistrailController(Vistrail()) # OutputPort operations = [('add', output_port)] # Rest of the pipeline operations += variable_gen._generator.operations # Connection connection = controller.create_connection( variable_gen._output_module, variable_gen._outputport_name, output_port, 'InternalPipe') operations.append(('add', connection)) # Materialize this action = create_action(operations) controller.add_new_action(action) version = controller.perform_action(action) controller.change_selected_version(version) assert version == controller.current_version == 1 return controller.current_pipeline, 1
def test_find_modules_by_type(self): """Tests the find_modules_by_type() function. """ vistrail = Vistrail() controller = VistrailController(vistrail) controller.change_selected_version(0) mod1 = controller.add_module('org.vistrails.vistrails.basic', 'String') mod2 = controller.add_module('org.vistrails.vistrails.basic', 'Float') mod3 = controller.add_module('org.vistrails.vistrails.basic', 'String') mod4 = controller.add_module('org.vistrails.vistrails.basic', 'Integer') from dat.vistrails_interface import find_modules_by_type from vistrails.core.modules.basic_modules import Boolean, Float, String self.assertEqual( set(m.id for m in find_modules_by_type(controller.current_pipeline, [String])), set([mod1.id, mod3.id])) self.assertEqual([ m.id for m in find_modules_by_type(controller.current_pipeline, [Float]) ], [mod2.id, mod4.id]) self.assertEqual( find_modules_by_type(controller.current_pipeline, [Boolean]), [])
def open_workflow(self, locator, version=None): self.close_first_vistrail_if_necessary() if self.single_document_mode and self.currentView(): self.closeVistrail() vistrail = Vistrail() try: if locator is not None: workflow = locator.load(Pipeline) action_list = [] for module in workflow.module_list: action_list.append(('add', module)) for connection in workflow.connection_list: action_list.append(('add', connection)) action = vistrails.core.db.action.create_action(action_list) vistrail.add_action(action, 0L) vistrail.update_id_scope() vistrail.addTag("Imported workflow", action.id) # FIXME might need different locator? except ModuleRegistryException, e: msg = ('Cannot find module "%s" in package "%s". ' 'Make sure package is ' 'enabled in the Preferences dialog.' % \ (e._name, e._identifier)) debug.critical(msg)
def open_workflow(self, locator): if isinstance(locator, basestring): locator = BaseLocator.from_url(locator) vistrail = Vistrail() try: if locator is None: return False if locator is not None: workflow = locator.load(Pipeline) action_list = [] for module in workflow.module_list: action_list.append(('add', module)) for connection in workflow.connection_list: action_list.append(('add', connection)) action = vistrails.core.db.action.create_action(action_list) vistrail.add_action(action, 0L) vistrail.update_id_scope() vistrail.addTag("Imported workflow", action.id) # FIXME might need different locator? controller = self.add_vistrail(vistrail, locator) except VistrailsDBException, e: import traceback debug.critical("Exception from the database", traceback.format_exc()) return None
def load_vistrail(locator, is_abstraction=False): from vistrails.core.vistrail.vistrail import Vistrail abstraction_files = [] thumbnail_files = [] mashups = [] vistrail = None if locator is None: vistrail = Vistrail() else: res = locator.load() if type(res) == type(SaveBundle(None)): vistrail = res.vistrail abstraction_files.extend(res.abstractions) thumbnail_files.extend(res.thumbnails) mashups.extend(res.mashups) else: vistrail = res vistrail.is_abstraction = is_abstraction return (vistrail, abstraction_files, thumbnail_files, mashups)
def build_widget(self): layout = QtGui.QVBoxLayout() layout.setMargin(0) layout.setSpacing(0) self.query_controller = QueryController(self) self.vt_controller = VistrailController(auto_save=False) self.p_controller = VistrailController(Vistrail(), auto_save=False) self.connect(self.p_controller, QtCore.SIGNAL('vistrailChanged()'), self.vistrailChanged) self.query_box = QQueryBox() self.query_box.set_controller(self.query_controller) layout.addWidget(self.query_box) self.stacked_widget = QtGui.QStackedWidget() self.pipeline_view = QQueryPipelineView() self.p_controller.current_pipeline_view = self.pipeline_view self.pipeline_view.set_controller(self.p_controller) self.pipeline_view.set_query_controller(self.query_controller) QQueryView.VISUAL_SEARCH_VIEW = \ self.stacked_widget.addWidget(self.pipeline_view) self.global_result_view = QQueryResultGlobalView() QQueryView.GLOBAL_RESULT_VIEW = \ self.stacked_widget.addWidget(self.global_result_view) self.version_result_view = QQueryResultVersionView() self.connect(self.version_result_view.scene(), QtCore.SIGNAL('versionSelected(int,bool,bool,bool,bool)'), self.result_version_selected) # self.version_result_view.set_controller(self.vt_controller) QQueryView.VERSION_RESULT_VIEW = \ self.stacked_widget.addWidget(self.version_result_view) self.workflow_result_view = QQueryResultWorkflowView() # self.workflow_result_view.set_controller(self.vt_controller) QQueryView.WORKFLOW_RESULT_VIEW = \ self.stacked_widget.addWidget(self.workflow_result_view) self.stacked_widget.setCurrentWidget(self.pipeline_view) layout.addWidget(self.stacked_widget) self.setLayout(layout) self.current_display = QQueryView.VISUAL_SEARCH_VIEW self.current_result_view = QQueryView.VERSION_RESULT_VIEW
def generate_vtl(locator,version,pipeline,execute=False,forceDB=False, showSpreadsheetOnly=False,embedWorkflow=False): """generate_vtl(locator:DBLocator or XMLLocator, version: str, pipeline:Pipeline, execute:boolean, forceDB:boolean, showspreadsheetOnly:boolean, embedWorkflow: boolean) -> str It generates the contents of a .vtl file with the information given. """ node = ElementTree.Element('vtlink') if isinstance(locator, DBLocator): node.set('host', str(locator.host)) node.set('port', str(locator.port)) node.set('database', str(locator.db)) node.set('vtid', str(locator.obj_id)) elif locator is not None: node.set('filename', str(locator.name)) node.set('version', str(version)) node.set('execute', str(execute)) node.set('forceDB', str(forceDB)) node.set('showSpreadsheetOnly', str(showSpreadsheetOnly)) if embedWorkflow == True: vistrail = Vistrail() action_list = [] for module in pipeline.module_list: action_list.append(('add', module)) for connection in pipeline.connection_list: action_list.append(('add', connection)) action = vistrails.core.db.action.create_action(action_list) vistrail.add_action(action, 0L) vistrail.addTag("Imported workflow", action.id) if not forceDB: node.set('version', str(action.id)) if not vistrail.db_version: vistrail.db_version = currentVersion pipxmlstr = io.serialize(vistrail) vtcontent = base64.b64encode(pipxmlstr) node.set('vtcontent',vtcontent) return ElementTree.tostring(node)
def execute_wf(wf, output_port): # Save the workflow in a temporary file temp_wf_fd, temp_wf = tempfile.mkstemp() try: f = open(temp_wf, 'w') f.write(wf) f.close() os.close(temp_wf_fd) # Clean the cache interpreter = get_default_interpreter() interpreter.flush() # Load the Pipeline from the temporary file vistrail = Vistrail() locator = XMLFileLocator(temp_wf) workflow = locator.load(Pipeline) # Build a Vistrail from this single Pipeline action_list = [] for module in workflow.module_list: action_list.append(('add', module)) for connection in workflow.connection_list: action_list.append(('add', connection)) action = vistrails.core.db.action.create_action(action_list) vistrail.add_action(action, 0L) vistrail.update_id_scope() tag = 'parallel flow' vistrail.addTag(tag, action.id) # Build a controller and execute controller = VistrailController() controller.set_vistrail(vistrail, None) controller.change_selected_version(vistrail.get_version_number(tag)) execution = controller.execute_current_workflow( custom_aliases=None, custom_params=None, extra_info=None, reason='API Pipeline Execution') # Build a list of errors errors = [] pipeline = vistrail.getPipeline(tag) execution_errors = execution[0][0].errors if execution_errors: for key in execution_errors: module = pipeline.modules[key] msg = '%s: %s' % (module.name, execution_errors[key]) errors.append(msg) # Get the execution log from the controller try: module_log = controller.log.workflow_execs[0].item_execs[0] except IndexError: errors.append("Module log not found") return dict(errors=errors) else: machine = controller.log.workflow_execs[0].machines[ module_log.machine_id] xml_log = serialize(module_log) machine_log = serialize(machine) # Get the output value output = None serializable = None if not execution_errors: executed_module, = execution[0][0].executed executed_module = execution[0][0].objects[executed_module] try: output = executed_module.get_output(output_port) except ModuleError: errors.append("Output port not found: %s" % output_port) return dict(errors=errors) reg = vistrails.core.modules.module_registry.get_module_registry() base_classes = inspect.getmro(type(output)) if Module in base_classes: serializable = reg.get_descriptor(type(output)).sigstring output = output.serialize() # Return the dictionary, that will be sent back to the client return dict(errors=errors, output=output, serializable=serializable, xml_log=xml_log, machine_log=machine_log) finally: os.unlink(temp_wf)
def read_vistrail_from_db(db_connection, abs_id, version): import vistrails.db.services.io from vistrails.core.vistrail.vistrail import Vistrail vistrail = vistrails.db.services.io.open_vistrail_from_db(db_connection, abs_id, version) Vistrail.convert(vistrail) return vistrail
def read_vistrail(vt_fname): import vistrails.db.services.io from vistrails.core.vistrail.vistrail import Vistrail vistrail = vistrails.db.services.io.open_vistrail_from_xml(vt_fname) Vistrail.convert(vistrail) return vistrail
def execute(workflowJSON): ''' Execute a workflow from it's JSON representation ''' debug('convert json to xml') workflowXML = json2xml(workflowJSON) #temp_wf_fd, temp_wf = tempfile.mkstemp('.xml') debug('create temporary file') temp_wf_fd, temp_wf = tempfile.mkstemp() try: f = open(temp_wf, 'w') f.write(workflowXML) f.close() os.close(temp_wf_fd) #load workflow temp file into vistrails #vt.load_workflow(temp_wf) #execute workflow #execution = vt.execute() debug('Load the Pipeline from the temporary file') vistrail = Vistrail() locator = XMLFileLocator(temp_wf) workflow = locator.load(Pipeline) debug('Build a Vistrail from this single Pipeline') action_list = [] for module in workflow.module_list: action_list.append(('add', module)) for connection in workflow.connection_list: action_list.append(('add', connection)) action = vistrails.core.db.action.create_action(action_list) debug('add actions') vistrail.add_action(action, 0L) vistrail.update_id_scope() tag = 'climatepipes' vistrail.addTag(tag, action.id) debug('Build a controller and execute') controller = VistrailController() controller.set_vistrail(vistrail, None) controller.change_selected_version(vistrail.get_version_number(tag)) execution = controller.execute_current_workflow( custom_aliases=None, custom_params=None, extra_info=None, reason='API Pipeline Execution') debug('get result') execution_pipeline = execution[0][0] if len(execution_pipeline.errors) > 0: error("Executing workflow") for key in execution_pipeline.errors: error(execution_pipeline.errors[key]) print execution_pipeline.errors[key] return None modules = execution_pipeline.objects for id, module in modules.iteritems(): if isinstance(module, ToGeoJSON): return json.dumps({'result': module.JSON, 'error': None }) finally: os.unlink(temp_wf)
def execute_wf(wf, output_port): # Save the workflow in a temporary file temp_wf_fd, temp_wf = tempfile.mkstemp() try: f = open(temp_wf, 'w') f.write(wf) f.close() os.close(temp_wf_fd) # Clean the cache interpreter = get_default_interpreter() interpreter.flush() # Load the Pipeline from the temporary file vistrail = Vistrail() locator = XMLFileLocator(temp_wf) workflow = locator.load(Pipeline) # Build a Vistrail from this single Pipeline action_list = [] for module in workflow.module_list: action_list.append(('add', module)) for connection in workflow.connection_list: action_list.append(('add', connection)) action = vistrails.core.db.action.create_action(action_list) vistrail.add_action(action, 0L) vistrail.update_id_scope() tag = 'parallel flow' vistrail.addTag(tag, action.id) # Build a controller and execute controller = VistrailController() controller.set_vistrail(vistrail, None) controller.change_selected_version(vistrail.get_version_number(tag)) execution = controller.execute_current_workflow( custom_aliases=None, custom_params=None, extra_info=None, reason='API Pipeline Execution') # Build a list of errors errors = [] pipeline = vistrail.getPipeline(tag) execution_errors = execution[0][0].errors if execution_errors: for key in execution_errors: module = pipeline.modules[key] msg = '%s: %s' %(module.name, execution_errors[key]) errors.append(msg) # Get the execution log from the controller try: module_log = controller.log.workflow_execs[0].item_execs[0] except IndexError: errors.append("Module log not found") return dict(errors=errors) else: machine = controller.log.workflow_execs[0].machines[ module_log.machine_id] xml_log = serialize(module_log) machine_log = serialize(machine) # Get the output value output = None serializable = None if not execution_errors: executed_module, = execution[0][0].executed executed_module = execution[0][0].objects[executed_module] try: output = executed_module.get_output(output_port) except ModuleError: errors.append("Output port not found: %s" % output_port) return dict(errors=errors) reg = vistrails.core.modules.module_registry.get_module_registry() base_classes = inspect.getmro(type(output)) if Module in base_classes: serializable = reg.get_descriptor(type(output)).sigstring output = output.serialize() # Return the dictionary, that will be sent back to the client return dict(errors=errors, output=output, serializable=serializable, xml_log=xml_log, machine_log=machine_log) finally: os.unlink(temp_wf)
def exportMashup(filename, vtcontroller, mashuptrail, mashupversion, etype): """exportMashup(filename: str, vtcontroller: VistrailController, mashuptrail: Mashuptrail, type: int) -> bool where etype is 0: include full tree 1: include only workflow and mashup identified by version 2: as a link, it will point to a local file. """ result = False if vtcontroller is not None and mashuptrail is not None: locator = vtcontroller.locator version = mashuptrail.vtVersion node = ElementTree.Element('vtlink') if isinstance(locator, DBLocator): node.set('host', str(locator.host)) node.set('port', str(locator.port)) node.set('database', str(locator.db)) node.set('vtid', str(locator.obj_id)) else: node.set('filename', str(locator.name)) node.set('version', str(version)) node.set('execute', "True") node.set('forceDB', "False") node.set('showSpreadsheetOnly', "True") node.set('mashuptrail', str(mashuptrail.id)) node.set('mashupVersion', str(mashupversion)) if etype in [0,1]: if etype == 1: #minimal pip = vtcontroller.vistrail.getPipeline(version) vistrail = Vistrail() id_remap = {} action = vistrails.core.db.action.create_paste_action(pip, vistrail.idScope, id_remap) vistrail.add_action(action, 0L, 0) tag = vtcontroller.vistrail.get_tag(version) if tag is None: tag = "Imported workflow" vistrail.addTag(tag, action.id) node.set('version', str(action.id)) id_scope = IdScope(1L) newmashuptrail = Mashuptrail( MashupsManager.getNewMashuptrailId(), action.id, id_scope) maction = mashuptrail.actionMap[mashupversion] mtag = mashuptrail.getTagForActionId(mashupversion) newmashup = copy.copy(maction.mashup) newmashup.remapPipelineObjects(id_remap) currVersion = newmashuptrail.addVersion( newmashuptrail.getLatestVersion(), newmashup, maction.user, maction.date) newmashuptrail.currentVersion = currVersion newmashuptrail.changeTag(currVersion, mtag, maction.user, maction.date) newvtcontroller = BaseVistrailController(vistrail, None) MashupsManager.addMashuptrailtoVistrailController(newvtcontroller, newmashuptrail) node.set('mashuptrail', str(newmashuptrail.id)) node.set('mashupVersion', str(newmashuptrail.currentVersion)) else: vistrail = vtcontroller.vistrail newvtcontroller = MashupsManager.copyBaseVistrailController(vtcontroller) #create temporary file (fd, name) = tempfile.mkstemp(prefix='vt_tmp', suffix='.vt') os.close(fd) try: fileLocator = FileLocator(name) newvtcontroller.write_vistrail(fileLocator) contents = open(name).read() vtcontent = base64.b64encode(contents) finally: os.unlink(name) #if not vistrail.db_version: # vistrail.db_version = currentVersion node.set('vtcontent',vtcontent) xmlstring = ElementTree.tostring(node) file_ = open(filename,'w') file_.write(xmlstring) file_.close() result = True return result
def exportMashup(filename, vtcontroller, mashuptrail, mashupversion, etype): """exportMashup(filename: str, vtcontroller: VistrailController, mashuptrail: Mashuptrail, type: int) -> bool where etype is 0: include full tree 1: include only workflow and mashup identified by version 2: as a link, it will point to a local file. """ result = False if vtcontroller is not None and mashuptrail is not None: locator = vtcontroller.locator version = mashuptrail.vtVersion node = ElementTree.Element('vtlink') if isinstance(locator, DBLocator): node.set('host', str(locator.host)) node.set('port', str(locator.port)) node.set('database', str(locator.db)) node.set('vtid', str(locator.obj_id)) else: node.set('filename', str(locator.name)) node.set('version', str(version)) node.set('execute', "True") node.set('forceDB', "False") node.set('showSpreadsheetOnly', "True") node.set('mashuptrail', str(mashuptrail.id)) node.set('mashupVersion', str(mashupversion)) if etype in [0, 1]: if etype == 1: #minimal pip = vtcontroller.vistrail.getPipeline(version) vistrail = Vistrail() id_remap = {} action = vistrails.core.db.action.create_paste_action( pip, vistrail.idScope, id_remap) vistrail.add_action(action, 0L, 0) tag = vtcontroller.vistrail.get_tag(version) if tag is None: tag = "Imported workflow" vistrail.addTag(tag, action.id) node.set('version', str(action.id)) id_scope = IdScope(1L) newmashuptrail = Mashuptrail( MashupsManager.getNewMashuptrailId(), action.id, id_scope) maction = mashuptrail.actionMap[mashupversion] mtag = mashuptrail.getTagForActionId(mashupversion) newmashup = copy.copy(maction.mashup) newmashup.remapPipelineObjects(id_remap) currVersion = newmashuptrail.addVersion( newmashuptrail.getLatestVersion(), newmashup, maction.user, maction.date) newmashuptrail.currentVersion = currVersion newmashuptrail.changeTag(currVersion, mtag, maction.user, maction.date) newvtcontroller = BaseVistrailController(vistrail, None) MashupsManager.addMashuptrailtoVistrailController( newvtcontroller, newmashuptrail) node.set('mashuptrail', str(newmashuptrail.id)) node.set('mashupVersion', str(newmashuptrail.currentVersion)) else: vistrail = vtcontroller.vistrail newvtcontroller = MashupsManager.copyBaseVistrailController( vtcontroller) #create temporary file (fd, name) = tempfile.mkstemp(prefix='vt_tmp', suffix='.vt') os.close(fd) try: fileLocator = FileLocator(name) newvtcontroller.write_vistrail(fileLocator) contents = open(name).read() vtcontent = base64.b64encode(contents) finally: os.unlink(name) #if not vistrail.db_version: # vistrail.db_version = currentVersion node.set('vtcontent', vtcontent) xmlstring = ElementTree.tostring(node) file_ = open(filename, 'w') file_.write(xmlstring) file_.close() result = True return result