def run_parameter_exploration( locator, pe_id, extra_info={}, reason="Console Mode Parameter Exploration Execution"): """run_parameter_exploration(w_list: (locator, version), pe_id: str/int, reason: str) -> (pe_id, [error msg]) Run parameter exploration in w, and returns an interpreter result object. version can be a tag name or a version id. """ if is_running_gui(): from vistrails.gui.vistrail_controller import VistrailController as \ GUIVistrailController try: (v, abstractions, thumbnails, mashups) = load_vistrail(locator) controller = GUIVistrailController(v, locator, abstractions, thumbnails, mashups) try: pe_id = int(pe_id) pe = controller.vistrail.get_paramexp(pe_id) except ValueError: pe = controller.vistrail.get_named_paramexp(pe_id) controller.change_selected_version(pe.action_id) controller.executeParameterExploration(pe, extra_info=extra_info, showProgress=False) except Exception, e: return (locator, pe_id, debug.format_exception(e), debug.format_exc())
def updateVistrail(self, url, vistrail=None): """ updateVistrail(self, string:url, Vistrail:vistrail) Update the specified entity url. Delete or reload as necessary. Need to make sure workspaces are updated if the entity is changed. """ entities = [e for e in self.entities.itervalues() if e.url == url] entity = entities[0] if len(entities) else None while entity and entity.parent: entity = entity.parent url = entity.url workspaces = [ p for p in self.workspaces if entity in self.workspaces[p] ] if entity: for p in workspaces: self.del_from_workspace(entity, p) self.delete_entity(entity) locator = BaseLocator.from_url(url) if locator.is_valid(): if not vistrail: (vistrail, abstractions, thumbnails, mashups) = load_vistrail(locator) vistrail.abstractions = abstractions vistrail.thumbnails = thumbnails vistrail.mashups = mashups entity = self.create_vistrail_entity(vistrail) for p in workspaces: self.add_to_workspace(entity, p) return entity else: # probably an unsaved vistrail pass
def updateVistrail(self, url, vistrail=None): """ updateVistrail(self, string:url, Vistrail:vistrail) Update the specified entity url. Delete or reload as necessary. Need to make sure workspaces are updated if the entity is changed. """ entities = [e for e in self.entities.itervalues() if e.url == url] entity = entities[0] if len(entities) else None while entity and entity.parent: entity = entity.parent url = entity.url workspaces = [p for p in self.workspaces if entity in self.workspaces[p]] if entity: for p in workspaces: self.del_from_workspace(entity, p) self.delete_entity(entity) locator = BaseLocator.from_url(url) if locator.is_valid(): if not vistrail: (vistrail, abstractions, thumbnails, mashups) = load_vistrail(locator) vistrail.abstractions = abstractions vistrail.thumbnails = thumbnails vistrail.mashups = mashups entity = self.create_vistrail_entity(vistrail) for p in workspaces: self.add_to_workspace(entity, p) return entity else: # probably an unsaved vistrail pass
def get_vt_graph(vt_list, tree_info, pdf=False): """get_vt_graph(vt_list: list of locator, tree_info:str) Load all vistrails in vt_list and dump their tree to tree_info. """ result = [] if is_running_gui(): from vistrails.gui.vistrail_controller import VistrailController as \ GUIVistrailController for locator in vt_list: try: (v, abstractions , thumbnails, mashups) = load_vistrail(locator) controller = GUIVistrailController(v, locator, abstractions, thumbnails, mashups) if tree_info is not None: from vistrails.gui.version_view import QVersionTreeView version_view = QVersionTreeView() version_view.scene().setupScene(controller) if pdf: base_fname = "graph_%s.pdf" % locator.short_filename filename = os.path.join(tree_info, base_fname) version_view.scene().saveToPDF(filename) else: base_fname = "graph_%s.png" % locator.short_filename filename = os.path.join(tree_info, base_fname) version_view.scene().saveToPNG(filename) del version_view result.append((True, "")) except Exception, e: result.append((False, debug.format_exception(e)))
def run_parameter_exploration(locator, pe_id, extra_info = {}, reason="Console Mode Parameter Exploration Execution"): """run_parameter_exploration(w_list: (locator, version), pe_id: str/int, reason: str) -> (pe_id, [error msg]) Run parameter exploration in w, and returns an interpreter result object. version can be a tag name or a version id. """ if is_running_gui(): from vistrails.gui.vistrail_controller import VistrailController as \ GUIVistrailController try: (v, abstractions , thumbnails, mashups) = load_vistrail(locator) controller = GUIVistrailController(v, locator, abstractions, thumbnails, mashups) try: pe_id = int(pe_id) pe = controller.vistrail.get_paramexp(pe_id) except ValueError: pe = controller.vistrail.get_named_paramexp(pe_id) controller.change_selected_version(pe.action_id) controller.executeParameterExploration(pe, extra_info=extra_info, showProgress=False) except Exception, e: return (locator, pe_id, debug.format_exception(e), debug.format_exc())
def get_vt_graph(vt_list, tree_info, pdf=False): """get_vt_graph(vt_list: list of locator, tree_info:str) Load all vistrails in vt_list and dump their tree to tree_info. """ result = [] if is_running_gui(): from vistrails.gui.vistrail_controller import VistrailController as \ GUIVistrailController for locator in vt_list: try: (v, abstractions, thumbnails, mashups) = load_vistrail(locator) controller = GUIVistrailController(v, locator, abstractions, thumbnails, mashups) if tree_info is not None: from vistrails.gui.version_view import QVersionTreeView version_view = QVersionTreeView() version_view.scene().setupScene(controller) if pdf: base_fname = "graph_%s.pdf" % locator.short_filename filename = os.path.join(tree_info, base_fname) version_view.scene().saveToPDF(filename) else: base_fname = "graph_%s.png" % locator.short_filename filename = os.path.join(tree_info, base_fname) version_view.scene().saveToPNG(filename) del version_view result.append((True, "")) except Exception, e: result.append((False, debug.format_exception(e)))
def get_wf_graph(w_list, output_dir, pdf=False): """run_and_get_results(w_list: list of (locator, version), output_dir:str, pdf:bool) Load all workflows in wf_list and dump their graph to output_dir. """ result = [] if is_running_gui(): from vistrails.gui.vistrail_controller import VistrailController as \ GUIVistrailController for locator, workflow in w_list: try: (v, abstractions, thumbnails, mashups) = load_vistrail(locator) controller = GUIVistrailController(v, locator, abstractions, thumbnails, mashups, auto_save=False) # FIXME TE: why is this needed controller.current_pipeline_view.set_controller(controller) version = None if isinstance(workflow, basestring): version = v.get_version_number(workflow) elif isinstance(workflow, (int, long)): version = workflow elif workflow is None: version = controller.get_latest_version_in_graph() else: msg = "Invalid version tag or number: %s" % workflow raise VistrailsInternalError(msg) controller.change_selected_version(version) if controller.current_pipeline is not None: controller.updatePipelineScene() if pdf: base_fname = "%s_%s_pipeline.pdf" % \ (locator.short_filename, version) filename = os.path.join(output_dir, base_fname) controller.current_pipeline_scene.saveToPDF(filename) else: base_fname = "%s_%s_pipeline.png" % \ (locator.short_filename, version) filename = os.path.join(output_dir, base_fname) controller.current_pipeline_scene.saveToPNG(filename) result.append((True, "")) except Exception, e: result.append((False, debug.format_exception(e)))
def open_vistrail(self, locator, version=None, is_abstraction=False): """open_vistrail(locator: Locator, version = None: int or str, is_abstraction: bool) opens a new vistrail from the given locator, selecting the given version. """ self.close_first_vistrail_if_necessary() if self.single_document_mode and self.currentView(): self.closeVistrail() view = self.ensureVistrail(locator) if view: if version is not None: if isinstance(version, basestring): try: version = view.vistrail.get_version_number(version) except: version = None if version is not None: view.setup_view(version) return view try: (vistrail, abstraction_files, thumbnail_files, _) = \ load_vistrail(locator, is_abstraction) result = self.set_vistrail_view(vistrail, locator, abstraction_files, thumbnail_files, version) # update collection try: vistrail.thumbnails = thumbnail_files vistrail.abstractions = abstraction_files collection = Collection.getInstance() url = locator.to_url() # create index if not exist entity = collection.fromUrl(url) if entity: # find parent vistrail while entity.parent: entity = entity.parent else: entity = collection.updateVistrail(url, vistrail) # add to relevant workspace categories collection.add_to_workspace(entity) collection.commit() except Exception, e: import traceback debug.critical('Failed to index vistrail', str(e) + traceback.format_exc()) return result
def test(self): try: errs = [] filename = os.path.join(EXAMPLES_PATH, vtfile) locator = FileLocator(os.path.abspath(filename)) (v, abstractions, thumbnails, mashups) = load_vistrail(locator) errs = vistrails.core.console_mode.run( [(locator, version)], update_vistrail=False, extra_info={'compare_thumbnails': compare_thumbnails}) if len(errs) > 0: for err in errs: print(" *** Error in %s:%s:%s -- %s" % err) self.fail(str(err)) except Exception, e: self.fail(debug.format_exception(e))
def get_wf_graph(w_list, output_dir, pdf=False): """run_and_get_results(w_list: list of (locator, version), output_dir:str, pdf:bool) Load all workflows in wf_list and dump their graph to output_dir. """ result = [] if is_running_gui(): from vistrails.gui.vistrail_controller import VistrailController as \ GUIVistrailController for locator, workflow in w_list: try: (v, abstractions , thumbnails, mashups) = load_vistrail(locator) controller = GUIVistrailController(v, locator, abstractions, thumbnails, mashups, auto_save=False) # FIXME TE: why is this needed controller.current_pipeline_view.set_controller(controller) version = None if isinstance(workflow, basestring): version = v.get_version_number(workflow) elif isinstance(workflow, (int, long)): version = workflow elif workflow is None: version = controller.get_latest_version_in_graph() else: msg = "Invalid version tag or number: %s" % workflow raise VistrailsInternalError(msg) controller.change_selected_version(version) if controller.current_pipeline is not None: controller.updatePipelineScene() if pdf: base_fname = "%s_%s_pipeline.pdf" % \ (locator.short_filename, version) filename = os.path.join(output_dir, base_fname) controller.current_pipeline_scene.saveToPDF(filename) else: base_fname = "%s_%s_pipeline.png" % \ (locator.short_filename, version) filename = os.path.join(output_dir, base_fname) controller.current_pipeline_scene.saveToPNG(filename) result.append((True, "")) except Exception, e: result.append((False, debug.format_exception(e)))
def test_cache(self): from vistrails.core.modules.basic_modules import StandardOutput old_compute = StandardOutput.compute StandardOutput.compute = lambda s: None try: from vistrails.core.db.locator import XMLFileLocator from vistrails.core.vistrail.controller import VistrailController from vistrails.core.db.io import load_vistrail """Test if basic caching is working.""" locator = XMLFileLocator( vistrails.core.system.vistrails_root_directory() + '/tests/resources/dummy.xml') (v, abstractions, thumbnails, mashups) = load_vistrail(locator) # the controller will take care of upgrades controller = VistrailController(v, locator, abstractions, thumbnails, mashups) p1 = v.getPipeline('int chain') n = v.get_version_number('int chain') controller.change_selected_version(n) controller.flush_delayed_actions() p1 = controller.current_pipeline view = DummyView() interpreter = CachedInterpreter.get() result = interpreter.execute( p1, locator=v, current_version=n, view=view, ) # to force fresh params p2 = v.getPipeline('int chain') controller.change_selected_version(n) controller.flush_delayed_actions() p2 = controller.current_pipeline result = interpreter.execute( p2, locator=v, current_version=n, view=view, ) self.assertEqual(len(result.modules_added), 1) finally: StandardOutput.compute = old_compute
def update_from_database(self, db_locator): # db_conn = db_locator.get_connection() config = (('host', db_locator._host), ('port', int(db_locator._port)), ('db', db_locator._db), ('user', db_locator._user), ('passwd', db_locator._passwd)) rows = vistrails.db.services.io.get_db_object_list(dict(config), 'vistrail') for row in rows: if row[0] in [1,]: continue kwargs = {'obj_type': 'vistrail', 'obj_id': row[0]} locator = DBLocator(*[x[1] for x in config], **kwargs) (vistrail, abstractions, thumbnails, mashups) = load_vistrail(locator) vistrail.abstractions = abstractions vistrail.thumbnails = thumbnails vistrail.mashups = mashups self.create_vistrail_entity(vistrail)
def test_cache(self): from vistrails.core.modules.basic_modules import StandardOutput old_compute = StandardOutput.compute StandardOutput.compute = lambda s: None try: from vistrails.core.db.locator import XMLFileLocator from vistrails.core.vistrail.controller import VistrailController from vistrails.core.db.io import load_vistrail """Test if basic caching is working.""" locator = XMLFileLocator(vistrails.core.system.vistrails_root_directory() + '/tests/resources/dummy.xml') (v, abstractions, thumbnails, mashups) = load_vistrail(locator) # the controller will take care of upgrades controller = VistrailController(v, locator, abstractions, thumbnails, mashups) p1 = v.getPipeline('int chain') n = v.get_version_number('int chain') controller.change_selected_version(n) controller.flush_delayed_actions() p1 = controller.current_pipeline view = DummyView() interpreter = CachedInterpreter.get() result = interpreter.execute(p1, locator=v, current_version=n, view=view, ) # to force fresh params p2 = v.getPipeline('int chain') controller.change_selected_version(n) controller.flush_delayed_actions() p2 = controller.current_pipeline result = interpreter.execute(p2, locator=v, current_version=n, view=view, ) self.assertEqual(len(result.modules_added), 1) finally: StandardOutput.compute = old_compute
def update_from_database(self, db_locator): # db_conn = db_locator.get_connection() config = {'host': db_locator._host, 'port': int(db_locator._port), 'db': db_locator._db, 'user': db_locator._user, 'passwd': db_locator._passwd} rows = vistrails.db.services.io.get_db_object_list(config, 'vistrail') for row in rows: if row[0] in [1,]: continue locator = DBLocator(config['host'], config['port'], config['db'], config['user'], config['passwd'], obj_type='vistrail', obj_id=row[0]) (vistrail, abstractions, thumbnails, mashups) = load_vistrail(locator) vistrail.abstractions = abstractions vistrail.thumbnails = thumbnails vistrail.mashups = mashups self.create_vistrail_entity(vistrail)
def newVistrail(self, recover_files=True): """ newVistrail() -> (None or QVistrailView) Create a new vistrail with no name. If user cancels process, returns None. FIXME: We should do the interactive parts separately. """ if self.single_document_mode and self.currentView(): if not self.closeVistrail(): return None if recover_files and untitled_locator().has_temporaries(): locator = copy.copy(untitled_locator()) else: locator = None try: (vistrail, abstraction_files, thumbnail_files, _) = load_vistrail(locator) except ModuleRegistryException, e: debug.critical("Module registry error for %s" % str(e.__class__.__name__), str(e))
def load(self, loadworkflow=True): config = ConfigParser.ConfigParser() if config.read(self.config_file): if config.has_section('global'): if config.has_option('global', 'cellnum'): self.cellnum = config.getint('global', 'cellnum') if config.has_option('global', 'filenum'): self.filenum = config.getint('global', 'filenum') if config.has_option('global', 'varnum'): self.varnum = config.getint('global', 'varnum') print " ------ Loaded plot %s, varnum = %d ------ " % ( self.name, self.varnum ) if config.has_option('global', 'workflow_tag'): self.workflow_tag = config.get('global', 'workflow_tag') # else: # debug.warning("CDAT Package: file %s does not contain a required option 'workflow_tag'. Widget will not be loaded."%self.config_file) # self.loaded = False # return if config.has_option('global', 'filetypes'): types = config.get('global', 'filetypes') tlist = [t.strip() for t in types.split(";")] for t in tlist: kv = t.split(":") self.filetypes[kv[0].strip()] = [v.strip() for v in kv[1].split(",")] if config.has_option('global', 'qt_filter'): self.qt_filter = config.get('global', 'qt_filter') if config.has_option('global', 'dependencies'): deps = config.get('global', 'dependencies') self.dependencies = [d.strip() for d in deps.split(",")] if config.has_option('global', 'serialized_config_alias'): self.serializedConfigAlias = config.get('global', 'serialized_config_alias') for y in range(self.filenum): self.files.append( 'Filename' + str(y+1) ) for v in range(self.varnum): self.vars.append( 'VariableName' + str(v+1) ) self.axes.append( 'Axes' + str(v+1) ) for x in range(self.cellnum): section_name = 'cell' + str(x+1) if config.has_section(section_name): cellType = config.get(section_name, 'celltype') if config.has_option(section_name, 'address_alias'): self.cells.append( Cell( cellType, None, None, config.get(section_name, 'address_alias') ) ) else: self.cells.append(Cell( cellType,"Row"+str(x+1), "Column"+str(x+1) ) ) else: for y in range(self.filenum): option_name = 'filename_alias' + str(y+1) if config.has_option('global', option_name): self.files.append(config.get('global', option_name)) for v in range(self.varnum): option_name = 'varname_alias' + str(v+1) if config.has_option('global', option_name): self.vars.append(config.get('global', option_name)) axes_name = 'axes_alias' + str(v+1) if config.has_option('global', axes_name): self.axes.append(config.get('global', axes_name)) for x in range(self.cellnum): section_name = 'cell' + str(x+1) if (config.has_section(section_name) and config.has_option(section_name, 'celltype') and config.has_option(section_name, 'row_alias') and config.has_option(section_name, 'col_alias')): self.cells.append(Cell(config.get(section_name, 'celltype'), config.get(section_name, 'row_alias'), config.get(section_name, 'col_alias'))) if loadworkflow: #load workflow in vistrail #only if dependencies are enabled manager = get_package_manager() self.unsatisfied_deps = [] for dep in self.dependencies: if not manager.has_package(dep): self.unsatisfied_deps.append(dep) if len(self.unsatisfied_deps) == 0: try: (self.plot_vistrail, abstractions , thumbnails, mashups) = load_vistrail(self.locator) controller = VistrailController() controller.set_vistrail(self.plot_vistrail, self.locator, abstractions, thumbnails, mashups) self.workflow_version = self.plot_vistrail.get_version_number(self.workflow_tag) if self.workflow_tag else controller.get_latest_version_in_graph() print " Loaded %s version: %s" % ( self.name, str( self.workflow_version ) ) controller.change_selected_version(self.workflow_version) self.workflow = controller.current_pipeline self.loaded = True except Exception, err: debug.warning( "Error loading workflow %s: %s" % ( self.name, err ) ) self.loaded = False else: debug.warning("UV-CDAT: %s widget could not be loaded \ because it depends on packages that are not loaded:"%self.name) debug.warning(" %s"%", ".join(self.unsatisfied_deps)) self.loaded = False else: debug.warning("UV-CDAT: file %s does not contain a 'global'\ section. Widget will not be loaded."%self.config_file) self.loaded = False
def run_and_get_results(w_list, parameters='', output_dir=None, update_vistrail=True, extra_info=None, reason='Console Mode Execution'): """run_and_get_results(w_list: list of (locator, version), parameters: str, output_dir:str, update_vistrail: boolean, extra_info:dict) Run all workflows in w_list, and returns an interpreter result object. version can be a tag name or a version id. """ elements = parameters.split("$&$") aliases = {} params = [] result = [] for locator, workflow in w_list: (v, abstractions , thumbnails, mashups) = load_vistrail(locator) controller = VistrailController(v, locator, abstractions, thumbnails, mashups, auto_save=update_vistrail) if isinstance(workflow, basestring): version = v.get_version_number(workflow) elif isinstance(workflow, (int, long)): version = workflow elif workflow is None: version = controller.get_latest_version_in_graph() else: msg = "Invalid version tag or number: %s" % workflow raise VistrailsInternalError(msg) controller.change_selected_version(version) for e in elements: pos = e.find("=") if pos != -1: key = e[:pos].strip() value = e[pos+1:].strip() if controller.current_pipeline.has_alias(key): aliases[key] = value elif 'mashup_id' in extra_info: # new-style mashups can have aliases not existing in pipeline for mashuptrail in mashups: if mashuptrail.vtVersion == version: mashup = mashuptrail.getMashup(extra_info['mashup_id']) c = mashup.getAliasByName(key).component params.append((c.vttype, c.vtid, value)) if output_dir is not None and controller.current_pipeline is not None: # FIXME DAK: why is this always done?!? there is a flag for it... if is_running_gui(): controller.updatePipelineScene() base_fname = "%s_%s_pipeline.pdf" % (locator.short_filename, version) filename = os.path.join(output_dir, base_fname) controller.current_pipeline_scene.saveToPDF(filename) else: debug.critical("Cannot save pipeline figure when not " "running in gui mode") base_fname = "%s_%s_pipeline.xml" % (locator.short_filename, version) filename = os.path.join(output_dir, base_fname) vistrails.core.db.io.save_workflow(controller.current_pipeline, filename) if not update_vistrail: conf = get_vistrails_configuration() if conf.has('thumbs'): conf.thumbs.autoSave = False jobMonitor = controller.jobMonitor current_workflow = jobMonitor.currentWorkflow() if not current_workflow: for job in jobMonitor.workflows.itervalues(): try: job_version = int(job.version) except ValueError: job_version = v.get_version_number(job.version) if version == job_version: current_workflow = job jobMonitor.startWorkflow(job) if not current_workflow: current_workflow = JobWorkflow(version) jobMonitor.startWorkflow(current_workflow) try: (results, _) = \ controller.execute_current_workflow(custom_aliases=aliases, custom_params=params, extra_info=extra_info, reason=reason) finally: jobMonitor.finishWorkflow() new_version = controller.current_version if new_version != version: debug.log("Version '%s' (%s) was upgraded. The actual " "version executed was %s" % ( workflow, version, new_version)) run = results[0] run.workflow_info = (locator.name, new_version) run.pipeline = controller.current_pipeline if update_vistrail: controller.write_vistrail(locator) result.append(run) if current_workflow.jobs: if current_workflow.completed(): run.job = "COMPLETED" else: run.job = "RUNNING: %s" % current_workflow.id for job in current_workflow.jobs.itervalues(): if not job.finished: run.job += "\n %s %s %s" % (job.start, job.name, job.description()) print run.job return result
def push_vistrail_to_repository(self, branching=False): """ uploads current VisTrail to web repository """ self._repository_status['details'] = "Pushing to repository..." self._push_button.setEnabled(False) self._branch_button.setEnabled(False) self.update_push_information() try: # create temp file (fd, filename) = tempfile.mkstemp(suffix='.vt', prefix='vt_tmp') os.close(fd) # writing tmp vt and switching back to orginal vt locator = ZIPFileLocator(filename) controller = vistrails.api.get_current_controller() tmp_controller = VistrailController(controller.vistrail.do_copy(), locator) tmp_controller.changed = True tmp_controller.write_vistrail(locator) # check if this vt is from the repository if controller.vistrail.get_annotation('repository_vt_id'): repository_vt_id = controller.vistrail.get_annotation('repository_vt_id').value else: repository_vt_id = -1 # upload vistrail temp file to repository register_openers(cookiejar=self.dialog.cookiejar) project = self.serverCombo.itemData(self.serverCombo.currentIndex())[0] if project == "Default": project = "" params = {'vistrail_file': open(filename, 'rb'), 'action': 'upload', 'name': controller.locator.short_name, 'repository_vt_id': repository_vt_id if not branching else -1, 'is_runnable': not bool(len(self.unsupported_packages)+ \ len(self.local_data_modules)), 'vt_id': 0, 'branched_from': "" if not branching else repository_vt_id, 'project': project, 'everyone_can_view': self.perm_view.checkState(), 'everyone_can_edit': self.perm_edit.checkState(), 'everyone_can_download': self.perm_download.checkState() } upload_url = "%s/vistrails/remote_upload/" % \ self.config.webRepositoryURL datagen, headers = multipart_encode(params) request = urllib2.Request(upload_url, datagen, headers) result = urllib2.urlopen(request) updated_response = result.read() os.unlink(filename) if updated_response[:6] == "upload": # No update, just upload if result.code != 200: self._repository_status['details'] = \ "Push to repository failed" debug.critical("Push to repository failed (Please contact an administrator)") else: repository_vt_id = int(updated_response[8:]) controller.vistrail.set_annotation('repository_vt_id', repository_vt_id) controller.vistrail.set_annotation('repository_creator', self.dialog.loginUser) # ensure that the annotations get saved controller.set_changed(True) self._repository_status['details'] = \ "Push to repository was successful" else: # update, load updated vistrail if result.code != 200: self._repository_status['details'] = "Update Failed" debug.critical("Update vistrail in web repository failed (Please contact an administrator)") else: debug.log("getting version from web") # request file to download download_url = "%s/vistrails/download/%s/" % \ (self.config.webRepositoryURL, updated_response) request = urllib2.Request(download_url) result = urllib2.urlopen(request) updated_file = result.read() # create temp file of updated vistrail (fd, updated_filename) = tempfile.mkstemp(suffix='.vtl', prefix='vtl_tmp') os.close(fd) updated_vt = open(updated_filename, 'w') updated_vt.write(updated_file) updated_vt.close() # switch vistrails to updated one controller = vistrails.api.get_current_controller() updated_locator = FileLocator(updated_filename) (up_vistrail, abstractions, thumbnails, mashups) = \ load_vistrail(updated_locator) # FIXME need to figure out what to do with this !!! controller.set_vistrail(up_vistrail, controller.vistrail.locator, abstractions, thumbnails, mashups) # update version tree drawing controller.recompute_terse_graph() controller.invalidate_version_tree() os.remove(updated_filename) os.remove(updated_filename[:-1]) self._repository_status['details'] = \ "Update to repository was successful" except Exception, e: debug.critical("An error occurred", str(e)) self._repository_status['details'] = "An error occurred"
def run_and_get_results(w_list, parameters='', workflow_info=None, update_vistrail=True, extra_info=None, reason='Console Mode Execution'): """run_and_get_results(w_list: list of (locator, version), parameters: str, workflow_info:str, update_vistrail: boolean, extra_info:dict) Run all workflows in w_list, and returns an interpreter result object. version can be a tag name or a version id. """ elements = parameters.split("$&$") aliases = {} result = [] for locator, workflow in w_list: (v, abstractions , thumbnails, mashups) = load_vistrail(locator) controller = VistrailController(v, locator, abstractions, thumbnails, mashups, auto_save=update_vistrail) if isinstance(workflow, basestring): version = v.get_version_number(workflow) elif isinstance(workflow, (int, long)): version = workflow elif workflow is None: version = controller.get_latest_version_in_graph() else: msg = "Invalid version tag or number: %s" % workflow raise VistrailsInternalError(msg) controller.change_selected_version(version) for e in elements: pos = e.find("=") if pos != -1: key = e[:pos].strip() value = e[pos+1:].strip() if controller.current_pipeline.has_alias(key): aliases[key] = value if workflow_info is not None and controller.current_pipeline is not None: # FIXME DAK: why is this always done?!? there is a flag for it... if is_running_gui(): controller.updatePipelineScene() base_fname = "%s_%s_pipeline.pdf" % (locator.short_filename, version) filename = os.path.join(workflow_info, base_fname) controller.current_pipeline_scene.saveToPDF(filename) else: debug.critical("Cannot save pipeline figure when not " "running in gui mode") base_fname = "%s_%s_pipeline.xml" % (locator.short_filename, version) filename = os.path.join(workflow_info, base_fname) vistrails.core.db.io.save_workflow(controller.current_pipeline, filename) if not update_vistrail: conf = get_vistrails_configuration() if conf.has('thumbs'): conf.thumbs.autoSave = False (results, _) = \ controller.execute_current_workflow(custom_aliases=aliases, extra_info=extra_info, reason=reason) new_version = controller.current_version if new_version != version: debug.warning("Version '%s' (%s) was upgraded. The actual " "version executed was %s" % \ (workflow, version, new_version)) run = results[0] run.workflow_info = (locator.name, new_version) run.pipeline = controller.current_pipeline if update_vistrail: controller.write_vistrail(locator) result.append(run) return result
def test_looping_pipeline_fix(self): """Chains upgrades and automatic package initialization.""" # Expected actions are as follow: # - loads workflow.xml # * pipeline is missing looping_fix.a version 0.1 # - enables looping_fix.a (version 0.2) # * pipeline is still missing looping_fix.a version 0.1 # - runs upgrade for looping_fix.a, 0.1 -> 0.2 # - upgrade changes modules to package looping_fix.b version 0.1 # * pipeline is missing looping_fix.b version 0.1 # - enables looping_fix.b (version 0.2) # * pipeline is still missing looping_fix.b version 0.1 # - runs upgrade for looping_fix.b, 0.1 -> 0.2 # - upgrade changes modules to package looping_fix.c version 1.0 # * pipeline is missing looping_fix.c version 1.0 # - enables looping_fix.c (version 1.0) # * pipeline is valid # 5 calls to handle_invalid_pipeline() # Pre-adds packages so that the package manager can find them packages = ["pkg_a", "pkg_b", "pkg_c"] prefix = "vistrails.tests.resources.looping_upgrades." pm = get_package_manager() for pkg in packages: pm.get_available_package(pkg, prefix=prefix) self.assertFalse(set(pkg.codepath for pkg in pm.enabled_package_list()).intersection(packages)) # Hooks handle_invalid_pipeline() from vistrails.core.vistrail.controller import VistrailController orig_hip = VistrailController.handle_invalid_pipeline count = [0] def new_hip(*args, **kwargs): count[0] += 1 return orig_hip(*args, **kwargs) VistrailController.handle_invalid_pipeline = new_hip try: # Loads workflow.xml from vistrails.core.db.io import load_vistrail from vistrails.core.db.locator import FileLocator from vistrails.core.system import vistrails_root_directory locator = FileLocator( os.path.join(vistrails_root_directory(), "tests", "resources", "looping_upgrades", "workflow.xml") ) loaded_objs = load_vistrail(locator) controller = VistrailController(loaded_objs[0], locator, *loaded_objs[1:]) # Select version (triggers all the validation/upgrade/loading) self.assertEqual(controller.get_latest_version_in_graph(), 1) controller.do_version_switch(1) self.assertEqual(count[0], 5) # Restores handle_invalid_pipeline() finally: VistrailController.handle_invalid_pipeline = orig_hip # disable packages for pkg in reversed(packages): try: pm.late_disable_package(pkg) except MissingPackage: pass
def push_vistrail_to_repository(self, branching=False): """ uploads current VisTrail to web repository """ self._repository_status['details'] = "Pushing to repository..." self._push_button.setEnabled(False) self._branch_button.setEnabled(False) self.update_push_information() try: # create temp file (fd, filename) = tempfile.mkstemp(suffix='.vt', prefix='vt_tmp') os.close(fd) # writing tmp vt and switching back to original vt locator = ZIPFileLocator(filename) controller = vistrails.api.get_current_controller() controller.write_vistrail(locator, export=True) # check if this vt is from the repository if controller.vistrail.get_annotation('repository_vt_id'): repository_vt_id = controller.vistrail.get_annotation( 'repository_vt_id').value else: repository_vt_id = -1 # upload vistrail temp file to repository register_openers(cookiejar=self.dialog.cookiejar) project = self.serverCombo.itemData( self.serverCombo.currentIndex())[0] if project == "Default": project = "" params = {'vistrail_file': open(filename, 'rb'), 'action': 'upload', 'name': controller.locator.short_name, 'repository_vt_id': repository_vt_id if not branching else -1, 'is_runnable': not bool(len(self.unsupported_packages)+ \ len(self.local_data_modules)), 'vt_id': 0, 'branched_from': "" if not branching else repository_vt_id, 'project': project, 'everyone_can_view': self.perm_view.checkState(), 'everyone_can_edit': self.perm_edit.checkState(), 'everyone_can_download': self.perm_download.checkState() } upload_url = "%s/vistrails/remote_upload/" % \ self.config.webRepositoryURL datagen, headers = multipart_encode(params) request = urllib2.Request(upload_url, datagen, headers) result = urllib2.urlopen(request) updated_response = result.read() os.unlink(filename) if updated_response[:6] == "upload": # No update, just upload if result.code != 200: self._repository_status['details'] = \ "Push to repository failed" debug.critical( "Push to repository failed (Please contact an administrator)" ) else: repository_vt_id = int(updated_response[8:]) controller.vistrail.set_annotation('repository_vt_id', repository_vt_id) controller.vistrail.set_annotation('repository_creator', self.dialog.loginUser) # ensure that the annotations get saved controller.set_changed(True) self._repository_status['details'] = \ "Push to repository was successful" else: # update, load updated vistrail if result.code != 200: self._repository_status['details'] = "Update Failed" debug.critical( "Update vistrail in web repository failed (Please contact an administrator)" ) else: debug.log("getting version from web") # request file to download download_url = "%s/vistrails/download/%s/" % \ (self.config.webRepositoryURL, updated_response) request = urllib2.Request(download_url) result = urllib2.urlopen(request) updated_file = result.read() # create temp file of updated vistrail (fd, updated_filename) = tempfile.mkstemp(suffix='.vtl', prefix='vtl_tmp') os.close(fd) updated_vt = open(updated_filename, 'w') updated_vt.write(updated_file) updated_vt.close() # switch vistrails to updated one controller = vistrails.api.get_current_controller() updated_locator = FileLocator(updated_filename) (up_vistrail, abstractions, thumbnails, mashups) = \ load_vistrail(updated_locator) # FIXME need to figure out what to do with this !!! current_version = controller.current_version controller.set_vistrail(up_vistrail, controller.vistrail.locator, abstractions, thumbnails, mashups) controller.change_selected_version(current_version) # update version tree drawing controller.recompute_terse_graph() controller.invalidate_version_tree() os.remove(updated_filename) os.remove(updated_filename[:-1]) self._repository_status['details'] = \ "Update to repository was successful" except Exception, e: debug.critical("An error occurred", e) self._repository_status['details'] = "An error occurred"
def run_and_get_results(w_list, parameters='', update_vistrail=True, extra_info=None, reason='Console Mode Execution'): """run_and_get_results(w_list: list of (locator, version), parameters: str, output_dir:str, update_vistrail: boolean, extra_info:dict) Run all workflows in w_list, and returns an interpreter result object. version can be a tag name or a version id. """ elements = parameters.split("$&$") aliases = {} params = [] result = [] for locator, workflow in w_list: (v, abstractions, thumbnails, mashups) = load_vistrail(locator) controller = VistrailController(v, locator, abstractions, thumbnails, mashups, auto_save=update_vistrail) if isinstance(workflow, basestring): version = v.get_version_number(workflow) elif isinstance(workflow, (int, long)): version = workflow elif workflow is None: version = controller.get_latest_version_in_graph() else: msg = "Invalid version tag or number: %s" % workflow raise VistrailsInternalError(msg) controller.change_selected_version(version) for e in elements: pos = e.find("=") if pos != -1: key = e[:pos].strip() value = e[pos + 1:].strip() if controller.current_pipeline.has_alias(key): aliases[key] = value elif 'mashup_id' in extra_info: # new-style mashups can have aliases not existing in pipeline for mashuptrail in mashups: if mashuptrail.vtVersion == version: mashup = mashuptrail.getMashup( extra_info['mashup_id']) c = mashup.getAliasByName(key).component params.append((c.vttype, c.vtid, value)) if not update_vistrail: conf = get_vistrails_configuration() if conf.has('thumbs'): conf.thumbs.autoSave = False jobMonitor = controller.jobMonitor current_workflow = jobMonitor.currentWorkflow() if not current_workflow: for job in jobMonitor.workflows.itervalues(): try: job_version = int(job.version) except ValueError: try: job_version = v.get_version_number(job.version) except KeyError: # this is a PE or mashup continue if version == job_version: current_workflow = job jobMonitor.startWorkflow(job) if not current_workflow: current_workflow = JobWorkflow(version) jobMonitor.startWorkflow(current_workflow) try: (results, _) = \ controller.execute_current_workflow(custom_aliases=aliases, custom_params=params, extra_info=extra_info, reason=reason) finally: jobMonitor.finishWorkflow() new_version = controller.current_version if new_version != version: debug.log("Version '%s' (%s) was upgraded. The actual " "version executed was %s" % (workflow, version, new_version)) run = results[0] run.workflow_info = (locator.name, new_version) run.pipeline = controller.current_pipeline if update_vistrail: controller.write_vistrail(locator) result.append(run) if current_workflow.jobs: if current_workflow.completed(): run.job = "COMPLETED" else: run.job = "RUNNING: %s" % current_workflow.id for job in current_workflow.jobs.itervalues(): if not job.finished: run.job += "\n %s %s %s" % (job.start, job.name, job.description()) print run.job return result
def test_infinite_looping_upgrade(self): """Test that circular upgrades fail gracefully""" # Expected actions are as follow: # - loads workflow2.xml # * pipeline is missing looping_fix.x version 0.1 # - enables looping_fix.x (version 0.2) # * pipeline is still missing looping_fix.x version 0.1 # - runs upgrade for looping_fix.x, 0.1 -> 0.2 # - upgrade changes modules to package looping_fix.y version 0.1 # * pipeline is missing looping_fix.y version 0.1 # - enables looping_fix.y (version 0.2) # * pipeline is still missing looping_fix.y version 0.1 # Loop 50 times: # - runs upgrade for looping_fix.y, 0.1 -> 0.2 # - upgrade changes modules to package looping_fix.x version 0.1 # * pipeline is missing looping_fix.x version 0.1 # - runs upgrade for looping_fix.x, 0.1 -> 0.2 # - upgrade changes modules to package looping_fix.y version 0.1 # * pipeline is missing looping_fix.y version 0.1 # 50 calls to handle_invalid_pipeline() # Pre-adds packages so that the package manager can find them packages = ["pkg_x", "pkg_y"] prefix = "vistrails.tests.resources.looping_upgrades." pm = get_package_manager() for pkg in packages: pm.get_available_package(pkg, prefix=prefix) # Hooks handle_invalid_pipeline() from vistrails.core.vistrail.controller import VistrailController orig_hip = VistrailController.handle_invalid_pipeline count = [0] def new_hip(*args, **kwargs): count[0] += 1 return orig_hip(*args, **kwargs) VistrailController.handle_invalid_pipeline = new_hip try: # Loads workflow.xml from vistrails.core.db.io import load_vistrail from vistrails.core.db.locator import FileLocator from vistrails.core.system import vistrails_root_directory locator = FileLocator( os.path.join(vistrails_root_directory(), "tests", "resources", "looping_upgrades", "workflow2.xml") ) loaded_objs = load_vistrail(locator) controller = VistrailController(loaded_objs[0], locator, *loaded_objs[1:]) # Select version (triggers all the validation/upgrade/loading) self.assertEqual(controller.get_latest_version_in_graph(), 1) try: controller.do_version_switch(1) except InvalidPipeline: pass else: self.fail("No InvalidPipeline exception raised!") # Restores handle_invalid_pipeline() finally: VistrailController.handle_invalid_pipeline = orig_hip # disable packages for pkg in reversed(packages): try: pm.late_disable_package(pkg) except MissingPackage: pass # make sure it looped 50 times before failing max_loops = getattr(get_vistrails_configuration(), "maxPipelineFixAttempts", 50) self.assertEqual(count[0], max_loops) # Check that original version gets selected self.assertEqual(1, controller.current_version)