def run_and_get_results(w_list, parameters='', output_dir=None, update_vistrail=True, extra_info=None, reason='Console Mode Execution'): """run_and_get_results(w_list: list of (locator, version), parameters: str, output_dir:str, update_vistrail: boolean, extra_info:dict) Run all workflows in w_list, and returns an interpreter result object. version can be a tag name or a version id. """ elements = parameters.split("$&$") aliases = {} params = [] result = [] for locator, workflow in w_list: (v, abstractions , thumbnails, mashups) = load_vistrail(locator) controller = VistrailController(v, locator, abstractions, thumbnails, mashups, auto_save=update_vistrail) if isinstance(workflow, basestring): version = v.get_version_number(workflow) elif isinstance(workflow, (int, long)): version = workflow elif workflow is None: version = controller.get_latest_version_in_graph() else: msg = "Invalid version tag or number: %s" % workflow raise VistrailsInternalError(msg) controller.change_selected_version(version) for e in elements: pos = e.find("=") if pos != -1: key = e[:pos].strip() value = e[pos+1:].strip() if controller.current_pipeline.has_alias(key): aliases[key] = value elif 'mashup_id' in extra_info: # new-style mashups can have aliases not existing in pipeline for mashuptrail in mashups: if mashuptrail.vtVersion == version: mashup = mashuptrail.getMashup(extra_info['mashup_id']) c = mashup.getAliasByName(key).component params.append((c.vttype, c.vtid, value)) if output_dir is not None and controller.current_pipeline is not None: # FIXME DAK: why is this always done?!? there is a flag for it... if is_running_gui(): controller.updatePipelineScene() base_fname = "%s_%s_pipeline.pdf" % (locator.short_filename, version) filename = os.path.join(output_dir, base_fname) controller.current_pipeline_scene.saveToPDF(filename) else: debug.critical("Cannot save pipeline figure when not " "running in gui mode") base_fname = "%s_%s_pipeline.xml" % (locator.short_filename, version) filename = os.path.join(output_dir, base_fname) vistrails.core.db.io.save_workflow(controller.current_pipeline, filename) if not update_vistrail: conf = get_vistrails_configuration() if conf.has('thumbs'): conf.thumbs.autoSave = False jobMonitor = controller.jobMonitor current_workflow = jobMonitor.currentWorkflow() if not current_workflow: for job in jobMonitor.workflows.itervalues(): try: job_version = int(job.version) except ValueError: job_version = v.get_version_number(job.version) if version == job_version: current_workflow = job jobMonitor.startWorkflow(job) if not current_workflow: current_workflow = JobWorkflow(version) jobMonitor.startWorkflow(current_workflow) try: (results, _) = \ controller.execute_current_workflow(custom_aliases=aliases, custom_params=params, extra_info=extra_info, reason=reason) finally: jobMonitor.finishWorkflow() new_version = controller.current_version if new_version != version: debug.log("Version '%s' (%s) was upgraded. The actual " "version executed was %s" % ( workflow, version, new_version)) run = results[0] run.workflow_info = (locator.name, new_version) run.pipeline = controller.current_pipeline if update_vistrail: controller.write_vistrail(locator) result.append(run) if current_workflow.jobs: if current_workflow.completed(): run.job = "COMPLETED" else: run.job = "RUNNING: %s" % current_workflow.id for job in current_workflow.jobs.itervalues(): if not job.finished: run.job += "\n %s %s %s" % (job.start, job.name, job.description()) print run.job return result
def exportMashup(filename, vtcontroller, mashuptrail, mashupversion, etype): """exportMashup(filename: str, vtcontroller: VistrailController, mashuptrail: Mashuptrail, type: int) -> bool where etype is 0: include full tree 1: include only workflow and mashup identified by version 2: as a link, it will point to a local file. """ result = False if vtcontroller is not None and mashuptrail is not None: locator = vtcontroller.locator version = mashuptrail.vtVersion node = ElementTree.Element('vtlink') if isinstance(locator, DBLocator): node.set('host', str(locator.host)) node.set('port', str(locator.port)) node.set('database', str(locator.db)) node.set('vtid', str(locator.obj_id)) else: node.set('filename', str(locator.name)) node.set('version', str(version)) node.set('execute', "True") node.set('forceDB', "False") node.set('showSpreadsheetOnly', "True") node.set('mashuptrail', str(mashuptrail.id)) node.set('mashupVersion', str(mashupversion)) if etype in [0,1]: if etype == 1: #minimal pip = vtcontroller.vistrail.getPipeline(version) vistrail = Vistrail() id_remap = {} action = vistrails.core.db.action.create_paste_action(pip, vistrail.idScope, id_remap) vistrail.add_action(action, 0L, 0) tag = vtcontroller.vistrail.get_tag(version) if tag is None: tag = "Imported workflow" vistrail.addTag(tag, action.id) node.set('version', str(action.id)) id_scope = IdScope(1L) newmashuptrail = Mashuptrail( MashupsManager.getNewMashuptrailId(), action.id, id_scope) maction = mashuptrail.actionMap[mashupversion] mtag = mashuptrail.getTagForActionId(mashupversion) newmashup = copy.copy(maction.mashup) newmashup.remapPipelineObjects(id_remap) currVersion = newmashuptrail.addVersion( newmashuptrail.getLatestVersion(), newmashup, maction.user, maction.date) newmashuptrail.currentVersion = currVersion newmashuptrail.changeTag(currVersion, mtag, maction.user, maction.date) newvtcontroller = BaseVistrailController(vistrail, None) MashupsManager.addMashuptrailtoVistrailController(newvtcontroller, newmashuptrail) node.set('mashuptrail', str(newmashuptrail.id)) node.set('mashupVersion', str(newmashuptrail.currentVersion)) else: vistrail = vtcontroller.vistrail newvtcontroller = MashupsManager.copyBaseVistrailController(vtcontroller) #create temporary file (fd, name) = tempfile.mkstemp(prefix='vt_tmp', suffix='.vt') os.close(fd) try: fileLocator = FileLocator(name) newvtcontroller.write_vistrail(fileLocator) contents = open(name).read() vtcontent = base64.b64encode(contents) finally: os.unlink(name) #if not vistrail.db_version: # vistrail.db_version = currentVersion node.set('vtcontent',vtcontent) xmlstring = ElementTree.tostring(node) file_ = open(filename,'w') file_.write(xmlstring) file_.close() result = True return result
def run_and_get_results(w_list, parameters='', update_vistrail=True, extra_info=None, reason='Console Mode Execution'): """run_and_get_results(w_list: list of (locator, version), parameters: str, output_dir:str, update_vistrail: boolean, extra_info:dict) Run all workflows in w_list, and returns an interpreter result object. version can be a tag name or a version id. """ elements = parameters.split("$&$") aliases = {} params = [] result = [] for locator, workflow in w_list: (v, abstractions, thumbnails, mashups) = load_vistrail(locator) controller = VistrailController(v, locator, abstractions, thumbnails, mashups, auto_save=update_vistrail) if isinstance(workflow, basestring): version = v.get_version_number(workflow) elif isinstance(workflow, (int, long)): version = workflow elif workflow is None: version = controller.get_latest_version_in_graph() else: msg = "Invalid version tag or number: %s" % workflow raise VistrailsInternalError(msg) controller.change_selected_version(version) for e in elements: pos = e.find("=") if pos != -1: key = e[:pos].strip() value = e[pos + 1:].strip() if controller.current_pipeline.has_alias(key): aliases[key] = value elif 'mashup_id' in extra_info: # new-style mashups can have aliases not existing in pipeline for mashuptrail in mashups: if mashuptrail.vtVersion == version: mashup = mashuptrail.getMashup( extra_info['mashup_id']) c = mashup.getAliasByName(key).component params.append((c.vttype, c.vtid, value)) if not update_vistrail: conf = get_vistrails_configuration() if conf.has('thumbs'): conf.thumbs.autoSave = False jobMonitor = controller.jobMonitor current_workflow = jobMonitor.currentWorkflow() if not current_workflow: for job in jobMonitor.workflows.itervalues(): try: job_version = int(job.version) except ValueError: try: job_version = v.get_version_number(job.version) except KeyError: # this is a PE or mashup continue if version == job_version: current_workflow = job jobMonitor.startWorkflow(job) if not current_workflow: current_workflow = JobWorkflow(version) jobMonitor.startWorkflow(current_workflow) try: (results, _) = \ controller.execute_current_workflow(custom_aliases=aliases, custom_params=params, extra_info=extra_info, reason=reason) finally: jobMonitor.finishWorkflow() new_version = controller.current_version if new_version != version: debug.log("Version '%s' (%s) was upgraded. The actual " "version executed was %s" % (workflow, version, new_version)) run = results[0] run.workflow_info = (locator.name, new_version) run.pipeline = controller.current_pipeline if update_vistrail: controller.write_vistrail(locator) result.append(run) if current_workflow.jobs: if current_workflow.completed(): run.job = "COMPLETED" else: run.job = "RUNNING: %s" % current_workflow.id for job in current_workflow.jobs.itervalues(): if not job.finished: run.job += "\n %s %s %s" % (job.start, job.name, job.description()) print run.job return result
def exportMashup(filename, vtcontroller, mashuptrail, mashupversion, etype): """exportMashup(filename: str, vtcontroller: VistrailController, mashuptrail: Mashuptrail, type: int) -> bool where etype is 0: include full tree 1: include only workflow and mashup identified by version 2: as a link, it will point to a local file. """ result = False if vtcontroller is not None and mashuptrail is not None: locator = vtcontroller.locator version = mashuptrail.vtVersion node = ElementTree.Element('vtlink') if isinstance(locator, DBLocator): node.set('host', str(locator.host)) node.set('port', str(locator.port)) node.set('database', str(locator.db)) node.set('vtid', str(locator.obj_id)) else: node.set('filename', str(locator.name)) node.set('version', str(version)) node.set('execute', "True") node.set('forceDB', "False") node.set('showSpreadsheetOnly', "True") node.set('mashuptrail', str(mashuptrail.id)) node.set('mashupVersion', str(mashupversion)) if etype in [0, 1]: if etype == 1: #minimal pip = vtcontroller.vistrail.getPipeline(version) vistrail = Vistrail() id_remap = {} action = vistrails.core.db.action.create_paste_action( pip, vistrail.idScope, id_remap) vistrail.add_action(action, 0L, 0) tag = vtcontroller.vistrail.get_tag(version) if tag is None: tag = "Imported workflow" vistrail.addTag(tag, action.id) node.set('version', str(action.id)) id_scope = IdScope(1L) newmashuptrail = Mashuptrail( MashupsManager.getNewMashuptrailId(), action.id, id_scope) maction = mashuptrail.actionMap[mashupversion] mtag = mashuptrail.getTagForActionId(mashupversion) newmashup = copy.copy(maction.mashup) newmashup.remapPipelineObjects(id_remap) currVersion = newmashuptrail.addVersion( newmashuptrail.getLatestVersion(), newmashup, maction.user, maction.date) newmashuptrail.currentVersion = currVersion newmashuptrail.changeTag(currVersion, mtag, maction.user, maction.date) newvtcontroller = BaseVistrailController(vistrail, None) MashupsManager.addMashuptrailtoVistrailController( newvtcontroller, newmashuptrail) node.set('mashuptrail', str(newmashuptrail.id)) node.set('mashupVersion', str(newmashuptrail.currentVersion)) else: vistrail = vtcontroller.vistrail newvtcontroller = MashupsManager.copyBaseVistrailController( vtcontroller) #create temporary file (fd, name) = tempfile.mkstemp(prefix='vt_tmp', suffix='.vt') os.close(fd) try: fileLocator = FileLocator(name) newvtcontroller.write_vistrail(fileLocator) contents = open(name).read() vtcontent = base64.b64encode(contents) finally: os.unlink(name) #if not vistrail.db_version: # vistrail.db_version = currentVersion node.set('vtcontent', vtcontent) xmlstring = ElementTree.tostring(node) file_ = open(filename, 'w') file_.write(xmlstring) file_.close() result = True return result
def run_and_get_results(w_list, parameters='', workflow_info=None, update_vistrail=True, extra_info=None, reason='Console Mode Execution'): """run_and_get_results(w_list: list of (locator, version), parameters: str, workflow_info:str, update_vistrail: boolean, extra_info:dict) Run all workflows in w_list, and returns an interpreter result object. version can be a tag name or a version id. """ elements = parameters.split("$&$") aliases = {} result = [] for locator, workflow in w_list: (v, abstractions , thumbnails, mashups) = load_vistrail(locator) controller = VistrailController(v, locator, abstractions, thumbnails, mashups, auto_save=update_vistrail) if isinstance(workflow, basestring): version = v.get_version_number(workflow) elif isinstance(workflow, (int, long)): version = workflow elif workflow is None: version = controller.get_latest_version_in_graph() else: msg = "Invalid version tag or number: %s" % workflow raise VistrailsInternalError(msg) controller.change_selected_version(version) for e in elements: pos = e.find("=") if pos != -1: key = e[:pos].strip() value = e[pos+1:].strip() if controller.current_pipeline.has_alias(key): aliases[key] = value if workflow_info is not None and controller.current_pipeline is not None: # FIXME DAK: why is this always done?!? there is a flag for it... if is_running_gui(): controller.updatePipelineScene() base_fname = "%s_%s_pipeline.pdf" % (locator.short_filename, version) filename = os.path.join(workflow_info, base_fname) controller.current_pipeline_scene.saveToPDF(filename) else: debug.critical("Cannot save pipeline figure when not " "running in gui mode") base_fname = "%s_%s_pipeline.xml" % (locator.short_filename, version) filename = os.path.join(workflow_info, base_fname) vistrails.core.db.io.save_workflow(controller.current_pipeline, filename) if not update_vistrail: conf = get_vistrails_configuration() if conf.has('thumbs'): conf.thumbs.autoSave = False (results, _) = \ controller.execute_current_workflow(custom_aliases=aliases, extra_info=extra_info, reason=reason) new_version = controller.current_version if new_version != version: debug.warning("Version '%s' (%s) was upgraded. The actual " "version executed was %s" % \ (workflow, version, new_version)) run = results[0] run.workflow_info = (locator.name, new_version) run.pipeline = controller.current_pipeline if update_vistrail: controller.write_vistrail(locator) result.append(run) return result
def push_vistrail_to_repository(self, branching=False): """ uploads current VisTrail to web repository """ self._repository_status['details'] = "Pushing to repository..." self._push_button.setEnabled(False) self._branch_button.setEnabled(False) self.update_push_information() try: # create temp file (fd, filename) = tempfile.mkstemp(suffix='.vt', prefix='vt_tmp') os.close(fd) # writing tmp vt and switching back to orginal vt locator = ZIPFileLocator(filename) controller = vistrails.api.get_current_controller() tmp_controller = VistrailController(controller.vistrail.do_copy(), locator) tmp_controller.changed = True tmp_controller.write_vistrail(locator) # check if this vt is from the repository if controller.vistrail.get_annotation('repository_vt_id'): repository_vt_id = controller.vistrail.get_annotation('repository_vt_id').value else: repository_vt_id = -1 # upload vistrail temp file to repository register_openers(cookiejar=self.dialog.cookiejar) project = self.serverCombo.itemData(self.serverCombo.currentIndex())[0] if project == "Default": project = "" params = {'vistrail_file': open(filename, 'rb'), 'action': 'upload', 'name': controller.locator.short_name, 'repository_vt_id': repository_vt_id if not branching else -1, 'is_runnable': not bool(len(self.unsupported_packages)+ \ len(self.local_data_modules)), 'vt_id': 0, 'branched_from': "" if not branching else repository_vt_id, 'project': project, 'everyone_can_view': self.perm_view.checkState(), 'everyone_can_edit': self.perm_edit.checkState(), 'everyone_can_download': self.perm_download.checkState() } upload_url = "%s/vistrails/remote_upload/" % \ self.config.webRepositoryURL datagen, headers = multipart_encode(params) request = urllib2.Request(upload_url, datagen, headers) result = urllib2.urlopen(request) updated_response = result.read() os.unlink(filename) if updated_response[:6] == "upload": # No update, just upload if result.code != 200: self._repository_status['details'] = \ "Push to repository failed" debug.critical("Push to repository failed (Please contact an administrator)") else: repository_vt_id = int(updated_response[8:]) controller.vistrail.set_annotation('repository_vt_id', repository_vt_id) controller.vistrail.set_annotation('repository_creator', self.dialog.loginUser) # ensure that the annotations get saved controller.set_changed(True) self._repository_status['details'] = \ "Push to repository was successful" else: # update, load updated vistrail if result.code != 200: self._repository_status['details'] = "Update Failed" debug.critical("Update vistrail in web repository failed (Please contact an administrator)") else: debug.log("getting version from web") # request file to download download_url = "%s/vistrails/download/%s/" % \ (self.config.webRepositoryURL, updated_response) request = urllib2.Request(download_url) result = urllib2.urlopen(request) updated_file = result.read() # create temp file of updated vistrail (fd, updated_filename) = tempfile.mkstemp(suffix='.vtl', prefix='vtl_tmp') os.close(fd) updated_vt = open(updated_filename, 'w') updated_vt.write(updated_file) updated_vt.close() # switch vistrails to updated one controller = vistrails.api.get_current_controller() updated_locator = FileLocator(updated_filename) (up_vistrail, abstractions, thumbnails, mashups) = \ load_vistrail(updated_locator) # FIXME need to figure out what to do with this !!! controller.set_vistrail(up_vistrail, controller.vistrail.locator, abstractions, thumbnails, mashups) # update version tree drawing controller.recompute_terse_graph() controller.invalidate_version_tree() os.remove(updated_filename) os.remove(updated_filename[:-1]) self._repository_status['details'] = \ "Update to repository was successful" except Exception, e: debug.critical("An error occurred", str(e)) self._repository_status['details'] = "An error occurred"