def run_and_get_results(w_list, parameters='', output_dir=None,
update_vistrail=True, extra_info=None,
reason='Console Mode Execution'):
"""run_and_get_results(w_list: list of (locator, version), parameters: str,
output_dir:str, update_vistrail: boolean,
extra_info:dict)
Run all workflows in w_list, and returns an interpreter result object.
version can be a tag name or a version id.
"""
elements = parameters.split("$&$")
aliases = {}
params = []
result = []
for locator, workflow in w_list:
(v, abstractions , thumbnails, mashups) = load_vistrail(locator)
controller = VistrailController(v, locator, abstractions, thumbnails,
mashups, auto_save=update_vistrail)
if isinstance(workflow, basestring):
version = v.get_version_number(workflow)
elif isinstance(workflow, (int, long)):
version = workflow
elif workflow is None:
version = controller.get_latest_version_in_graph()
else:
msg = "Invalid version tag or number: %s" % workflow
raise VistrailsInternalError(msg)
controller.change_selected_version(version)
for e in elements:
pos = e.find("=")
if pos != -1:
key = e[:pos].strip()
value = e[pos+1:].strip()
if controller.current_pipeline.has_alias(key):
aliases[key] = value
elif 'mashup_id' in extra_info:
# new-style mashups can have aliases not existing in pipeline
for mashuptrail in mashups:
if mashuptrail.vtVersion == version:
mashup = mashuptrail.getMashup(extra_info['mashup_id'])
c = mashup.getAliasByName(key).component
params.append((c.vttype, c.vtid, value))
if output_dir is not None and controller.current_pipeline is not None:
# FIXME DAK: why is this always done?!? there is a flag for it...
if is_running_gui():
controller.updatePipelineScene()
base_fname = "%s_%s_pipeline.pdf" % (locator.short_filename, version)
filename = os.path.join(output_dir, base_fname)
controller.current_pipeline_scene.saveToPDF(filename)
else:
debug.critical("Cannot save pipeline figure when not "
"running in gui mode")
base_fname = "%s_%s_pipeline.xml" % (locator.short_filename, version)
filename = os.path.join(output_dir, base_fname)
vistrails.core.db.io.save_workflow(controller.current_pipeline, filename)
if not update_vistrail:
conf = get_vistrails_configuration()
if conf.has('thumbs'):
conf.thumbs.autoSave = False
jobMonitor = controller.jobMonitor
current_workflow = jobMonitor.currentWorkflow()
if not current_workflow:
for job in jobMonitor.workflows.itervalues():
try:
job_version = int(job.version)
except ValueError:
job_version = v.get_version_number(job.version)
if version == job_version:
current_workflow = job
jobMonitor.startWorkflow(job)
if not current_workflow:
current_workflow = JobWorkflow(version)
jobMonitor.startWorkflow(current_workflow)
try:
(results, _) = \
controller.execute_current_workflow(custom_aliases=aliases,
custom_params=params,
extra_info=extra_info,
reason=reason)
finally:
jobMonitor.finishWorkflow()
new_version = controller.current_version
if new_version != version:
debug.log("Version '%s' (%s) was upgraded. The actual "
"version executed was %s" % (
workflow, version, new_version))
run = results[0]
run.workflow_info = (locator.name, new_version)
run.pipeline = controller.current_pipeline
if update_vistrail:
controller.write_vistrail(locator)
result.append(run)
if current_workflow.jobs:
if current_workflow.completed():
run.job = "COMPLETED"
else:
run.job = "RUNNING: %s" % current_workflow.id
for job in current_workflow.jobs.itervalues():
if not job.finished:
run.job += "\n %s %s %s" % (job.start, job.name, job.description())
print run.job
return result
def exportMashup(filename, vtcontroller, mashuptrail, mashupversion, etype):
"""exportMashup(filename: str, vtcontroller: VistrailController,
mashuptrail: Mashuptrail, type: int) -> bool
where etype is
0: include full tree
1: include only workflow and mashup identified by version
2: as a link, it will point to a local file.
"""
result = False
if vtcontroller is not None and mashuptrail is not None:
locator = vtcontroller.locator
version = mashuptrail.vtVersion
node = ElementTree.Element('vtlink')
if isinstance(locator, DBLocator):
node.set('host', str(locator.host))
node.set('port', str(locator.port))
node.set('database', str(locator.db))
node.set('vtid', str(locator.obj_id))
else:
node.set('filename', str(locator.name))
node.set('version', str(version))
node.set('execute', "True")
node.set('forceDB', "False")
node.set('showSpreadsheetOnly', "True")
node.set('mashuptrail', str(mashuptrail.id))
node.set('mashupVersion', str(mashupversion))
if etype in [0,1]:
if etype == 1: #minimal
pip = vtcontroller.vistrail.getPipeline(version)
vistrail = Vistrail()
id_remap = {}
action = vistrails.core.db.action.create_paste_action(pip,
vistrail.idScope,
id_remap)
vistrail.add_action(action, 0L, 0)
tag = vtcontroller.vistrail.get_tag(version)
if tag is None:
tag = "Imported workflow"
vistrail.addTag(tag, action.id)
node.set('version', str(action.id))
id_scope = IdScope(1L)
newmashuptrail = Mashuptrail(
MashupsManager.getNewMashuptrailId(),
action.id,
id_scope)
maction = mashuptrail.actionMap[mashupversion]
mtag = mashuptrail.getTagForActionId(mashupversion)
newmashup = copy.copy(maction.mashup)
newmashup.remapPipelineObjects(id_remap)
currVersion = newmashuptrail.addVersion(
newmashuptrail.getLatestVersion(),
newmashup, maction.user,
maction.date)
newmashuptrail.currentVersion = currVersion
newmashuptrail.changeTag(currVersion, mtag, maction.user,
maction.date)
newvtcontroller = BaseVistrailController(vistrail, None)
MashupsManager.addMashuptrailtoVistrailController(newvtcontroller,
newmashuptrail)
node.set('mashuptrail', str(newmashuptrail.id))
node.set('mashupVersion', str(newmashuptrail.currentVersion))
else:
vistrail = vtcontroller.vistrail
newvtcontroller = MashupsManager.copyBaseVistrailController(vtcontroller)
#create temporary file
(fd, name) = tempfile.mkstemp(prefix='vt_tmp',
suffix='.vt')
os.close(fd)
try:
fileLocator = FileLocator(name)
newvtcontroller.write_vistrail(fileLocator)
contents = open(name).read()
vtcontent = base64.b64encode(contents)
finally:
os.unlink(name)
#if not vistrail.db_version:
# vistrail.db_version = currentVersion
node.set('vtcontent',vtcontent)
xmlstring = ElementTree.tostring(node)
file_ = open(filename,'w')
file_.write(xmlstring)
file_.close()
result = True
return result
def run_and_get_results(w_list,
parameters='',
update_vistrail=True,
extra_info=None,
reason='Console Mode Execution'):
"""run_and_get_results(w_list: list of (locator, version), parameters: str,
output_dir:str, update_vistrail: boolean,
extra_info:dict)
Run all workflows in w_list, and returns an interpreter result object.
version can be a tag name or a version id.
"""
elements = parameters.split("$&$")
aliases = {}
params = []
result = []
for locator, workflow in w_list:
(v, abstractions, thumbnails, mashups) = load_vistrail(locator)
controller = VistrailController(v,
locator,
abstractions,
thumbnails,
mashups,
auto_save=update_vistrail)
if isinstance(workflow, basestring):
version = v.get_version_number(workflow)
elif isinstance(workflow, (int, long)):
version = workflow
elif workflow is None:
version = controller.get_latest_version_in_graph()
else:
msg = "Invalid version tag or number: %s" % workflow
raise VistrailsInternalError(msg)
controller.change_selected_version(version)
for e in elements:
pos = e.find("=")
if pos != -1:
key = e[:pos].strip()
value = e[pos + 1:].strip()
if controller.current_pipeline.has_alias(key):
aliases[key] = value
elif 'mashup_id' in extra_info:
# new-style mashups can have aliases not existing in pipeline
for mashuptrail in mashups:
if mashuptrail.vtVersion == version:
mashup = mashuptrail.getMashup(
extra_info['mashup_id'])
c = mashup.getAliasByName(key).component
params.append((c.vttype, c.vtid, value))
if not update_vistrail:
conf = get_vistrails_configuration()
if conf.has('thumbs'):
conf.thumbs.autoSave = False
jobMonitor = controller.jobMonitor
current_workflow = jobMonitor.currentWorkflow()
if not current_workflow:
for job in jobMonitor.workflows.itervalues():
try:
job_version = int(job.version)
except ValueError:
try:
job_version = v.get_version_number(job.version)
except KeyError:
# this is a PE or mashup
continue
if version == job_version:
current_workflow = job
jobMonitor.startWorkflow(job)
if not current_workflow:
current_workflow = JobWorkflow(version)
jobMonitor.startWorkflow(current_workflow)
try:
(results, _) = \
controller.execute_current_workflow(custom_aliases=aliases,
custom_params=params,
extra_info=extra_info,
reason=reason)
finally:
jobMonitor.finishWorkflow()
new_version = controller.current_version
if new_version != version:
debug.log("Version '%s' (%s) was upgraded. The actual "
"version executed was %s" %
(workflow, version, new_version))
run = results[0]
run.workflow_info = (locator.name, new_version)
run.pipeline = controller.current_pipeline
if update_vistrail:
controller.write_vistrail(locator)
result.append(run)
if current_workflow.jobs:
if current_workflow.completed():
run.job = "COMPLETED"
else:
run.job = "RUNNING: %s" % current_workflow.id
for job in current_workflow.jobs.itervalues():
if not job.finished:
run.job += "\n %s %s %s" % (job.start, job.name,
job.description())
print run.job
return result
def exportMashup(filename, vtcontroller, mashuptrail, mashupversion,
etype):
"""exportMashup(filename: str, vtcontroller: VistrailController,
mashuptrail: Mashuptrail, type: int) -> bool
where etype is
0: include full tree
1: include only workflow and mashup identified by version
2: as a link, it will point to a local file.
"""
result = False
if vtcontroller is not None and mashuptrail is not None:
locator = vtcontroller.locator
version = mashuptrail.vtVersion
node = ElementTree.Element('vtlink')
if isinstance(locator, DBLocator):
node.set('host', str(locator.host))
node.set('port', str(locator.port))
node.set('database', str(locator.db))
node.set('vtid', str(locator.obj_id))
else:
node.set('filename', str(locator.name))
node.set('version', str(version))
node.set('execute', "True")
node.set('forceDB', "False")
node.set('showSpreadsheetOnly', "True")
node.set('mashuptrail', str(mashuptrail.id))
node.set('mashupVersion', str(mashupversion))
if etype in [0, 1]:
if etype == 1: #minimal
pip = vtcontroller.vistrail.getPipeline(version)
vistrail = Vistrail()
id_remap = {}
action = vistrails.core.db.action.create_paste_action(
pip, vistrail.idScope, id_remap)
vistrail.add_action(action, 0L, 0)
tag = vtcontroller.vistrail.get_tag(version)
if tag is None:
tag = "Imported workflow"
vistrail.addTag(tag, action.id)
node.set('version', str(action.id))
id_scope = IdScope(1L)
newmashuptrail = Mashuptrail(
MashupsManager.getNewMashuptrailId(), action.id,
id_scope)
maction = mashuptrail.actionMap[mashupversion]
mtag = mashuptrail.getTagForActionId(mashupversion)
newmashup = copy.copy(maction.mashup)
newmashup.remapPipelineObjects(id_remap)
currVersion = newmashuptrail.addVersion(
newmashuptrail.getLatestVersion(), newmashup,
maction.user, maction.date)
newmashuptrail.currentVersion = currVersion
newmashuptrail.changeTag(currVersion, mtag, maction.user,
maction.date)
newvtcontroller = BaseVistrailController(vistrail, None)
MashupsManager.addMashuptrailtoVistrailController(
newvtcontroller, newmashuptrail)
node.set('mashuptrail', str(newmashuptrail.id))
node.set('mashupVersion',
str(newmashuptrail.currentVersion))
else:
vistrail = vtcontroller.vistrail
newvtcontroller = MashupsManager.copyBaseVistrailController(
vtcontroller)
#create temporary file
(fd, name) = tempfile.mkstemp(prefix='vt_tmp', suffix='.vt')
os.close(fd)
try:
fileLocator = FileLocator(name)
newvtcontroller.write_vistrail(fileLocator)
contents = open(name).read()
vtcontent = base64.b64encode(contents)
finally:
os.unlink(name)
#if not vistrail.db_version:
# vistrail.db_version = currentVersion
node.set('vtcontent', vtcontent)
xmlstring = ElementTree.tostring(node)
file_ = open(filename, 'w')
file_.write(xmlstring)
file_.close()
result = True
return result
def run_and_get_results(w_list, parameters='', workflow_info=None,
update_vistrail=True, extra_info=None,
reason='Console Mode Execution'):
"""run_and_get_results(w_list: list of (locator, version), parameters: str,
workflow_info:str, update_vistrail: boolean,
extra_info:dict)
Run all workflows in w_list, and returns an interpreter result object.
version can be a tag name or a version id.
"""
elements = parameters.split("$&$")
aliases = {}
result = []
for locator, workflow in w_list:
(v, abstractions , thumbnails, mashups) = load_vistrail(locator)
controller = VistrailController(v, locator, abstractions, thumbnails,
mashups, auto_save=update_vistrail)
if isinstance(workflow, basestring):
version = v.get_version_number(workflow)
elif isinstance(workflow, (int, long)):
version = workflow
elif workflow is None:
version = controller.get_latest_version_in_graph()
else:
msg = "Invalid version tag or number: %s" % workflow
raise VistrailsInternalError(msg)
controller.change_selected_version(version)
for e in elements:
pos = e.find("=")
if pos != -1:
key = e[:pos].strip()
value = e[pos+1:].strip()
if controller.current_pipeline.has_alias(key):
aliases[key] = value
if workflow_info is not None and controller.current_pipeline is not None:
# FIXME DAK: why is this always done?!? there is a flag for it...
if is_running_gui():
controller.updatePipelineScene()
base_fname = "%s_%s_pipeline.pdf" % (locator.short_filename, version)
filename = os.path.join(workflow_info, base_fname)
controller.current_pipeline_scene.saveToPDF(filename)
else:
debug.critical("Cannot save pipeline figure when not "
"running in gui mode")
base_fname = "%s_%s_pipeline.xml" % (locator.short_filename, version)
filename = os.path.join(workflow_info, base_fname)
vistrails.core.db.io.save_workflow(controller.current_pipeline, filename)
if not update_vistrail:
conf = get_vistrails_configuration()
if conf.has('thumbs'):
conf.thumbs.autoSave = False
(results, _) = \
controller.execute_current_workflow(custom_aliases=aliases,
extra_info=extra_info,
reason=reason)
new_version = controller.current_version
if new_version != version:
debug.warning("Version '%s' (%s) was upgraded. The actual "
"version executed was %s" % \
(workflow, version, new_version))
run = results[0]
run.workflow_info = (locator.name, new_version)
run.pipeline = controller.current_pipeline
if update_vistrail:
controller.write_vistrail(locator)
result.append(run)
return result
def push_vistrail_to_repository(self, branching=False):
""" uploads current VisTrail to web repository """
self._repository_status['details'] = "Pushing to repository..."
self._push_button.setEnabled(False)
self._branch_button.setEnabled(False)
self.update_push_information()
try:
# create temp file
(fd, filename) = tempfile.mkstemp(suffix='.vt', prefix='vt_tmp')
os.close(fd)
# writing tmp vt and switching back to orginal vt
locator = ZIPFileLocator(filename)
controller = vistrails.api.get_current_controller()
tmp_controller = VistrailController(controller.vistrail.do_copy(),
locator)
tmp_controller.changed = True
tmp_controller.write_vistrail(locator)
# check if this vt is from the repository
if controller.vistrail.get_annotation('repository_vt_id'):
repository_vt_id = controller.vistrail.get_annotation('repository_vt_id').value
else:
repository_vt_id = -1
# upload vistrail temp file to repository
register_openers(cookiejar=self.dialog.cookiejar)
project = self.serverCombo.itemData(self.serverCombo.currentIndex())[0]
if project == "Default": project = ""
params = {'vistrail_file': open(filename, 'rb'),
'action': 'upload',
'name': controller.locator.short_name,
'repository_vt_id': repository_vt_id if not branching else -1,
'is_runnable': not bool(len(self.unsupported_packages)+ \
len(self.local_data_modules)),
'vt_id': 0,
'branched_from': "" if not branching else repository_vt_id,
'project': project,
'everyone_can_view': self.perm_view.checkState(),
'everyone_can_edit': self.perm_edit.checkState(),
'everyone_can_download': self.perm_download.checkState()
}
upload_url = "%s/vistrails/remote_upload/" % \
self.config.webRepositoryURL
datagen, headers = multipart_encode(params)
request = urllib2.Request(upload_url, datagen, headers)
result = urllib2.urlopen(request)
updated_response = result.read()
os.unlink(filename)
if updated_response[:6] == "upload":
# No update, just upload
if result.code != 200:
self._repository_status['details'] = \
"Push to repository failed"
debug.critical("Push to repository failed (Please contact an administrator)")
else:
repository_vt_id = int(updated_response[8:])
controller.vistrail.set_annotation('repository_vt_id',
repository_vt_id)
controller.vistrail.set_annotation('repository_creator',
self.dialog.loginUser)
# ensure that the annotations get saved
controller.set_changed(True)
self._repository_status['details'] = \
"Push to repository was successful"
else:
# update, load updated vistrail
if result.code != 200:
self._repository_status['details'] = "Update Failed"
debug.critical("Update vistrail in web repository failed (Please contact an administrator)")
else:
debug.log("getting version from web")
# request file to download
download_url = "%s/vistrails/download/%s/" % \
(self.config.webRepositoryURL, updated_response)
request = urllib2.Request(download_url)
result = urllib2.urlopen(request)
updated_file = result.read()
# create temp file of updated vistrail
(fd, updated_filename) = tempfile.mkstemp(suffix='.vtl',
prefix='vtl_tmp')
os.close(fd)
updated_vt = open(updated_filename, 'w')
updated_vt.write(updated_file)
updated_vt.close()
# switch vistrails to updated one
controller = vistrails.api.get_current_controller()
updated_locator = FileLocator(updated_filename)
(up_vistrail, abstractions, thumbnails, mashups) = \
load_vistrail(updated_locator)
# FIXME need to figure out what to do with this !!!
controller.set_vistrail(up_vistrail,
controller.vistrail.locator,
abstractions, thumbnails, mashups)
# update version tree drawing
controller.recompute_terse_graph()
controller.invalidate_version_tree()
os.remove(updated_filename)
os.remove(updated_filename[:-1])
self._repository_status['details'] = \
"Update to repository was successful"
except Exception, e:
debug.critical("An error occurred", str(e))
self._repository_status['details'] = "An error occurred"
