def get_vt_graph(vt_list, tree_info, pdf=False):
"""get_vt_graph(vt_list: list of locator, tree_info:str)
Load all vistrails in vt_list and dump their tree to tree_info.
"""
result = []
if is_running_gui():
from vistrails.gui.vistrail_controller import VistrailController as \
GUIVistrailController
for locator in vt_list:
try:
(v, abstractions , thumbnails, mashups) = load_vistrail(locator)
controller = GUIVistrailController(v, locator, abstractions,
thumbnails, mashups)
if tree_info is not None:
from vistrails.gui.version_view import QVersionTreeView
version_view = QVersionTreeView()
version_view.scene().setupScene(controller)
if pdf:
base_fname = "graph_%s.pdf" % locator.short_filename
filename = os.path.join(tree_info, base_fname)
version_view.scene().saveToPDF(filename)
else:
base_fname = "graph_%s.png" % locator.short_filename
filename = os.path.join(tree_info, base_fname)
version_view.scene().saveToPNG(filename)
del version_view
result.append((True, ""))
except Exception, e:
result.append((False, debug.format_exception(e)))
def run_parameter_exploration(locator, pe_id, extra_info = {},
reason="Console Mode Parameter Exploration Execution"):
"""run_parameter_exploration(w_list: (locator, version),
pe_id: str/int,
reason: str) -> (pe_id, [error msg])
Run parameter exploration in w, and returns an interpreter result object.
version can be a tag name or a version id.
"""
if is_running_gui():
from vistrails.gui.vistrail_controller import VistrailController as \
GUIVistrailController
try:
(v, abstractions , thumbnails, mashups) = load_vistrail(locator)
controller = GUIVistrailController(v, locator, abstractions,
thumbnails, mashups)
try:
pe_id = int(pe_id)
pe = controller.vistrail.get_paramexp(pe_id)
except ValueError:
pe = controller.vistrail.get_named_paramexp(pe_id)
controller.change_selected_version(pe.action_id)
controller.executeParameterExploration(pe, extra_info=extra_info,
showProgress=False)
except Exception, e:
return (locator, pe_id,
debug.format_exception(e), debug.format_exc())
def run_parameter_exploration(
locator,
pe_id,
extra_info={},
reason="Console Mode Parameter Exploration Execution"):
"""run_parameter_exploration(w_list: (locator, version),
pe_id: str/int,
reason: str) -> (pe_id, [error msg])
Run parameter exploration in w, and returns an interpreter result object.
version can be a tag name or a version id.
"""
if is_running_gui():
from vistrails.gui.vistrail_controller import VistrailController as \
GUIVistrailController
try:
(v, abstractions, thumbnails, mashups) = load_vistrail(locator)
controller = GUIVistrailController(v, locator, abstractions,
thumbnails, mashups)
try:
pe_id = int(pe_id)
pe = controller.vistrail.get_paramexp(pe_id)
except ValueError:
pe = controller.vistrail.get_named_paramexp(pe_id)
controller.change_selected_version(pe.action_id)
controller.executeParameterExploration(pe,
extra_info=extra_info,
showProgress=False)
except Exception, e:
return (locator, pe_id, debug.format_exception(e),
debug.format_exc())
def get_vt_graph(vt_list, tree_info, pdf=False):
"""get_vt_graph(vt_list: list of locator, tree_info:str)
Load all vistrails in vt_list and dump their tree to tree_info.
"""
result = []
if is_running_gui():
from vistrails.gui.vistrail_controller import VistrailController as \
GUIVistrailController
for locator in vt_list:
try:
(v, abstractions, thumbnails, mashups) = load_vistrail(locator)
controller = GUIVistrailController(v, locator, abstractions,
thumbnails, mashups)
if tree_info is not None:
from vistrails.gui.version_view import QVersionTreeView
version_view = QVersionTreeView()
version_view.scene().setupScene(controller)
if pdf:
base_fname = "graph_%s.pdf" % locator.short_filename
filename = os.path.join(tree_info, base_fname)
version_view.scene().saveToPDF(filename)
else:
base_fname = "graph_%s.png" % locator.short_filename
filename = os.path.join(tree_info, base_fname)
version_view.scene().saveToPNG(filename)
del version_view
result.append((True, ""))
except Exception, e:
result.append((False, debug.format_exception(e)))
def get_wf_graph(w_list, output_dir, pdf=False):
"""run_and_get_results(w_list: list of (locator, version),
output_dir:str, pdf:bool)
Load all workflows in wf_list and dump their graph to output_dir.
"""
result = []
if is_running_gui():
from vistrails.gui.vistrail_controller import VistrailController as \
GUIVistrailController
for locator, workflow in w_list:
try:
(v, abstractions, thumbnails, mashups) = load_vistrail(locator)
controller = GUIVistrailController(v,
locator,
abstractions,
thumbnails,
mashups,
auto_save=False)
# FIXME TE: why is this needed
controller.current_pipeline_view.set_controller(controller)
version = None
if isinstance(workflow, basestring):
version = v.get_version_number(workflow)
elif isinstance(workflow, (int, long)):
version = workflow
elif workflow is None:
version = controller.get_latest_version_in_graph()
else:
msg = "Invalid version tag or number: %s" % workflow
raise VistrailsInternalError(msg)
controller.change_selected_version(version)
if controller.current_pipeline is not None:
controller.updatePipelineScene()
if pdf:
base_fname = "%s_%s_pipeline.pdf" % \
(locator.short_filename, version)
filename = os.path.join(output_dir, base_fname)
controller.current_pipeline_scene.saveToPDF(filename)
else:
base_fname = "%s_%s_pipeline.png" % \
(locator.short_filename, version)
filename = os.path.join(output_dir, base_fname)
controller.current_pipeline_scene.saveToPNG(filename)
result.append((True, ""))
except Exception, e:
result.append((False, debug.format_exception(e)))
def get_wf_graph(w_list, output_dir, pdf=False):
"""run_and_get_results(w_list: list of (locator, version),
output_dir:str, pdf:bool)
Load all workflows in wf_list and dump their graph to output_dir.
"""
result = []
if is_running_gui():
from vistrails.gui.vistrail_controller import VistrailController as \
GUIVistrailController
for locator, workflow in w_list:
try:
(v, abstractions , thumbnails, mashups) = load_vistrail(locator)
controller = GUIVistrailController(v, locator, abstractions,
thumbnails, mashups,
auto_save=False)
# FIXME TE: why is this needed
controller.current_pipeline_view.set_controller(controller)
version = None
if isinstance(workflow, basestring):
version = v.get_version_number(workflow)
elif isinstance(workflow, (int, long)):
version = workflow
elif workflow is None:
version = controller.get_latest_version_in_graph()
else:
msg = "Invalid version tag or number: %s" % workflow
raise VistrailsInternalError(msg)
controller.change_selected_version(version)
if controller.current_pipeline is not None:
controller.updatePipelineScene()
if pdf:
base_fname = "%s_%s_pipeline.pdf" % \
(locator.short_filename, version)
filename = os.path.join(output_dir, base_fname)
controller.current_pipeline_scene.saveToPDF(filename)
else:
base_fname = "%s_%s_pipeline.png" % \
(locator.short_filename, version)
filename = os.path.join(output_dir, base_fname)
controller.current_pipeline_scene.saveToPNG(filename)
result.append((True, ""))
except Exception, e:
result.append((False, debug.format_exception(e)))
def data_sync(self):
""" downloads/uploads/uses the local file depending on availability """
self.checksum_lookup()
# local file not on repository, so upload
if not self.on_server and os.path.isfile(self.in_file.name):
cookiejar = None
if is_running_gui():
import vistrails.gui.repository
cookiejar = vistrails.gui.repository.QRepositoryDialog.cookiejar
if cookiejar:
register_openers(cookiejar=cookiejar)
params = {'dataset_file': open(self.in_file.name, 'rb'),
'name': self.in_file.name.split('/')[-1],
'origin': 'vistrails',
'checksum': self.checksum}
upload_url = "%s/datasets/upload/" % self.base_url
datagen, headers = multipart_encode(params)
request = urllib2.Request(upload_url, datagen, headers)
try:
result = urllib2.urlopen(request)
if result.code != 200:
debug.warning("Failed to upload data to the "
"repository")
# make temporarily uncachable
self.is_cacheable = self.invalidate_cache
else:
debug.log("Push to repository was successful")
# make sure module caches
self.is_cacheable = self.validate_cache
except Exception, e:
debug.warning("Failed to upload data to the repository")
# make temporarily uncachable
self.is_cacheable = self.invalidate_cache
debug.warning('RepoSync uploaded %s to the repository' % \
self.in_file.name)
else:
debug.warning("You must be logged into the web repository in "
"order to upload data. No data was synced")
# make temporarily uncachable
self.is_cacheable = self.invalidate_cache
# use local data
self.set_output("file", self.in_file)
def run_and_get_results(w_list, parameters='', output_dir=None,
update_vistrail=True, extra_info=None,
reason='Console Mode Execution'):
"""run_and_get_results(w_list: list of (locator, version), parameters: str,
output_dir:str, update_vistrail: boolean,
extra_info:dict)
Run all workflows in w_list, and returns an interpreter result object.
version can be a tag name or a version id.
"""
elements = parameters.split("$&$")
aliases = {}
params = []
result = []
for locator, workflow in w_list:
(v, abstractions , thumbnails, mashups) = load_vistrail(locator)
controller = VistrailController(v, locator, abstractions, thumbnails,
mashups, auto_save=update_vistrail)
if isinstance(workflow, basestring):
version = v.get_version_number(workflow)
elif isinstance(workflow, (int, long)):
version = workflow
elif workflow is None:
version = controller.get_latest_version_in_graph()
else:
msg = "Invalid version tag or number: %s" % workflow
raise VistrailsInternalError(msg)
controller.change_selected_version(version)
for e in elements:
pos = e.find("=")
if pos != -1:
key = e[:pos].strip()
value = e[pos+1:].strip()
if controller.current_pipeline.has_alias(key):
aliases[key] = value
elif 'mashup_id' in extra_info:
# new-style mashups can have aliases not existing in pipeline
for mashuptrail in mashups:
if mashuptrail.vtVersion == version:
mashup = mashuptrail.getMashup(extra_info['mashup_id'])
c = mashup.getAliasByName(key).component
params.append((c.vttype, c.vtid, value))
if output_dir is not None and controller.current_pipeline is not None:
# FIXME DAK: why is this always done?!? there is a flag for it...
if is_running_gui():
controller.updatePipelineScene()
base_fname = "%s_%s_pipeline.pdf" % (locator.short_filename, version)
filename = os.path.join(output_dir, base_fname)
controller.current_pipeline_scene.saveToPDF(filename)
else:
debug.critical("Cannot save pipeline figure when not "
"running in gui mode")
base_fname = "%s_%s_pipeline.xml" % (locator.short_filename, version)
filename = os.path.join(output_dir, base_fname)
vistrails.core.db.io.save_workflow(controller.current_pipeline, filename)
if not update_vistrail:
conf = get_vistrails_configuration()
if conf.has('thumbs'):
conf.thumbs.autoSave = False
jobMonitor = controller.jobMonitor
current_workflow = jobMonitor.currentWorkflow()
if not current_workflow:
for job in jobMonitor.workflows.itervalues():
try:
job_version = int(job.version)
except ValueError:
job_version = v.get_version_number(job.version)
if version == job_version:
current_workflow = job
jobMonitor.startWorkflow(job)
if not current_workflow:
current_workflow = JobWorkflow(version)
jobMonitor.startWorkflow(current_workflow)
try:
(results, _) = \
controller.execute_current_workflow(custom_aliases=aliases,
custom_params=params,
extra_info=extra_info,
reason=reason)
finally:
jobMonitor.finishWorkflow()
new_version = controller.current_version
if new_version != version:
debug.log("Version '%s' (%s) was upgraded. The actual "
"version executed was %s" % (
workflow, version, new_version))
run = results[0]
run.workflow_info = (locator.name, new_version)
run.pipeline = controller.current_pipeline
if update_vistrail:
controller.write_vistrail(locator)
result.append(run)
if current_workflow.jobs:
if current_workflow.completed():
run.job = "COMPLETED"
else:
run.job = "RUNNING: %s" % current_workflow.id
for job in current_workflow.jobs.itervalues():
if not job.finished:
run.job += "\n %s %s %s" % (job.start, job.name, job.description())
print run.job
return result
def run_and_get_results(w_list, parameters='', workflow_info=None,
update_vistrail=True, extra_info=None,
reason='Console Mode Execution'):
"""run_and_get_results(w_list: list of (locator, version), parameters: str,
workflow_info:str, update_vistrail: boolean,
extra_info:dict)
Run all workflows in w_list, and returns an interpreter result object.
version can be a tag name or a version id.
"""
elements = parameters.split("$&$")
aliases = {}
result = []
for locator, workflow in w_list:
(v, abstractions , thumbnails, mashups) = load_vistrail(locator)
controller = VistrailController(v, locator, abstractions, thumbnails,
mashups, auto_save=update_vistrail)
if isinstance(workflow, basestring):
version = v.get_version_number(workflow)
elif isinstance(workflow, (int, long)):
version = workflow
elif workflow is None:
version = controller.get_latest_version_in_graph()
else:
msg = "Invalid version tag or number: %s" % workflow
raise VistrailsInternalError(msg)
controller.change_selected_version(version)
for e in elements:
pos = e.find("=")
if pos != -1:
key = e[:pos].strip()
value = e[pos+1:].strip()
if controller.current_pipeline.has_alias(key):
aliases[key] = value
if workflow_info is not None and controller.current_pipeline is not None:
# FIXME DAK: why is this always done?!? there is a flag for it...
if is_running_gui():
controller.updatePipelineScene()
base_fname = "%s_%s_pipeline.pdf" % (locator.short_filename, version)
filename = os.path.join(workflow_info, base_fname)
controller.current_pipeline_scene.saveToPDF(filename)
else:
debug.critical("Cannot save pipeline figure when not "
"running in gui mode")
base_fname = "%s_%s_pipeline.xml" % (locator.short_filename, version)
filename = os.path.join(workflow_info, base_fname)
vistrails.core.db.io.save_workflow(controller.current_pipeline, filename)
if not update_vistrail:
conf = get_vistrails_configuration()
if conf.has('thumbs'):
conf.thumbs.autoSave = False
(results, _) = \
controller.execute_current_workflow(custom_aliases=aliases,
extra_info=extra_info,
reason=reason)
new_version = controller.current_version
if new_version != version:
debug.warning("Version '%s' (%s) was upgraded. The actual "
"version executed was %s" % \
(workflow, version, new_version))
run = results[0]
run.workflow_info = (locator.name, new_version)
run.pipeline = controller.current_pipeline
if update_vistrail:
controller.write_vistrail(locator)
result.append(run)
return result
def run_and_get_results(w_list,
parameters='',
output_dir=None,
update_vistrail=True,
extra_info=None,
reason='Console Mode Execution'):
"""run_and_get_results(w_list: list of (locator, version), parameters: str,
output_dir:str, update_vistrail: boolean,
extra_info:dict)
Run all workflows in w_list, and returns an interpreter result object.
version can be a tag name or a version id.
"""
elements = parameters.split("$&$")
aliases = {}
params = []
result = []
for locator, workflow in w_list:
(v, abstractions, thumbnails, mashups) = load_vistrail(locator)
controller = VistrailController(v,
locator,
abstractions,
thumbnails,
mashups,
auto_save=update_vistrail)
if isinstance(workflow, basestring):
version = v.get_version_number(workflow)
elif isinstance(workflow, (int, long)):
version = workflow
elif workflow is None:
version = controller.get_latest_version_in_graph()
else:
msg = "Invalid version tag or number: %s" % workflow
raise VistrailsInternalError(msg)
controller.change_selected_version(version)
for e in elements:
pos = e.find("=")
if pos != -1:
key = e[:pos].strip()
value = e[pos + 1:].strip()
if controller.current_pipeline.has_alias(key):
aliases[key] = value
elif 'mashup_id' in extra_info:
# new-style mashups can have aliases not existing in pipeline
for mashuptrail in mashups:
if mashuptrail.vtVersion == version:
mashup = mashuptrail.getMashup(
extra_info['mashup_id'])
c = mashup.getAliasByName(key).component
params.append((c.vttype, c.vtid, value))
if output_dir is not None and controller.current_pipeline is not None:
# FIXME DAK: why is this always done?!? there is a flag for it...
if is_running_gui():
controller.updatePipelineScene()
base_fname = "%s_%s_pipeline.pdf" % (locator.short_filename,
version)
filename = os.path.join(output_dir, base_fname)
controller.current_pipeline_scene.saveToPDF(filename)
else:
debug.critical("Cannot save pipeline figure when not "
"running in gui mode")
base_fname = "%s_%s_pipeline.xml" % (locator.short_filename,
version)
filename = os.path.join(output_dir, base_fname)
vistrails.core.db.io.save_workflow(controller.current_pipeline,
filename)
if not update_vistrail:
conf = get_vistrails_configuration()
if conf.has('thumbs'):
conf.thumbs.autoSave = False
jobMonitor = controller.jobMonitor
current_workflow = jobMonitor.currentWorkflow()
if not current_workflow:
for job in jobMonitor.workflows.itervalues():
try:
job_version = int(job.version)
except ValueError:
job_version = v.get_version_number(job.version)
if version == job_version:
current_workflow = job
jobMonitor.startWorkflow(job)
if not current_workflow:
current_workflow = JobWorkflow(version)
jobMonitor.startWorkflow(current_workflow)
try:
(results, _) = \
controller.execute_current_workflow(custom_aliases=aliases,
custom_params=params,
extra_info=extra_info,
reason=reason)
finally:
jobMonitor.finishWorkflow()
new_version = controller.current_version
if new_version != version:
debug.log("Version '%s' (%s) was upgraded. The actual "
"version executed was %s" %
(workflow, version, new_version))
run = results[0]
run.workflow_info = (locator.name, new_version)
run.pipeline = controller.current_pipeline
if update_vistrail:
controller.write_vistrail(locator)
result.append(run)
if current_workflow.jobs:
if current_workflow.completed():
run.job = "COMPLETED"
else:
run.job = "RUNNING: %s" % current_workflow.id
for job in current_workflow.jobs.itervalues():
if not job.finished:
run.job += "\n %s %s %s" % (job.start, job.name,
job.description())
print run.job
return result
