def run_parameter_exploration(
locator,
pe_id,
extra_info={},
reason="Console Mode Parameter Exploration Execution"):
"""run_parameter_exploration(w_list: (locator, version),
pe_id: str/int,
reason: str) -> (pe_id, [error msg])
Run parameter exploration in w, and returns an interpreter result object.
version can be a tag name or a version id.
"""
if is_running_gui():
from vistrails.gui.vistrail_controller import VistrailController as \
GUIVistrailController
try:
(v, abstractions, thumbnails, mashups) = load_vistrail(locator)
controller = GUIVistrailController(v, locator, abstractions,
thumbnails, mashups)
try:
pe_id = int(pe_id)
pe = controller.vistrail.get_paramexp(pe_id)
except ValueError:
pe = controller.vistrail.get_named_paramexp(pe_id)
controller.change_selected_version(pe.action_id)
controller.executeParameterExploration(pe,
extra_info=extra_info,
showProgress=False)
except Exception, e:
return (locator, pe_id, debug.format_exception(e),
debug.format_exc())
def updateVistrail(self, url, vistrail=None):
""" updateVistrail(self, string:url, Vistrail:vistrail)
Update the specified entity url. Delete or reload as necessary.
Need to make sure workspaces are updated if the entity is changed.
"""
entities = [e for e in self.entities.itervalues() if e.url == url]
entity = entities[0] if len(entities) else None
while entity and entity.parent:
entity = entity.parent
url = entity.url
workspaces = [
p for p in self.workspaces if entity in self.workspaces[p]
]
if entity:
for p in workspaces:
self.del_from_workspace(entity, p)
self.delete_entity(entity)
locator = BaseLocator.from_url(url)
if locator.is_valid():
if not vistrail:
(vistrail, abstractions, thumbnails,
mashups) = load_vistrail(locator)
vistrail.abstractions = abstractions
vistrail.thumbnails = thumbnails
vistrail.mashups = mashups
entity = self.create_vistrail_entity(vistrail)
for p in workspaces:
self.add_to_workspace(entity, p)
return entity
else:
# probably an unsaved vistrail
pass
def updateVistrail(self, url, vistrail=None):
""" updateVistrail(self, string:url, Vistrail:vistrail)
Update the specified entity url. Delete or reload as necessary.
Need to make sure workspaces are updated if the entity is changed.
"""
entities = [e for e in self.entities.itervalues() if e.url == url]
entity = entities[0] if len(entities) else None
while entity and entity.parent:
entity = entity.parent
url = entity.url
workspaces = [p for p in self.workspaces if entity in self.workspaces[p]]
if entity:
for p in workspaces:
self.del_from_workspace(entity, p)
self.delete_entity(entity)
locator = BaseLocator.from_url(url)
if locator.is_valid():
if not vistrail:
(vistrail, abstractions, thumbnails, mashups) = load_vistrail(locator)
vistrail.abstractions = abstractions
vistrail.thumbnails = thumbnails
vistrail.mashups = mashups
entity = self.create_vistrail_entity(vistrail)
for p in workspaces:
self.add_to_workspace(entity, p)
return entity
else:
# probably an unsaved vistrail
pass
def get_vt_graph(vt_list, tree_info, pdf=False):
"""get_vt_graph(vt_list: list of locator, tree_info:str)
Load all vistrails in vt_list and dump their tree to tree_info.
"""
result = []
if is_running_gui():
from vistrails.gui.vistrail_controller import VistrailController as \
GUIVistrailController
for locator in vt_list:
try:
(v, abstractions , thumbnails, mashups) = load_vistrail(locator)
controller = GUIVistrailController(v, locator, abstractions,
thumbnails, mashups)
if tree_info is not None:
from vistrails.gui.version_view import QVersionTreeView
version_view = QVersionTreeView()
version_view.scene().setupScene(controller)
if pdf:
base_fname = "graph_%s.pdf" % locator.short_filename
filename = os.path.join(tree_info, base_fname)
version_view.scene().saveToPDF(filename)
else:
base_fname = "graph_%s.png" % locator.short_filename
filename = os.path.join(tree_info, base_fname)
version_view.scene().saveToPNG(filename)
del version_view
result.append((True, ""))
except Exception, e:
result.append((False, debug.format_exception(e)))
def run_parameter_exploration(locator, pe_id, extra_info = {},
reason="Console Mode Parameter Exploration Execution"):
"""run_parameter_exploration(w_list: (locator, version),
pe_id: str/int,
reason: str) -> (pe_id, [error msg])
Run parameter exploration in w, and returns an interpreter result object.
version can be a tag name or a version id.
"""
if is_running_gui():
from vistrails.gui.vistrail_controller import VistrailController as \
GUIVistrailController
try:
(v, abstractions , thumbnails, mashups) = load_vistrail(locator)
controller = GUIVistrailController(v, locator, abstractions,
thumbnails, mashups)
try:
pe_id = int(pe_id)
pe = controller.vistrail.get_paramexp(pe_id)
except ValueError:
pe = controller.vistrail.get_named_paramexp(pe_id)
controller.change_selected_version(pe.action_id)
controller.executeParameterExploration(pe, extra_info=extra_info,
showProgress=False)
except Exception, e:
return (locator, pe_id,
debug.format_exception(e), debug.format_exc())
def get_vt_graph(vt_list, tree_info, pdf=False):
"""get_vt_graph(vt_list: list of locator, tree_info:str)
Load all vistrails in vt_list and dump their tree to tree_info.
"""
result = []
if is_running_gui():
from vistrails.gui.vistrail_controller import VistrailController as \
GUIVistrailController
for locator in vt_list:
try:
(v, abstractions, thumbnails, mashups) = load_vistrail(locator)
controller = GUIVistrailController(v, locator, abstractions,
thumbnails, mashups)
if tree_info is not None:
from vistrails.gui.version_view import QVersionTreeView
version_view = QVersionTreeView()
version_view.scene().setupScene(controller)
if pdf:
base_fname = "graph_%s.pdf" % locator.short_filename
filename = os.path.join(tree_info, base_fname)
version_view.scene().saveToPDF(filename)
else:
base_fname = "graph_%s.png" % locator.short_filename
filename = os.path.join(tree_info, base_fname)
version_view.scene().saveToPNG(filename)
del version_view
result.append((True, ""))
except Exception, e:
result.append((False, debug.format_exception(e)))
def get_wf_graph(w_list, output_dir, pdf=False):
"""run_and_get_results(w_list: list of (locator, version),
output_dir:str, pdf:bool)
Load all workflows in wf_list and dump their graph to output_dir.
"""
result = []
if is_running_gui():
from vistrails.gui.vistrail_controller import VistrailController as \
GUIVistrailController
for locator, workflow in w_list:
try:
(v, abstractions, thumbnails, mashups) = load_vistrail(locator)
controller = GUIVistrailController(v,
locator,
abstractions,
thumbnails,
mashups,
auto_save=False)
# FIXME TE: why is this needed
controller.current_pipeline_view.set_controller(controller)
version = None
if isinstance(workflow, basestring):
version = v.get_version_number(workflow)
elif isinstance(workflow, (int, long)):
version = workflow
elif workflow is None:
version = controller.get_latest_version_in_graph()
else:
msg = "Invalid version tag or number: %s" % workflow
raise VistrailsInternalError(msg)
controller.change_selected_version(version)
if controller.current_pipeline is not None:
controller.updatePipelineScene()
if pdf:
base_fname = "%s_%s_pipeline.pdf" % \
(locator.short_filename, version)
filename = os.path.join(output_dir, base_fname)
controller.current_pipeline_scene.saveToPDF(filename)
else:
base_fname = "%s_%s_pipeline.png" % \
(locator.short_filename, version)
filename = os.path.join(output_dir, base_fname)
controller.current_pipeline_scene.saveToPNG(filename)
result.append((True, ""))
except Exception, e:
result.append((False, debug.format_exception(e)))
def open_vistrail(self, locator, version=None, is_abstraction=False):
"""open_vistrail(locator: Locator, version = None: int or str,
is_abstraction: bool)
opens a new vistrail from the given locator, selecting the
given version.
"""
self.close_first_vistrail_if_necessary()
if self.single_document_mode and self.currentView():
self.closeVistrail()
view = self.ensureVistrail(locator)
if view:
if version is not None:
if isinstance(version, basestring):
try:
version = view.vistrail.get_version_number(version)
except:
version = None
if version is not None:
view.setup_view(version)
return view
try:
(vistrail, abstraction_files, thumbnail_files, _) = \
load_vistrail(locator, is_abstraction)
result = self.set_vistrail_view(vistrail, locator,
abstraction_files, thumbnail_files,
version)
# update collection
try:
vistrail.thumbnails = thumbnail_files
vistrail.abstractions = abstraction_files
collection = Collection.getInstance()
url = locator.to_url()
# create index if not exist
entity = collection.fromUrl(url)
if entity:
# find parent vistrail
while entity.parent:
entity = entity.parent
else:
entity = collection.updateVistrail(url, vistrail)
# add to relevant workspace categories
collection.add_to_workspace(entity)
collection.commit()
except Exception, e:
import traceback
debug.critical('Failed to index vistrail', str(e) + traceback.format_exc())
return result
def test(self):
try:
errs = []
filename = os.path.join(EXAMPLES_PATH, vtfile)
locator = FileLocator(os.path.abspath(filename))
(v, abstractions, thumbnails, mashups) = load_vistrail(locator)
errs = vistrails.core.console_mode.run(
[(locator, version)],
update_vistrail=False,
extra_info={'compare_thumbnails': compare_thumbnails})
if len(errs) > 0:
for err in errs:
print(" *** Error in %s:%s:%s -- %s" % err)
self.fail(str(err))
except Exception, e:
self.fail(debug.format_exception(e))
def test(self):
try:
errs = []
filename = os.path.join(EXAMPLES_PATH, vtfile)
locator = FileLocator(os.path.abspath(filename))
(v, abstractions, thumbnails, mashups) = load_vistrail(locator)
errs = vistrails.core.console_mode.run(
[(locator, version)],
update_vistrail=False,
extra_info={'compare_thumbnails': compare_thumbnails})
if len(errs) > 0:
for err in errs:
print(" *** Error in %s:%s:%s -- %s" % err)
self.fail(str(err))
except Exception, e:
self.fail(debug.format_exception(e))
def get_wf_graph(w_list, output_dir, pdf=False):
"""run_and_get_results(w_list: list of (locator, version),
output_dir:str, pdf:bool)
Load all workflows in wf_list and dump their graph to output_dir.
"""
result = []
if is_running_gui():
from vistrails.gui.vistrail_controller import VistrailController as \
GUIVistrailController
for locator, workflow in w_list:
try:
(v, abstractions , thumbnails, mashups) = load_vistrail(locator)
controller = GUIVistrailController(v, locator, abstractions,
thumbnails, mashups,
auto_save=False)
# FIXME TE: why is this needed
controller.current_pipeline_view.set_controller(controller)
version = None
if isinstance(workflow, basestring):
version = v.get_version_number(workflow)
elif isinstance(workflow, (int, long)):
version = workflow
elif workflow is None:
version = controller.get_latest_version_in_graph()
else:
msg = "Invalid version tag or number: %s" % workflow
raise VistrailsInternalError(msg)
controller.change_selected_version(version)
if controller.current_pipeline is not None:
controller.updatePipelineScene()
if pdf:
base_fname = "%s_%s_pipeline.pdf" % \
(locator.short_filename, version)
filename = os.path.join(output_dir, base_fname)
controller.current_pipeline_scene.saveToPDF(filename)
else:
base_fname = "%s_%s_pipeline.png" % \
(locator.short_filename, version)
filename = os.path.join(output_dir, base_fname)
controller.current_pipeline_scene.saveToPNG(filename)
result.append((True, ""))
except Exception, e:
result.append((False, debug.format_exception(e)))
def test_cache(self):
from vistrails.core.modules.basic_modules import StandardOutput
old_compute = StandardOutput.compute
StandardOutput.compute = lambda s: None
try:
from vistrails.core.db.locator import XMLFileLocator
from vistrails.core.vistrail.controller import VistrailController
from vistrails.core.db.io import load_vistrail
"""Test if basic caching is working."""
locator = XMLFileLocator(
vistrails.core.system.vistrails_root_directory() +
'/tests/resources/dummy.xml')
(v, abstractions, thumbnails, mashups) = load_vistrail(locator)
# the controller will take care of upgrades
controller = VistrailController(v, locator, abstractions,
thumbnails, mashups)
p1 = v.getPipeline('int chain')
n = v.get_version_number('int chain')
controller.change_selected_version(n)
controller.flush_delayed_actions()
p1 = controller.current_pipeline
view = DummyView()
interpreter = CachedInterpreter.get()
result = interpreter.execute(
p1,
locator=v,
current_version=n,
view=view,
)
# to force fresh params
p2 = v.getPipeline('int chain')
controller.change_selected_version(n)
controller.flush_delayed_actions()
p2 = controller.current_pipeline
result = interpreter.execute(
p2,
locator=v,
current_version=n,
view=view,
)
self.assertEqual(len(result.modules_added), 1)
finally:
StandardOutput.compute = old_compute
def update_from_database(self, db_locator):
# db_conn = db_locator.get_connection()
config = (('host', db_locator._host),
('port', int(db_locator._port)),
('db', db_locator._db),
('user', db_locator._user),
('passwd', db_locator._passwd))
rows = vistrails.db.services.io.get_db_object_list(dict(config), 'vistrail')
for row in rows:
if row[0] in [1,]:
continue
kwargs = {'obj_type': 'vistrail', 'obj_id': row[0]}
locator = DBLocator(*[x[1] for x in config], **kwargs)
(vistrail, abstractions, thumbnails, mashups) = load_vistrail(locator)
vistrail.abstractions = abstractions
vistrail.thumbnails = thumbnails
vistrail.mashups = mashups
self.create_vistrail_entity(vistrail)
def test_cache(self):
from vistrails.core.modules.basic_modules import StandardOutput
old_compute = StandardOutput.compute
StandardOutput.compute = lambda s: None
try:
from vistrails.core.db.locator import XMLFileLocator
from vistrails.core.vistrail.controller import VistrailController
from vistrails.core.db.io import load_vistrail
"""Test if basic caching is working."""
locator = XMLFileLocator(vistrails.core.system.vistrails_root_directory() +
'/tests/resources/dummy.xml')
(v, abstractions, thumbnails, mashups) = load_vistrail(locator)
# the controller will take care of upgrades
controller = VistrailController(v, locator, abstractions,
thumbnails, mashups)
p1 = v.getPipeline('int chain')
n = v.get_version_number('int chain')
controller.change_selected_version(n)
controller.flush_delayed_actions()
p1 = controller.current_pipeline
view = DummyView()
interpreter = CachedInterpreter.get()
result = interpreter.execute(p1,
locator=v,
current_version=n,
view=view,
)
# to force fresh params
p2 = v.getPipeline('int chain')
controller.change_selected_version(n)
controller.flush_delayed_actions()
p2 = controller.current_pipeline
result = interpreter.execute(p2,
locator=v,
current_version=n,
view=view,
)
self.assertEqual(len(result.modules_added), 1)
finally:
StandardOutput.compute = old_compute
def update_from_database(self, db_locator):
# db_conn = db_locator.get_connection()
config = {'host': db_locator._host,
'port': int(db_locator._port),
'db': db_locator._db,
'user': db_locator._user,
'passwd': db_locator._passwd}
rows = vistrails.db.services.io.get_db_object_list(config, 'vistrail')
for row in rows:
if row[0] in [1,]:
continue
locator = DBLocator(config['host'], config['port'], config['db'],
config['user'], config['passwd'],
obj_type='vistrail', obj_id=row[0])
(vistrail, abstractions, thumbnails, mashups) = load_vistrail(locator)
vistrail.abstractions = abstractions
vistrail.thumbnails = thumbnails
vistrail.mashups = mashups
self.create_vistrail_entity(vistrail)
def newVistrail(self, recover_files=True):
""" newVistrail() -> (None or QVistrailView)
Create a new vistrail with no name. If user cancels process,
returns None.
FIXME: We should do the interactive parts separately.
"""
if self.single_document_mode and self.currentView():
if not self.closeVistrail():
return None
if recover_files and untitled_locator().has_temporaries():
locator = copy.copy(untitled_locator())
else:
locator = None
try:
(vistrail, abstraction_files, thumbnail_files, _) = load_vistrail(locator)
except ModuleRegistryException, e:
debug.critical("Module registry error for %s" %
str(e.__class__.__name__), str(e))
def load(self, loadworkflow=True):
config = ConfigParser.ConfigParser()
if config.read(self.config_file):
if config.has_section('global'):
if config.has_option('global', 'cellnum'):
self.cellnum = config.getint('global', 'cellnum')
if config.has_option('global', 'filenum'):
self.filenum = config.getint('global', 'filenum')
if config.has_option('global', 'varnum'):
self.varnum = config.getint('global', 'varnum')
print " ------ Loaded plot %s, varnum = %d ------ " % ( self.name, self.varnum )
if config.has_option('global', 'workflow_tag'):
self.workflow_tag = config.get('global', 'workflow_tag')
# else:
# debug.warning("CDAT Package: file %s does not contain a required option 'workflow_tag'. Widget will not be loaded."%self.config_file)
# self.loaded = False
# return
if config.has_option('global', 'filetypes'):
types = config.get('global', 'filetypes')
tlist = [t.strip() for t in types.split(";")]
for t in tlist:
kv = t.split(":")
self.filetypes[kv[0].strip()] = [v.strip()
for v in kv[1].split(",")]
if config.has_option('global', 'qt_filter'):
self.qt_filter = config.get('global', 'qt_filter')
if config.has_option('global', 'dependencies'):
deps = config.get('global', 'dependencies')
self.dependencies = [d.strip() for d in deps.split(",")]
if config.has_option('global', 'serialized_config_alias'):
self.serializedConfigAlias = config.get('global', 'serialized_config_alias')
for y in range(self.filenum):
self.files.append( 'Filename' + str(y+1) )
for v in range(self.varnum):
self.vars.append( 'VariableName' + str(v+1) )
self.axes.append( 'Axes' + str(v+1) )
for x in range(self.cellnum):
section_name = 'cell' + str(x+1)
if config.has_section(section_name):
cellType = config.get(section_name, 'celltype')
if config.has_option(section_name, 'address_alias'):
self.cells.append( Cell( cellType, None, None,
config.get(section_name, 'address_alias') ) )
else:
self.cells.append(Cell( cellType,"Row"+str(x+1), "Column"+str(x+1) ) )
else:
for y in range(self.filenum):
option_name = 'filename_alias' + str(y+1)
if config.has_option('global', option_name):
self.files.append(config.get('global', option_name))
for v in range(self.varnum):
option_name = 'varname_alias' + str(v+1)
if config.has_option('global', option_name):
self.vars.append(config.get('global', option_name))
axes_name = 'axes_alias' + str(v+1)
if config.has_option('global', axes_name):
self.axes.append(config.get('global', axes_name))
for x in range(self.cellnum):
section_name = 'cell' + str(x+1)
if (config.has_section(section_name) and
config.has_option(section_name, 'celltype') and
config.has_option(section_name, 'row_alias') and
config.has_option(section_name, 'col_alias')):
self.cells.append(Cell(config.get(section_name, 'celltype'),
config.get(section_name, 'row_alias'),
config.get(section_name, 'col_alias')))
if loadworkflow:
#load workflow in vistrail
#only if dependencies are enabled
manager = get_package_manager()
self.unsatisfied_deps = []
for dep in self.dependencies:
if not manager.has_package(dep):
self.unsatisfied_deps.append(dep)
if len(self.unsatisfied_deps) == 0:
try:
(self.plot_vistrail, abstractions , thumbnails, mashups) = load_vistrail(self.locator)
controller = VistrailController()
controller.set_vistrail(self.plot_vistrail, self.locator,
abstractions, thumbnails,
mashups)
self.workflow_version = self.plot_vistrail.get_version_number(self.workflow_tag) if self.workflow_tag else controller.get_latest_version_in_graph()
print " Loaded %s version: %s" % ( self.name, str( self.workflow_version ) )
controller.change_selected_version(self.workflow_version)
self.workflow = controller.current_pipeline
self.loaded = True
except Exception, err:
debug.warning( "Error loading workflow %s: %s" % ( self.name, err ) )
self.loaded = False
else:
debug.warning("UV-CDAT: %s widget could not be loaded \
because it depends on packages that are not loaded:"%self.name)
debug.warning(" %s"%", ".join(self.unsatisfied_deps))
self.loaded = False
else:
debug.warning("UV-CDAT: file %s does not contain a 'global'\
section. Widget will not be loaded."%self.config_file)
self.loaded = False
def run_and_get_results(w_list, parameters='', output_dir=None,
update_vistrail=True, extra_info=None,
reason='Console Mode Execution'):
"""run_and_get_results(w_list: list of (locator, version), parameters: str,
output_dir:str, update_vistrail: boolean,
extra_info:dict)
Run all workflows in w_list, and returns an interpreter result object.
version can be a tag name or a version id.
"""
elements = parameters.split("$&$")
aliases = {}
params = []
result = []
for locator, workflow in w_list:
(v, abstractions , thumbnails, mashups) = load_vistrail(locator)
controller = VistrailController(v, locator, abstractions, thumbnails,
mashups, auto_save=update_vistrail)
if isinstance(workflow, basestring):
version = v.get_version_number(workflow)
elif isinstance(workflow, (int, long)):
version = workflow
elif workflow is None:
version = controller.get_latest_version_in_graph()
else:
msg = "Invalid version tag or number: %s" % workflow
raise VistrailsInternalError(msg)
controller.change_selected_version(version)
for e in elements:
pos = e.find("=")
if pos != -1:
key = e[:pos].strip()
value = e[pos+1:].strip()
if controller.current_pipeline.has_alias(key):
aliases[key] = value
elif 'mashup_id' in extra_info:
# new-style mashups can have aliases not existing in pipeline
for mashuptrail in mashups:
if mashuptrail.vtVersion == version:
mashup = mashuptrail.getMashup(extra_info['mashup_id'])
c = mashup.getAliasByName(key).component
params.append((c.vttype, c.vtid, value))
if output_dir is not None and controller.current_pipeline is not None:
# FIXME DAK: why is this always done?!? there is a flag for it...
if is_running_gui():
controller.updatePipelineScene()
base_fname = "%s_%s_pipeline.pdf" % (locator.short_filename, version)
filename = os.path.join(output_dir, base_fname)
controller.current_pipeline_scene.saveToPDF(filename)
else:
debug.critical("Cannot save pipeline figure when not "
"running in gui mode")
base_fname = "%s_%s_pipeline.xml" % (locator.short_filename, version)
filename = os.path.join(output_dir, base_fname)
vistrails.core.db.io.save_workflow(controller.current_pipeline, filename)
if not update_vistrail:
conf = get_vistrails_configuration()
if conf.has('thumbs'):
conf.thumbs.autoSave = False
jobMonitor = controller.jobMonitor
current_workflow = jobMonitor.currentWorkflow()
if not current_workflow:
for job in jobMonitor.workflows.itervalues():
try:
job_version = int(job.version)
except ValueError:
job_version = v.get_version_number(job.version)
if version == job_version:
current_workflow = job
jobMonitor.startWorkflow(job)
if not current_workflow:
current_workflow = JobWorkflow(version)
jobMonitor.startWorkflow(current_workflow)
try:
(results, _) = \
controller.execute_current_workflow(custom_aliases=aliases,
custom_params=params,
extra_info=extra_info,
reason=reason)
finally:
jobMonitor.finishWorkflow()
new_version = controller.current_version
if new_version != version:
debug.log("Version '%s' (%s) was upgraded. The actual "
"version executed was %s" % (
workflow, version, new_version))
run = results[0]
run.workflow_info = (locator.name, new_version)
run.pipeline = controller.current_pipeline
if update_vistrail:
controller.write_vistrail(locator)
result.append(run)
if current_workflow.jobs:
if current_workflow.completed():
run.job = "COMPLETED"
else:
run.job = "RUNNING: %s" % current_workflow.id
for job in current_workflow.jobs.itervalues():
if not job.finished:
run.job += "\n %s %s %s" % (job.start, job.name, job.description())
print run.job
return result
def push_vistrail_to_repository(self, branching=False):
""" uploads current VisTrail to web repository """
self._repository_status['details'] = "Pushing to repository..."
self._push_button.setEnabled(False)
self._branch_button.setEnabled(False)
self.update_push_information()
try:
# create temp file
(fd, filename) = tempfile.mkstemp(suffix='.vt', prefix='vt_tmp')
os.close(fd)
# writing tmp vt and switching back to orginal vt
locator = ZIPFileLocator(filename)
controller = vistrails.api.get_current_controller()
tmp_controller = VistrailController(controller.vistrail.do_copy(),
locator)
tmp_controller.changed = True
tmp_controller.write_vistrail(locator)
# check if this vt is from the repository
if controller.vistrail.get_annotation('repository_vt_id'):
repository_vt_id = controller.vistrail.get_annotation('repository_vt_id').value
else:
repository_vt_id = -1
# upload vistrail temp file to repository
register_openers(cookiejar=self.dialog.cookiejar)
project = self.serverCombo.itemData(self.serverCombo.currentIndex())[0]
if project == "Default": project = ""
params = {'vistrail_file': open(filename, 'rb'),
'action': 'upload',
'name': controller.locator.short_name,
'repository_vt_id': repository_vt_id if not branching else -1,
'is_runnable': not bool(len(self.unsupported_packages)+ \
len(self.local_data_modules)),
'vt_id': 0,
'branched_from': "" if not branching else repository_vt_id,
'project': project,
'everyone_can_view': self.perm_view.checkState(),
'everyone_can_edit': self.perm_edit.checkState(),
'everyone_can_download': self.perm_download.checkState()
}
upload_url = "%s/vistrails/remote_upload/" % \
self.config.webRepositoryURL
datagen, headers = multipart_encode(params)
request = urllib2.Request(upload_url, datagen, headers)
result = urllib2.urlopen(request)
updated_response = result.read()
os.unlink(filename)
if updated_response[:6] == "upload":
# No update, just upload
if result.code != 200:
self._repository_status['details'] = \
"Push to repository failed"
debug.critical("Push to repository failed (Please contact an administrator)")
else:
repository_vt_id = int(updated_response[8:])
controller.vistrail.set_annotation('repository_vt_id',
repository_vt_id)
controller.vistrail.set_annotation('repository_creator',
self.dialog.loginUser)
# ensure that the annotations get saved
controller.set_changed(True)
self._repository_status['details'] = \
"Push to repository was successful"
else:
# update, load updated vistrail
if result.code != 200:
self._repository_status['details'] = "Update Failed"
debug.critical("Update vistrail in web repository failed (Please contact an administrator)")
else:
debug.log("getting version from web")
# request file to download
download_url = "%s/vistrails/download/%s/" % \
(self.config.webRepositoryURL, updated_response)
request = urllib2.Request(download_url)
result = urllib2.urlopen(request)
updated_file = result.read()
# create temp file of updated vistrail
(fd, updated_filename) = tempfile.mkstemp(suffix='.vtl',
prefix='vtl_tmp')
os.close(fd)
updated_vt = open(updated_filename, 'w')
updated_vt.write(updated_file)
updated_vt.close()
# switch vistrails to updated one
controller = vistrails.api.get_current_controller()
updated_locator = FileLocator(updated_filename)
(up_vistrail, abstractions, thumbnails, mashups) = \
load_vistrail(updated_locator)
# FIXME need to figure out what to do with this !!!
controller.set_vistrail(up_vistrail,
controller.vistrail.locator,
abstractions, thumbnails, mashups)
# update version tree drawing
controller.recompute_terse_graph()
controller.invalidate_version_tree()
os.remove(updated_filename)
os.remove(updated_filename[:-1])
self._repository_status['details'] = \
"Update to repository was successful"
except Exception, e:
debug.critical("An error occurred", str(e))
self._repository_status['details'] = "An error occurred"
def run_and_get_results(w_list, parameters='', workflow_info=None,
update_vistrail=True, extra_info=None,
reason='Console Mode Execution'):
"""run_and_get_results(w_list: list of (locator, version), parameters: str,
workflow_info:str, update_vistrail: boolean,
extra_info:dict)
Run all workflows in w_list, and returns an interpreter result object.
version can be a tag name or a version id.
"""
elements = parameters.split("$&$")
aliases = {}
result = []
for locator, workflow in w_list:
(v, abstractions , thumbnails, mashups) = load_vistrail(locator)
controller = VistrailController(v, locator, abstractions, thumbnails,
mashups, auto_save=update_vistrail)
if isinstance(workflow, basestring):
version = v.get_version_number(workflow)
elif isinstance(workflow, (int, long)):
version = workflow
elif workflow is None:
version = controller.get_latest_version_in_graph()
else:
msg = "Invalid version tag or number: %s" % workflow
raise VistrailsInternalError(msg)
controller.change_selected_version(version)
for e in elements:
pos = e.find("=")
if pos != -1:
key = e[:pos].strip()
value = e[pos+1:].strip()
if controller.current_pipeline.has_alias(key):
aliases[key] = value
if workflow_info is not None and controller.current_pipeline is not None:
# FIXME DAK: why is this always done?!? there is a flag for it...
if is_running_gui():
controller.updatePipelineScene()
base_fname = "%s_%s_pipeline.pdf" % (locator.short_filename, version)
filename = os.path.join(workflow_info, base_fname)
controller.current_pipeline_scene.saveToPDF(filename)
else:
debug.critical("Cannot save pipeline figure when not "
"running in gui mode")
base_fname = "%s_%s_pipeline.xml" % (locator.short_filename, version)
filename = os.path.join(workflow_info, base_fname)
vistrails.core.db.io.save_workflow(controller.current_pipeline, filename)
if not update_vistrail:
conf = get_vistrails_configuration()
if conf.has('thumbs'):
conf.thumbs.autoSave = False
(results, _) = \
controller.execute_current_workflow(custom_aliases=aliases,
extra_info=extra_info,
reason=reason)
new_version = controller.current_version
if new_version != version:
debug.warning("Version '%s' (%s) was upgraded. The actual "
"version executed was %s" % \
(workflow, version, new_version))
run = results[0]
run.workflow_info = (locator.name, new_version)
run.pipeline = controller.current_pipeline
if update_vistrail:
controller.write_vistrail(locator)
result.append(run)
return result
def test_looping_pipeline_fix(self):
"""Chains upgrades and automatic package initialization."""
# Expected actions are as follow:
# - loads workflow.xml
# * pipeline is missing looping_fix.a version 0.1
# - enables looping_fix.a (version 0.2)
# * pipeline is still missing looping_fix.a version 0.1
# - runs upgrade for looping_fix.a, 0.1 -> 0.2
# - upgrade changes modules to package looping_fix.b version 0.1
# * pipeline is missing looping_fix.b version 0.1
# - enables looping_fix.b (version 0.2)
# * pipeline is still missing looping_fix.b version 0.1
# - runs upgrade for looping_fix.b, 0.1 -> 0.2
# - upgrade changes modules to package looping_fix.c version 1.0
# * pipeline is missing looping_fix.c version 1.0
# - enables looping_fix.c (version 1.0)
# * pipeline is valid
# 5 calls to handle_invalid_pipeline()
# Pre-adds packages so that the package manager can find them
packages = ["pkg_a", "pkg_b", "pkg_c"]
prefix = "vistrails.tests.resources.looping_upgrades."
pm = get_package_manager()
for pkg in packages:
pm.get_available_package(pkg, prefix=prefix)
self.assertFalse(set(pkg.codepath for pkg in pm.enabled_package_list()).intersection(packages))
# Hooks handle_invalid_pipeline()
from vistrails.core.vistrail.controller import VistrailController
orig_hip = VistrailController.handle_invalid_pipeline
count = [0]
def new_hip(*args, **kwargs):
count[0] += 1
return orig_hip(*args, **kwargs)
VistrailController.handle_invalid_pipeline = new_hip
try:
# Loads workflow.xml
from vistrails.core.db.io import load_vistrail
from vistrails.core.db.locator import FileLocator
from vistrails.core.system import vistrails_root_directory
locator = FileLocator(
os.path.join(vistrails_root_directory(), "tests", "resources", "looping_upgrades", "workflow.xml")
)
loaded_objs = load_vistrail(locator)
controller = VistrailController(loaded_objs[0], locator, *loaded_objs[1:])
# Select version (triggers all the validation/upgrade/loading)
self.assertEqual(controller.get_latest_version_in_graph(), 1)
controller.do_version_switch(1)
self.assertEqual(count[0], 5)
# Restores handle_invalid_pipeline()
finally:
VistrailController.handle_invalid_pipeline = orig_hip
# disable packages
for pkg in reversed(packages):
try:
pm.late_disable_package(pkg)
except MissingPackage:
pass
def push_vistrail_to_repository(self, branching=False):
""" uploads current VisTrail to web repository """
self._repository_status['details'] = "Pushing to repository..."
self._push_button.setEnabled(False)
self._branch_button.setEnabled(False)
self.update_push_information()
try:
# create temp file
(fd, filename) = tempfile.mkstemp(suffix='.vt', prefix='vt_tmp')
os.close(fd)
# writing tmp vt and switching back to original vt
locator = ZIPFileLocator(filename)
controller = vistrails.api.get_current_controller()
controller.write_vistrail(locator, export=True)
# check if this vt is from the repository
if controller.vistrail.get_annotation('repository_vt_id'):
repository_vt_id = controller.vistrail.get_annotation(
'repository_vt_id').value
else:
repository_vt_id = -1
# upload vistrail temp file to repository
register_openers(cookiejar=self.dialog.cookiejar)
project = self.serverCombo.itemData(
self.serverCombo.currentIndex())[0]
if project == "Default": project = ""
params = {'vistrail_file': open(filename, 'rb'),
'action': 'upload',
'name': controller.locator.short_name,
'repository_vt_id': repository_vt_id if not branching else -1,
'is_runnable': not bool(len(self.unsupported_packages)+ \
len(self.local_data_modules)),
'vt_id': 0,
'branched_from': "" if not branching else repository_vt_id,
'project': project,
'everyone_can_view': self.perm_view.checkState(),
'everyone_can_edit': self.perm_edit.checkState(),
'everyone_can_download': self.perm_download.checkState()
}
upload_url = "%s/vistrails/remote_upload/" % \
self.config.webRepositoryURL
datagen, headers = multipart_encode(params)
request = urllib2.Request(upload_url, datagen, headers)
result = urllib2.urlopen(request)
updated_response = result.read()
os.unlink(filename)
if updated_response[:6] == "upload":
# No update, just upload
if result.code != 200:
self._repository_status['details'] = \
"Push to repository failed"
debug.critical(
"Push to repository failed (Please contact an administrator)"
)
else:
repository_vt_id = int(updated_response[8:])
controller.vistrail.set_annotation('repository_vt_id',
repository_vt_id)
controller.vistrail.set_annotation('repository_creator',
self.dialog.loginUser)
# ensure that the annotations get saved
controller.set_changed(True)
self._repository_status['details'] = \
"Push to repository was successful"
else:
# update, load updated vistrail
if result.code != 200:
self._repository_status['details'] = "Update Failed"
debug.critical(
"Update vistrail in web repository failed (Please contact an administrator)"
)
else:
debug.log("getting version from web")
# request file to download
download_url = "%s/vistrails/download/%s/" % \
(self.config.webRepositoryURL, updated_response)
request = urllib2.Request(download_url)
result = urllib2.urlopen(request)
updated_file = result.read()
# create temp file of updated vistrail
(fd, updated_filename) = tempfile.mkstemp(suffix='.vtl',
prefix='vtl_tmp')
os.close(fd)
updated_vt = open(updated_filename, 'w')
updated_vt.write(updated_file)
updated_vt.close()
# switch vistrails to updated one
controller = vistrails.api.get_current_controller()
updated_locator = FileLocator(updated_filename)
(up_vistrail, abstractions, thumbnails, mashups) = \
load_vistrail(updated_locator)
# FIXME need to figure out what to do with this !!!
current_version = controller.current_version
controller.set_vistrail(up_vistrail,
controller.vistrail.locator,
abstractions, thumbnails, mashups)
controller.change_selected_version(current_version)
# update version tree drawing
controller.recompute_terse_graph()
controller.invalidate_version_tree()
os.remove(updated_filename)
os.remove(updated_filename[:-1])
self._repository_status['details'] = \
"Update to repository was successful"
except Exception, e:
debug.critical("An error occurred", e)
self._repository_status['details'] = "An error occurred"
def run_and_get_results(w_list,
parameters='',
update_vistrail=True,
extra_info=None,
reason='Console Mode Execution'):
"""run_and_get_results(w_list: list of (locator, version), parameters: str,
output_dir:str, update_vistrail: boolean,
extra_info:dict)
Run all workflows in w_list, and returns an interpreter result object.
version can be a tag name or a version id.
"""
elements = parameters.split("$&$")
aliases = {}
params = []
result = []
for locator, workflow in w_list:
(v, abstractions, thumbnails, mashups) = load_vistrail(locator)
controller = VistrailController(v,
locator,
abstractions,
thumbnails,
mashups,
auto_save=update_vistrail)
if isinstance(workflow, basestring):
version = v.get_version_number(workflow)
elif isinstance(workflow, (int, long)):
version = workflow
elif workflow is None:
version = controller.get_latest_version_in_graph()
else:
msg = "Invalid version tag or number: %s" % workflow
raise VistrailsInternalError(msg)
controller.change_selected_version(version)
for e in elements:
pos = e.find("=")
if pos != -1:
key = e[:pos].strip()
value = e[pos + 1:].strip()
if controller.current_pipeline.has_alias(key):
aliases[key] = value
elif 'mashup_id' in extra_info:
# new-style mashups can have aliases not existing in pipeline
for mashuptrail in mashups:
if mashuptrail.vtVersion == version:
mashup = mashuptrail.getMashup(
extra_info['mashup_id'])
c = mashup.getAliasByName(key).component
params.append((c.vttype, c.vtid, value))
if not update_vistrail:
conf = get_vistrails_configuration()
if conf.has('thumbs'):
conf.thumbs.autoSave = False
jobMonitor = controller.jobMonitor
current_workflow = jobMonitor.currentWorkflow()
if not current_workflow:
for job in jobMonitor.workflows.itervalues():
try:
job_version = int(job.version)
except ValueError:
try:
job_version = v.get_version_number(job.version)
except KeyError:
# this is a PE or mashup
continue
if version == job_version:
current_workflow = job
jobMonitor.startWorkflow(job)
if not current_workflow:
current_workflow = JobWorkflow(version)
jobMonitor.startWorkflow(current_workflow)
try:
(results, _) = \
controller.execute_current_workflow(custom_aliases=aliases,
custom_params=params,
extra_info=extra_info,
reason=reason)
finally:
jobMonitor.finishWorkflow()
new_version = controller.current_version
if new_version != version:
debug.log("Version '%s' (%s) was upgraded. The actual "
"version executed was %s" %
(workflow, version, new_version))
run = results[0]
run.workflow_info = (locator.name, new_version)
run.pipeline = controller.current_pipeline
if update_vistrail:
controller.write_vistrail(locator)
result.append(run)
if current_workflow.jobs:
if current_workflow.completed():
run.job = "COMPLETED"
else:
run.job = "RUNNING: %s" % current_workflow.id
for job in current_workflow.jobs.itervalues():
if not job.finished:
run.job += "\n %s %s %s" % (job.start, job.name,
job.description())
print run.job
return result
def test_infinite_looping_upgrade(self):
"""Test that circular upgrades fail gracefully"""
# Expected actions are as follow:
# - loads workflow2.xml
# * pipeline is missing looping_fix.x version 0.1
# - enables looping_fix.x (version 0.2)
# * pipeline is still missing looping_fix.x version 0.1
# - runs upgrade for looping_fix.x, 0.1 -> 0.2
# - upgrade changes modules to package looping_fix.y version 0.1
# * pipeline is missing looping_fix.y version 0.1
# - enables looping_fix.y (version 0.2)
# * pipeline is still missing looping_fix.y version 0.1
# Loop 50 times:
# - runs upgrade for looping_fix.y, 0.1 -> 0.2
# - upgrade changes modules to package looping_fix.x version 0.1
# * pipeline is missing looping_fix.x version 0.1
# - runs upgrade for looping_fix.x, 0.1 -> 0.2
# - upgrade changes modules to package looping_fix.y version 0.1
# * pipeline is missing looping_fix.y version 0.1
# 50 calls to handle_invalid_pipeline()
# Pre-adds packages so that the package manager can find them
packages = ["pkg_x", "pkg_y"]
prefix = "vistrails.tests.resources.looping_upgrades."
pm = get_package_manager()
for pkg in packages:
pm.get_available_package(pkg, prefix=prefix)
# Hooks handle_invalid_pipeline()
from vistrails.core.vistrail.controller import VistrailController
orig_hip = VistrailController.handle_invalid_pipeline
count = [0]
def new_hip(*args, **kwargs):
count[0] += 1
return orig_hip(*args, **kwargs)
VistrailController.handle_invalid_pipeline = new_hip
try:
# Loads workflow.xml
from vistrails.core.db.io import load_vistrail
from vistrails.core.db.locator import FileLocator
from vistrails.core.system import vistrails_root_directory
locator = FileLocator(
os.path.join(vistrails_root_directory(), "tests", "resources", "looping_upgrades", "workflow2.xml")
)
loaded_objs = load_vistrail(locator)
controller = VistrailController(loaded_objs[0], locator, *loaded_objs[1:])
# Select version (triggers all the validation/upgrade/loading)
self.assertEqual(controller.get_latest_version_in_graph(), 1)
try:
controller.do_version_switch(1)
except InvalidPipeline:
pass
else:
self.fail("No InvalidPipeline exception raised!")
# Restores handle_invalid_pipeline()
finally:
VistrailController.handle_invalid_pipeline = orig_hip
# disable packages
for pkg in reversed(packages):
try:
pm.late_disable_package(pkg)
except MissingPackage:
pass
# make sure it looped 50 times before failing
max_loops = getattr(get_vistrails_configuration(), "maxPipelineFixAttempts", 50)
self.assertEqual(count[0], max_loops)
# Check that original version gets selected
self.assertEqual(1, controller.current_version)
