def load_pipeline(filename):
"""Loads a single pipeline from a filename.
"""
initialize()
if not isinstance(filename, basestring):
raise TypeError("load_vistrails() expects a filename, got %r" %
type(filename).__name__)
# Copied from VistrailsApplicationInterface#open_workflow()
locator = FileLocator(filename)
pipeline = locator.load(_Pipeline)
return Pipeline(pipeline)
def load_pipeline(filename):
"""Loads a single pipeline from a filename.
"""
initialize()
if not isinstance(filename, basestring):
raise TypeError("load_vistrails() expects a filename, got %r" %
type(filename).__name__)
# Copied from VistrailsApplicationInterface#open_workflow()
locator = FileLocator(filename)
pipeline = locator.load(_Pipeline)
return Pipeline(pipeline)
def do_export(self, filename, expected_source):
import os
from vistrails.core.db.locator import FileLocator
from vistrails.core.system import vistrails_root_directory
from vistrails.core.vistrail.pipeline import Pipeline
locator = FileLocator(
os.path.join(vistrails_root_directory(), 'tests', 'resources',
filename))
pipeline = locator.load(Pipeline)
self.assertEqual('\n'.join(write_workflow_to_python(pipeline)) + '\n',
expected_source)
def setFigureInfo(self, opt_dict):
self.doCheckLink(True, opt_dict)
if 'showworkflow' in opt_dict:
self.figure_type.setCurrentIndex(1)
elif 'showtree' in opt_dict:
self.figure_type.setCurrentIndex(2)
else:
self.figure_type.setCurrentIndex(0)
if 'version' in opt_dict:
self.figure_version.setText(str(opt_dict['version']))
else:
self.figure_version.setText("")
if 'tag' in opt_dict:
self.figure_tag.setEditText(str(opt_dict['tag'])[1:-1])
else:
self.figure_tag.setEditText("")
if '_args' in opt_dict:
self.graphicx_edit.setText(opt_dict['_args'])
# build locator
if 'filename' in opt_dict:
# set using basedir of tex file
fname = opt_dict['filename']
if not os.path.isabs(fname):
source_dir = os.path.dirname(str(self.source_edit.text()))
#print 'source_dir:', str(self.source_edit.text()), source_dir
fname = os.path.join(source_dir, fname)
locator = FileLocator(fname)
elif 'host' in opt_dict:
if 'port' not in opt_dict:
opt_dict['port'] = '3306'
port = int(opt_dict['port'])
locator = DBLocator(opt_dict['host'],
port,
opt_dict['db'],
'',
'',
obj_id=opt_dict['vtid'])
else:
locator = None
if locator is not None:
self.figure_ref.setText(locator.to_url())
else:
self.figure_ref.setText("")
self.figure_ref.locator = locator
self.readImage(opt_dict)
def setFigureInfo(self, opt_dict):
self.doCheckLink(True, opt_dict)
if 'showworkflow' in opt_dict:
self.figure_type.setCurrentIndex(1)
elif 'showtree' in opt_dict:
self.figure_type.setCurrentIndex(2)
else:
self.figure_type.setCurrentIndex(0)
if 'version' in opt_dict:
self.figure_version.setText(str(opt_dict['version']))
else:
self.figure_version.setText("")
if 'tag' in opt_dict:
self.figure_tag.setEditText(str(opt_dict['tag'])[1:-1])
else:
self.figure_tag.setEditText("")
if '_args' in opt_dict:
self.graphicx_edit.setText(opt_dict['_args'])
# build locator
if 'filename' in opt_dict:
# set using basedir of tex file
fname = opt_dict['filename']
if not os.path.isabs(fname):
source_dir = os.path.dirname(str(self.source_edit.text()))
#print 'source_dir:', str(self.source_edit.text()), source_dir
fname = os.path.join(source_dir, fname)
locator = FileLocator(fname)
elif 'host' in opt_dict:
if 'port' not in opt_dict:
opt_dict['port'] = '3306'
port = int(opt_dict['port'])
locator = DBLocator(opt_dict['host'], port,
opt_dict['db'], '', '',
obj_id=opt_dict['vtid'])
else:
locator = None
if locator is not None:
self.figure_ref.setText(locator.to_url())
else:
self.figure_ref.setText("")
self.figure_ref.locator = locator
self.readImage(opt_dict)
def _open_vt(self):
items = self._result_tree.selectedItems()
item = items[0]
vt = self._find_vt(item)
if vt is None:
return
workflow, module_id = vt
if ':' in workflow:
filename, version = workflow.rsplit(':', 1)
try:
version = int(version)
except ValueError:
filename, version = workflow, None
else:
filename, version = workflow, None
app = get_vistrails_application()
if (not app.is_running_gui() or not hasattr(app, 'builderWindow')
or app.builderWindow is None):
return
view = app.builderWindow.open_vistrail(FileLocator(filename),
version=version)
if module_id is not None:
from vistrails.gui.pipeline_view import QGraphicsModuleItem
scene = view.controller.current_pipeline_view.scene()
for module_item in (i for i in scene.items()
if isinstance(i, QGraphicsModuleItem)):
if module_item.module.id == module_id:
module_item.setSelected(True)
break
def get_vistrail_from_file(filename):
from vistrails.core.db.locator import FileLocator
from vistrails.core.vistrail.vistrail import Vistrail
v = FileLocator(filename).load()
if not isinstance(v, Vistrail):
v = v.vistrail
return v
def parse_locator(text):
locator = None
wrapper = XMLWrapper()
dom = wrapper.create_document_from_string(text)
root = dom.documentElement
version = None
version = root.getAttribute('version')
if version == '1.0':
for element in named_elements(root, 'locator'):
if str(element.getAttribute('type')) == 'file':
locator = FileLocator.parse(element)
elif str(element.getAttribute('type')) == 'db':
locator = DBLocator.parse(element)
return locator
def parse_locator(text):
locator = None
wrapper = XMLWrapper()
dom = wrapper.create_document_from_string(text)
root = dom.documentElement
version = None
version = root.getAttribute('version')
if version == '1.0':
for element in named_elements(root, 'locator'):
if str(element.getAttribute('type')) == 'file':
locator = FileLocator.parse(element)
elif str(element.getAttribute('type')) == 'db':
locator = DBLocator.parse(element)
return locator
def load_vistrail(filename, version=None):
"""Loads a Vistrail from a filename.
"""
initialize()
if not isinstance(filename, basestring):
raise TypeError("load_vistrails() expects a filename, got %r" %
type(filename).__name__)
locator = FileLocator(filename)
# Copied from VistrailsApplicationInterface#open_vistrail()
loaded_objs = vistrails.core.db.io.load_vistrail(locator)
controller = VistrailController(loaded_objs[0], locator, *loaded_objs[1:])
return Vistrail(controller)
def get_load_file_locator_from_gui(parent, obj_type):
suffixes = "*" + " *".join(suffix_map[obj_type])
fileName = QtGui.QFileDialog.getOpenFileName(
parent, "Open %s..." % obj_type.capitalize(),
vistrails.core.system.vistrails_file_directory(),
"VisTrails files (%s)\nOther files (*)" % suffixes)
if not fileName:
return None
filename = os.path.abspath(str(QtCore.QFile.encodeName(fileName)))
dirName = os.path.dirname(filename)
setattr(get_vistrails_persistent_configuration(), 'fileDir', dirName)
setattr(get_vistrails_configuration(), 'fileDir', dirName)
vistrails.core.system.set_vistrails_file_directory(dirName)
return FileLocator(filename)
def unserialize(text):
""" unserialize(text) -> RecentVistrailList """
root = ElementTree.fromstring(text)
if root.tag != "recentVistrails":
return None
vtlist = RecentVistrailList()
for node in root.getchildren():
loc = FileLocator.from_xml(node)
if loc is None:
loc = DBLocator.from_xml(node, include_name=True)
if loc is not None:
vtlist.locators.append(loc)
vtlist.locators_map[loc.name] = loc
return vtlist
def unserialize(text):
""" unserialize(text) -> RecentVistrailList """
root = ElementTree.fromstring(text)
if root.tag != 'recentVistrails':
return None
vtlist = RecentVistrailList()
for node in root.getchildren():
loc = FileLocator.from_xml(node)
if loc is None:
loc = DBLocator.from_xml(node, include_name=True)
if loc is not None:
vtlist.locators.append(loc)
vtlist.locators_map[loc.name] = loc
return vtlist
def test(self):
try:
errs = []
filename = os.path.join(EXAMPLES_PATH, vtfile)
locator = FileLocator(os.path.abspath(filename))
(v, abstractions, thumbnails, mashups) = load_vistrail(locator)
errs = vistrails.core.console_mode.run(
[(locator, version)],
update_vistrail=False,
extra_info={'compare_thumbnails': compare_thumbnails})
if len(errs) > 0:
for err in errs:
print(" *** Error in %s:%s:%s -- %s" % err)
self.fail(str(err))
except Exception, e:
self.fail(debug.format_exception(e))
def get_save_file_locator_from_gui(parent, obj_type, locator=None):
# Ignore current locator for now
# In the future, use locator to guide GUI for better starting directory
suffixes = "*" + " *".join(suffix_map[obj_type])
fileName = QtGui.QFileDialog.getSaveFileName(
parent,
"Save Vistrail...",
vistrails.core.system.vistrails_file_directory(),
filter="VisTrails files (%s)" % suffixes, # filetypes.strip()
options=QtGui.QFileDialog.DontConfirmOverwrite)
if not fileName:
return None
f = str(QtCore.QFile.encodeName(fileName))
# check for proper suffix
found_suffix = False
for suffix in suffix_map[obj_type]:
if f.endswith(suffix):
found_suffix = True
break
if not found_suffix:
if obj_type == 'vistrail':
f += get_vistrails_configuration().defaultFileType
else:
f += suffix_map[obj_type][0]
if os.path.isfile(f):
msg = QtGui.QMessageBox(QtGui.QMessageBox.Question,
"VisTrails",
"File exists. Overwrite?",
(QtGui.QMessageBox.Yes |
QtGui.QMessageBox.No),
parent)
if msg.exec_() == QtGui.QMessageBox.No:
return None
dirName = os.path.dirname(f)
setattr(get_vistrails_persistent_configuration(), 'fileDir', dirName)
setattr(get_vistrails_configuration(), 'fileDir', dirName)
vistrails.core.system.set_vistrails_file_directory(dirName)
return FileLocator(f)
def run_file(filename, tag_filter=lambda x: True):
"""Loads a .vt file and runs all the tagged versions in it.
"""
import vistrails.core.db.io
from vistrails.core.db.locator import FileLocator
from vistrails.core.system import vistrails_root_directory
from vistrails.core.vistrail.controller import VistrailController
filename = os.path.join(vistrails_root_directory(), '..', filename)
locator = FileLocator(filename)
loaded_objs = vistrails.core.db.io.load_vistrail(locator)
controller = VistrailController(loaded_objs[0], locator, *loaded_objs[1:])
errors = []
for version, name in controller.vistrail.get_tagMap().iteritems():
if tag_filter(name):
controller.change_selected_version(version)
(result,), _ = controller.execute_current_workflow()
if result.errors:
errors.append(("%d: %s" % (version, name), result.errors))
return errors
def open_vistrail_from_file(filename):
from vistrails.core.db.locator import FileLocator
f = FileLocator(filename)
view = get_builder_window().open_vistrail(f)
return view
def process_interactive_input(self):
pe = None
usedb = False
if self.temp_configuration.check('host'):
usedb = True
if self.input:
#check if versions are embedded in the filename
for filename in self.input:
f_name, version = self._parse_vtinfo(filename, not usedb)
locator = None
if f_name is None:
debug.critical("Could not find file %s" % filename)
return False
elif not usedb:
locator = FileLocator(os.path.abspath(f_name))
#_vnode and _vtag will be set when a .vtl file is open and
# it can be either a FileLocator or a DBLocator
elif usedb:
locator = DBLocator(
host=self.temp_configuration.check('host'),
port=self.temp_configuration.check('port') or 3306,
database=self.temp_configuration.check('db'),
user='',
passwd='',
obj_id=f_name,
obj_type=None,
connection_id=None)
if locator:
if self.temp_configuration.check('parameterExploration'):
pe = version
version = None
else:
if hasattr(locator, '_vnode') and \
locator._vnode is not None:
version = locator._vnode
if hasattr(locator, '_vtag'):
# if a tag is set, it should be used instead of the
# version number
if locator._vtag != '':
version = locator._vtag
execute = self.temp_configuration.check('execute')
mashuptrail = None
mashupversion = None
if hasattr(locator, '_mshptrail'):
mashuptrail = locator._mshptrail
if hasattr(locator, '_mshpversion'):
mashupversion = locator._mshpversion
if mashupversion:
execute = True
if self.temp_configuration.showWindow:
self.showBuilderWindow()
self.builderWindow.open_vistrail_without_prompt(
locator,
version,
execute,
mashuptrail=mashuptrail,
mashupVersion=mashupversion)
if self.temp_configuration.check('parameterExploration'):
self.builderWindow.executeParameterExploration(pe)
return True
def push_vistrail_to_repository(self, branching=False):
""" uploads current VisTrail to web repository """
self._repository_status['details'] = "Pushing to repository..."
self._push_button.setEnabled(False)
self._branch_button.setEnabled(False)
self.update_push_information()
try:
# create temp file
(fd, filename) = tempfile.mkstemp(suffix='.vt', prefix='vt_tmp')
os.close(fd)
# writing tmp vt and switching back to original vt
locator = ZIPFileLocator(filename)
controller = vistrails.api.get_current_controller()
controller.write_vistrail(locator, export=True)
# check if this vt is from the repository
if controller.vistrail.get_annotation('repository_vt_id'):
repository_vt_id = controller.vistrail.get_annotation(
'repository_vt_id').value
else:
repository_vt_id = -1
# upload vistrail temp file to repository
register_openers(cookiejar=self.dialog.cookiejar)
project = self.serverCombo.itemData(
self.serverCombo.currentIndex())[0]
if project == "Default": project = ""
params = {'vistrail_file': open(filename, 'rb'),
'action': 'upload',
'name': controller.locator.short_name,
'repository_vt_id': repository_vt_id if not branching else -1,
'is_runnable': not bool(len(self.unsupported_packages)+ \
len(self.local_data_modules)),
'vt_id': 0,
'branched_from': "" if not branching else repository_vt_id,
'project': project,
'everyone_can_view': self.perm_view.checkState(),
'everyone_can_edit': self.perm_edit.checkState(),
'everyone_can_download': self.perm_download.checkState()
}
upload_url = "%s/vistrails/remote_upload/" % \
self.config.webRepositoryURL
datagen, headers = multipart_encode(params)
request = urllib2.Request(upload_url, datagen, headers)
result = urllib2.urlopen(request)
updated_response = result.read()
os.unlink(filename)
if updated_response[:6] == "upload":
# No update, just upload
if result.code != 200:
self._repository_status['details'] = \
"Push to repository failed"
debug.critical(
"Push to repository failed (Please contact an administrator)"
)
else:
repository_vt_id = int(updated_response[8:])
controller.vistrail.set_annotation('repository_vt_id',
repository_vt_id)
controller.vistrail.set_annotation('repository_creator',
self.dialog.loginUser)
# ensure that the annotations get saved
controller.set_changed(True)
self._repository_status['details'] = \
"Push to repository was successful"
else:
# update, load updated vistrail
if result.code != 200:
self._repository_status['details'] = "Update Failed"
debug.critical(
"Update vistrail in web repository failed (Please contact an administrator)"
)
else:
debug.log("getting version from web")
# request file to download
download_url = "%s/vistrails/download/%s/" % \
(self.config.webRepositoryURL, updated_response)
request = urllib2.Request(download_url)
result = urllib2.urlopen(request)
updated_file = result.read()
# create temp file of updated vistrail
(fd, updated_filename) = tempfile.mkstemp(suffix='.vtl',
prefix='vtl_tmp')
os.close(fd)
updated_vt = open(updated_filename, 'w')
updated_vt.write(updated_file)
updated_vt.close()
# switch vistrails to updated one
controller = vistrails.api.get_current_controller()
updated_locator = FileLocator(updated_filename)
(up_vistrail, abstractions, thumbnails, mashups) = \
load_vistrail(updated_locator)
# FIXME need to figure out what to do with this !!!
current_version = controller.current_version
controller.set_vistrail(up_vistrail,
controller.vistrail.locator,
abstractions, thumbnails, mashups)
controller.change_selected_version(current_version)
# update version tree drawing
controller.recompute_terse_graph()
controller.invalidate_version_tree()
os.remove(updated_filename)
os.remove(updated_filename[:-1])
self._repository_status['details'] = \
"Update to repository was successful"
except Exception, e:
debug.critical("An error occurred", e)
self._repository_status['details'] = "An error occurred"
def run_workflow(fname, version=None, params=''):
app = vistrails.core.application.init()
del vistrails.core.application.VistrailsApplicationInterface.__del__
# print app.configuration
loc = FileLocator(fname)
run_and_get_results([(loc, version)], params)
def exportMashup(filename, vtcontroller, mashuptrail, mashupversion,
etype):
"""exportMashup(filename: str, vtcontroller: VistrailController,
mashuptrail: Mashuptrail, type: int) -> bool
where etype is
0: include full tree
1: include only workflow and mashup identified by version
2: as a link, it will point to a local file.
"""
result = False
if vtcontroller is not None and mashuptrail is not None:
locator = vtcontroller.locator
version = mashuptrail.vtVersion
node = ElementTree.Element('vtlink')
if isinstance(locator, DBLocator):
node.set('host', str(locator.host))
node.set('port', str(locator.port))
node.set('database', str(locator.db))
node.set('vtid', str(locator.obj_id))
else:
node.set('filename', str(locator.name))
node.set('version', str(version))
node.set('execute', "True")
node.set('forceDB', "False")
node.set('showSpreadsheetOnly', "True")
node.set('mashuptrail', str(mashuptrail.id))
node.set('mashupVersion', str(mashupversion))
if etype in [0, 1]:
if etype == 1: #minimal
pip = vtcontroller.vistrail.getPipeline(version)
vistrail = Vistrail()
id_remap = {}
action = vistrails.core.db.action.create_paste_action(
pip, vistrail.idScope, id_remap)
vistrail.add_action(action, 0L, 0)
tag = vtcontroller.vistrail.get_tag(version)
if tag is None:
tag = "Imported workflow"
vistrail.addTag(tag, action.id)
node.set('version', str(action.id))
id_scope = IdScope(1L)
newmashuptrail = Mashuptrail(
MashupsManager.getNewMashuptrailId(), action.id,
id_scope)
maction = mashuptrail.actionMap[mashupversion]
mtag = mashuptrail.getTagForActionId(mashupversion)
newmashup = copy.copy(maction.mashup)
newmashup.remapPipelineObjects(id_remap)
currVersion = newmashuptrail.addVersion(
newmashuptrail.getLatestVersion(), newmashup,
maction.user, maction.date)
newmashuptrail.currentVersion = currVersion
newmashuptrail.changeTag(currVersion, mtag, maction.user,
maction.date)
newvtcontroller = BaseVistrailController(vistrail, None)
MashupsManager.addMashuptrailtoVistrailController(
newvtcontroller, newmashuptrail)
node.set('mashuptrail', str(newmashuptrail.id))
node.set('mashupVersion',
str(newmashuptrail.currentVersion))
else:
vistrail = vtcontroller.vistrail
newvtcontroller = MashupsManager.copyBaseVistrailController(
vtcontroller)
#create temporary file
(fd, name) = tempfile.mkstemp(prefix='vt_tmp', suffix='.vt')
os.close(fd)
try:
fileLocator = FileLocator(name)
newvtcontroller.write_vistrail(fileLocator)
contents = open(name).read()
vtcontent = base64.b64encode(contents)
finally:
os.unlink(name)
#if not vistrail.db_version:
# vistrail.db_version = currentVersion
node.set('vtcontent', vtcontent)
xmlstring = ElementTree.tostring(node)
file_ = open(filename, 'w')
file_.write(xmlstring)
file_.close()
result = True
return result
# Socket to send messages to
sender = context.socket(zmq.PUSH)
sender.connect("tcp://{0}:{1}".format(HOST, SEND))
while True:
# Receiving pipeline instance configuration
data = receiver.recv()
logging.debug('Receiving: ' + data)
fields = data.split("|")
filename = fields[0]
parameter_list = ast.literal_eval(fields[1])
inputs = ast.literal_eval(fields[2])
outputs = ast.literal_eval(fields[3])
locator = FileLocator(filename)
loaded_objs = vistrails.core.db.io.load_vistrail(locator)
controller = VistrailController(loaded_objs[0], locator, *loaded_objs[1:])
controller.do_version_switch(controller.get_latest_version_in_graph())
pipeline = Pipeline(controller)
kwargs = {}
for i in range(len(parameter_list)):
kwargs[inputs[i]] = parameter_list[i]
try:
#Executing pipeline instance and retieving the result
result = pipeline.execute(**kwargs)
for output in outputs:
parameter_list.append(str(result.output_port(output)))
except:
traceback.print_exc(file=sys.stdout)
def saveAsFile(self):
from vistrails.core.db.locator import DBLocator, FileLocator
bw = get_vistrails_application().builderWindow
bw.get_current_view().save_vistrail_as(bw.dbDefault and DBLocator
or FileLocator())
def update_from_directory(self, directory):
filenames = glob.glob(os.path.join(directory, '*.vt'))
for filename in filenames:
locator = FileLocator(filename)
url = locator.to_url()
self.updateVistrail(url)
def update_from_directory(self, directory):
filenames = glob.glob(os.path.join(directory, '*.vt'))
for filename in filenames:
locator = FileLocator(filename)
url = locator.to_url()
self.updateVistrail(url)