def read_vistrail_from_db(db_connection, abs_id, version):
import vistrails.db.services.io
from vistrails.core.vistrail.vistrail import Vistrail
vistrail = vistrails.db.services.io.open_vistrail_from_db(
db_connection, abs_id, version)
Vistrail.convert(vistrail)
return vistrail
def generate_vtl(locator,
version,
pipeline,
execute=False,
forceDB=False,
showSpreadsheetOnly=False,
embedWorkflow=False):
"""generate_vtl(locator:DBLocator or XMLLocator,
version: str, pipeline:Pipeline, execute:boolean,
forceDB:boolean, showspreadsheetOnly:boolean,
embedWorkflow: boolean) -> str
It generates the contents of a .vtl file with the information
given.
"""
node = ElementTree.Element('vtlink')
if isinstance(locator, DBLocator):
node.set('host', str(locator.host))
node.set('port', str(locator.port))
node.set('database', str(locator.db))
node.set('vtid', str(locator.obj_id))
elif locator is not None:
node.set('filename', str(locator.name))
node.set('version', str(version))
node.set('execute', str(execute))
node.set('forceDB', str(forceDB))
node.set('showSpreadsheetOnly', str(showSpreadsheetOnly))
if embedWorkflow == True:
vistrail = Vistrail()
action_list = []
for module in pipeline.module_list:
action_list.append(('add', module))
for connection in pipeline.connection_list:
action_list.append(('add', connection))
action = vistrails.core.db.action.create_action(action_list)
vistrail.add_action(action, 0L)
vistrail.addTag("Imported workflow", action.id)
if not forceDB:
node.set('version', str(action.id))
if not vistrail.db_version:
vistrail.db_version = currentVersion
pipxmlstr = io.serialize(vistrail)
vtcontent = base64.b64encode(pipxmlstr)
node.set('vtcontent', vtcontent)
return ElementTree.tostring(node)
def make_pipeline(self):
"""Creates an example pipeline that is used to conduct tests.
"""
vistrail = Vistrail()
controller = VistrailController(vistrail)
controller.change_selected_version(0)
# 0 1 2 7 8
# | / \ \ /
# 3 4 5 9
# | / \
# 6 10 11
modules = [
controller.add_module('org.vistrails.vistrails.basic', 'String')
for i in xrange(12)
]
def connect(outmod, inmod):
controller.add_connection(modules[outmod].id, 'value',
modules[inmod].id, 'value')
for (outmod, inmod) in [(0, 3), (1, 4), (1, 5), (5, 6), (7, 9), (8, 9),
(9, 10), (9, 11)]:
connect(outmod, inmod)
return controller, modules
def pipeline_from_generator(variable_gen):
# Get the original OutputPort module
orig_controller = variable_gen._generator.controller
base_pipeline = orig_controller.vistrail.getPipeline('dat-vars')
if len(base_pipeline.module_list) != 1:
raise ValueError("dat-vars version is invalid")
output_port = base_pipeline.module_list[0]
controller = VistrailController(Vistrail())
# OutputPort
operations = [('add', output_port)]
# Rest of the pipeline
operations += variable_gen._generator.operations
# Connection
connection = controller.create_connection(
variable_gen._output_module, variable_gen._outputport_name,
output_port, 'InternalPipe')
operations.append(('add', connection))
# Materialize this
action = create_action(operations)
controller.add_new_action(action)
version = controller.perform_action(action)
controller.change_selected_version(version)
assert version == controller.current_version == 1
return controller.current_pipeline, 1
def test_find_modules_by_type(self):
"""Tests the find_modules_by_type() function.
"""
vistrail = Vistrail()
controller = VistrailController(vistrail)
controller.change_selected_version(0)
mod1 = controller.add_module('org.vistrails.vistrails.basic', 'String')
mod2 = controller.add_module('org.vistrails.vistrails.basic', 'Float')
mod3 = controller.add_module('org.vistrails.vistrails.basic', 'String')
mod4 = controller.add_module('org.vistrails.vistrails.basic',
'Integer')
from dat.vistrails_interface import find_modules_by_type
from vistrails.core.modules.basic_modules import Boolean, Float, String
self.assertEqual(
set(m.id for m in find_modules_by_type(controller.current_pipeline,
[String])),
set([mod1.id, mod3.id]))
self.assertEqual([
m.id
for m in find_modules_by_type(controller.current_pipeline, [Float])
], [mod2.id, mod4.id])
self.assertEqual(
find_modules_by_type(controller.current_pipeline, [Boolean]), [])
def open_workflow(self, locator, version=None):
self.close_first_vistrail_if_necessary()
if self.single_document_mode and self.currentView():
self.closeVistrail()
vistrail = Vistrail()
try:
if locator is not None:
workflow = locator.load(Pipeline)
action_list = []
for module in workflow.module_list:
action_list.append(('add', module))
for connection in workflow.connection_list:
action_list.append(('add', connection))
action = vistrails.core.db.action.create_action(action_list)
vistrail.add_action(action, 0L)
vistrail.update_id_scope()
vistrail.addTag("Imported workflow", action.id)
# FIXME might need different locator?
except ModuleRegistryException, e:
msg = ('Cannot find module "%s" in package "%s". '
'Make sure package is '
'enabled in the Preferences dialog.' % \
(e._name, e._identifier))
debug.critical(msg)
def open_workflow(self, locator):
if isinstance(locator, basestring):
locator = BaseLocator.from_url(locator)
vistrail = Vistrail()
try:
if locator is None:
return False
if locator is not None:
workflow = locator.load(Pipeline)
action_list = []
for module in workflow.module_list:
action_list.append(('add', module))
for connection in workflow.connection_list:
action_list.append(('add', connection))
action = vistrails.core.db.action.create_action(action_list)
vistrail.add_action(action, 0L)
vistrail.update_id_scope()
vistrail.addTag("Imported workflow", action.id)
# FIXME might need different locator?
controller = self.add_vistrail(vistrail, locator)
except VistrailsDBException, e:
import traceback
debug.critical("Exception from the database",
traceback.format_exc())
return None
def load_vistrail(locator, is_abstraction=False):
from vistrails.core.vistrail.vistrail import Vistrail
abstraction_files = []
thumbnail_files = []
mashups = []
vistrail = None
if locator is None:
vistrail = Vistrail()
else:
res = locator.load()
if type(res) == type(SaveBundle(None)):
vistrail = res.vistrail
abstraction_files.extend(res.abstractions)
thumbnail_files.extend(res.thumbnails)
mashups.extend(res.mashups)
else:
vistrail = res
vistrail.is_abstraction = is_abstraction
return (vistrail, abstraction_files, thumbnail_files, mashups)
def build_widget(self):
layout = QtGui.QVBoxLayout()
layout.setMargin(0)
layout.setSpacing(0)
self.query_controller = QueryController(self)
self.vt_controller = VistrailController(auto_save=False)
self.p_controller = VistrailController(Vistrail(), auto_save=False)
self.connect(self.p_controller,
QtCore.SIGNAL('vistrailChanged()'),
self.vistrailChanged)
self.query_box = QQueryBox()
self.query_box.set_controller(self.query_controller)
layout.addWidget(self.query_box)
self.stacked_widget = QtGui.QStackedWidget()
self.pipeline_view = QQueryPipelineView()
self.p_controller.current_pipeline_view = self.pipeline_view
self.pipeline_view.set_controller(self.p_controller)
self.pipeline_view.set_query_controller(self.query_controller)
QQueryView.VISUAL_SEARCH_VIEW = \
self.stacked_widget.addWidget(self.pipeline_view)
self.global_result_view = QQueryResultGlobalView()
QQueryView.GLOBAL_RESULT_VIEW = \
self.stacked_widget.addWidget(self.global_result_view)
self.version_result_view = QQueryResultVersionView()
self.connect(self.version_result_view.scene(),
QtCore.SIGNAL('versionSelected(int,bool,bool,bool,bool)'),
self.result_version_selected)
# self.version_result_view.set_controller(self.vt_controller)
QQueryView.VERSION_RESULT_VIEW = \
self.stacked_widget.addWidget(self.version_result_view)
self.workflow_result_view = QQueryResultWorkflowView()
# self.workflow_result_view.set_controller(self.vt_controller)
QQueryView.WORKFLOW_RESULT_VIEW = \
self.stacked_widget.addWidget(self.workflow_result_view)
self.stacked_widget.setCurrentWidget(self.pipeline_view)
layout.addWidget(self.stacked_widget)
self.setLayout(layout)
self.current_display = QQueryView.VISUAL_SEARCH_VIEW
self.current_result_view = QQueryView.VERSION_RESULT_VIEW
def generate_vtl(locator,version,pipeline,execute=False,forceDB=False,
showSpreadsheetOnly=False,embedWorkflow=False):
"""generate_vtl(locator:DBLocator or XMLLocator,
version: str, pipeline:Pipeline, execute:boolean,
forceDB:boolean, showspreadsheetOnly:boolean,
embedWorkflow: boolean) -> str
It generates the contents of a .vtl file with the information
given.
"""
node = ElementTree.Element('vtlink')
if isinstance(locator, DBLocator):
node.set('host', str(locator.host))
node.set('port', str(locator.port))
node.set('database', str(locator.db))
node.set('vtid', str(locator.obj_id))
elif locator is not None:
node.set('filename', str(locator.name))
node.set('version', str(version))
node.set('execute', str(execute))
node.set('forceDB', str(forceDB))
node.set('showSpreadsheetOnly', str(showSpreadsheetOnly))
if embedWorkflow == True:
vistrail = Vistrail()
action_list = []
for module in pipeline.module_list:
action_list.append(('add', module))
for connection in pipeline.connection_list:
action_list.append(('add', connection))
action = vistrails.core.db.action.create_action(action_list)
vistrail.add_action(action, 0L)
vistrail.addTag("Imported workflow", action.id)
if not forceDB:
node.set('version', str(action.id))
if not vistrail.db_version:
vistrail.db_version = currentVersion
pipxmlstr = io.serialize(vistrail)
vtcontent = base64.b64encode(pipxmlstr)
node.set('vtcontent',vtcontent)
return ElementTree.tostring(node)
def execute_wf(wf, output_port):
# Save the workflow in a temporary file
temp_wf_fd, temp_wf = tempfile.mkstemp()
try:
f = open(temp_wf, 'w')
f.write(wf)
f.close()
os.close(temp_wf_fd)
# Clean the cache
interpreter = get_default_interpreter()
interpreter.flush()
# Load the Pipeline from the temporary file
vistrail = Vistrail()
locator = XMLFileLocator(temp_wf)
workflow = locator.load(Pipeline)
# Build a Vistrail from this single Pipeline
action_list = []
for module in workflow.module_list:
action_list.append(('add', module))
for connection in workflow.connection_list:
action_list.append(('add', connection))
action = vistrails.core.db.action.create_action(action_list)
vistrail.add_action(action, 0L)
vistrail.update_id_scope()
tag = 'parallel flow'
vistrail.addTag(tag, action.id)
# Build a controller and execute
controller = VistrailController()
controller.set_vistrail(vistrail, None)
controller.change_selected_version(vistrail.get_version_number(tag))
execution = controller.execute_current_workflow(
custom_aliases=None,
custom_params=None,
extra_info=None,
reason='API Pipeline Execution')
# Build a list of errors
errors = []
pipeline = vistrail.getPipeline(tag)
execution_errors = execution[0][0].errors
if execution_errors:
for key in execution_errors:
module = pipeline.modules[key]
msg = '%s: %s' % (module.name, execution_errors[key])
errors.append(msg)
# Get the execution log from the controller
try:
module_log = controller.log.workflow_execs[0].item_execs[0]
except IndexError:
errors.append("Module log not found")
return dict(errors=errors)
else:
machine = controller.log.workflow_execs[0].machines[
module_log.machine_id]
xml_log = serialize(module_log)
machine_log = serialize(machine)
# Get the output value
output = None
serializable = None
if not execution_errors:
executed_module, = execution[0][0].executed
executed_module = execution[0][0].objects[executed_module]
try:
output = executed_module.get_output(output_port)
except ModuleError:
errors.append("Output port not found: %s" % output_port)
return dict(errors=errors)
reg = vistrails.core.modules.module_registry.get_module_registry()
base_classes = inspect.getmro(type(output))
if Module in base_classes:
serializable = reg.get_descriptor(type(output)).sigstring
output = output.serialize()
# Return the dictionary, that will be sent back to the client
return dict(errors=errors,
output=output,
serializable=serializable,
xml_log=xml_log,
machine_log=machine_log)
finally:
os.unlink(temp_wf)
def read_vistrail_from_db(db_connection, abs_id, version):
import vistrails.db.services.io
from vistrails.core.vistrail.vistrail import Vistrail
vistrail = vistrails.db.services.io.open_vistrail_from_db(db_connection, abs_id, version)
Vistrail.convert(vistrail)
return vistrail
def read_vistrail(vt_fname):
import vistrails.db.services.io
from vistrails.core.vistrail.vistrail import Vistrail
vistrail = vistrails.db.services.io.open_vistrail_from_xml(vt_fname)
Vistrail.convert(vistrail)
return vistrail
def read_vistrail(vt_fname):
import vistrails.db.services.io
from vistrails.core.vistrail.vistrail import Vistrail
vistrail = vistrails.db.services.io.open_vistrail_from_xml(vt_fname)
Vistrail.convert(vistrail)
return vistrail
def execute(workflowJSON):
''' Execute a workflow from it's JSON representation
'''
debug('convert json to xml')
workflowXML = json2xml(workflowJSON)
#temp_wf_fd, temp_wf = tempfile.mkstemp('.xml')
debug('create temporary file')
temp_wf_fd, temp_wf = tempfile.mkstemp()
try:
f = open(temp_wf, 'w')
f.write(workflowXML)
f.close()
os.close(temp_wf_fd)
#load workflow temp file into vistrails
#vt.load_workflow(temp_wf)
#execute workflow
#execution = vt.execute()
debug('Load the Pipeline from the temporary file')
vistrail = Vistrail()
locator = XMLFileLocator(temp_wf)
workflow = locator.load(Pipeline)
debug('Build a Vistrail from this single Pipeline')
action_list = []
for module in workflow.module_list:
action_list.append(('add', module))
for connection in workflow.connection_list:
action_list.append(('add', connection))
action = vistrails.core.db.action.create_action(action_list)
debug('add actions')
vistrail.add_action(action, 0L)
vistrail.update_id_scope()
tag = 'climatepipes'
vistrail.addTag(tag, action.id)
debug('Build a controller and execute')
controller = VistrailController()
controller.set_vistrail(vistrail, None)
controller.change_selected_version(vistrail.get_version_number(tag))
execution = controller.execute_current_workflow(
custom_aliases=None,
custom_params=None,
extra_info=None,
reason='API Pipeline Execution')
debug('get result')
execution_pipeline = execution[0][0]
if len(execution_pipeline.errors) > 0:
error("Executing workflow")
for key in execution_pipeline.errors:
error(execution_pipeline.errors[key])
print execution_pipeline.errors[key]
return None
modules = execution_pipeline.objects
for id, module in modules.iteritems():
if isinstance(module, ToGeoJSON):
return json.dumps({'result': module.JSON, 'error': None })
finally:
os.unlink(temp_wf)
def execute_wf(wf, output_port):
# Save the workflow in a temporary file
temp_wf_fd, temp_wf = tempfile.mkstemp()
try:
f = open(temp_wf, 'w')
f.write(wf)
f.close()
os.close(temp_wf_fd)
# Clean the cache
interpreter = get_default_interpreter()
interpreter.flush()
# Load the Pipeline from the temporary file
vistrail = Vistrail()
locator = XMLFileLocator(temp_wf)
workflow = locator.load(Pipeline)
# Build a Vistrail from this single Pipeline
action_list = []
for module in workflow.module_list:
action_list.append(('add', module))
for connection in workflow.connection_list:
action_list.append(('add', connection))
action = vistrails.core.db.action.create_action(action_list)
vistrail.add_action(action, 0L)
vistrail.update_id_scope()
tag = 'parallel flow'
vistrail.addTag(tag, action.id)
# Build a controller and execute
controller = VistrailController()
controller.set_vistrail(vistrail, None)
controller.change_selected_version(vistrail.get_version_number(tag))
execution = controller.execute_current_workflow(
custom_aliases=None,
custom_params=None,
extra_info=None,
reason='API Pipeline Execution')
# Build a list of errors
errors = []
pipeline = vistrail.getPipeline(tag)
execution_errors = execution[0][0].errors
if execution_errors:
for key in execution_errors:
module = pipeline.modules[key]
msg = '%s: %s' %(module.name, execution_errors[key])
errors.append(msg)
# Get the execution log from the controller
try:
module_log = controller.log.workflow_execs[0].item_execs[0]
except IndexError:
errors.append("Module log not found")
return dict(errors=errors)
else:
machine = controller.log.workflow_execs[0].machines[
module_log.machine_id]
xml_log = serialize(module_log)
machine_log = serialize(machine)
# Get the output value
output = None
serializable = None
if not execution_errors:
executed_module, = execution[0][0].executed
executed_module = execution[0][0].objects[executed_module]
try:
output = executed_module.get_output(output_port)
except ModuleError:
errors.append("Output port not found: %s" % output_port)
return dict(errors=errors)
reg = vistrails.core.modules.module_registry.get_module_registry()
base_classes = inspect.getmro(type(output))
if Module in base_classes:
serializable = reg.get_descriptor(type(output)).sigstring
output = output.serialize()
# Return the dictionary, that will be sent back to the client
return dict(errors=errors,
output=output,
serializable=serializable,
xml_log=xml_log,
machine_log=machine_log)
finally:
os.unlink(temp_wf)
def exportMashup(filename, vtcontroller, mashuptrail, mashupversion, etype):
"""exportMashup(filename: str, vtcontroller: VistrailController,
mashuptrail: Mashuptrail, type: int) -> bool
where etype is
0: include full tree
1: include only workflow and mashup identified by version
2: as a link, it will point to a local file.
"""
result = False
if vtcontroller is not None and mashuptrail is not None:
locator = vtcontroller.locator
version = mashuptrail.vtVersion
node = ElementTree.Element('vtlink')
if isinstance(locator, DBLocator):
node.set('host', str(locator.host))
node.set('port', str(locator.port))
node.set('database', str(locator.db))
node.set('vtid', str(locator.obj_id))
else:
node.set('filename', str(locator.name))
node.set('version', str(version))
node.set('execute', "True")
node.set('forceDB', "False")
node.set('showSpreadsheetOnly', "True")
node.set('mashuptrail', str(mashuptrail.id))
node.set('mashupVersion', str(mashupversion))
if etype in [0,1]:
if etype == 1: #minimal
pip = vtcontroller.vistrail.getPipeline(version)
vistrail = Vistrail()
id_remap = {}
action = vistrails.core.db.action.create_paste_action(pip,
vistrail.idScope,
id_remap)
vistrail.add_action(action, 0L, 0)
tag = vtcontroller.vistrail.get_tag(version)
if tag is None:
tag = "Imported workflow"
vistrail.addTag(tag, action.id)
node.set('version', str(action.id))
id_scope = IdScope(1L)
newmashuptrail = Mashuptrail(
MashupsManager.getNewMashuptrailId(),
action.id,
id_scope)
maction = mashuptrail.actionMap[mashupversion]
mtag = mashuptrail.getTagForActionId(mashupversion)
newmashup = copy.copy(maction.mashup)
newmashup.remapPipelineObjects(id_remap)
currVersion = newmashuptrail.addVersion(
newmashuptrail.getLatestVersion(),
newmashup, maction.user,
maction.date)
newmashuptrail.currentVersion = currVersion
newmashuptrail.changeTag(currVersion, mtag, maction.user,
maction.date)
newvtcontroller = BaseVistrailController(vistrail, None)
MashupsManager.addMashuptrailtoVistrailController(newvtcontroller,
newmashuptrail)
node.set('mashuptrail', str(newmashuptrail.id))
node.set('mashupVersion', str(newmashuptrail.currentVersion))
else:
vistrail = vtcontroller.vistrail
newvtcontroller = MashupsManager.copyBaseVistrailController(vtcontroller)
#create temporary file
(fd, name) = tempfile.mkstemp(prefix='vt_tmp',
suffix='.vt')
os.close(fd)
try:
fileLocator = FileLocator(name)
newvtcontroller.write_vistrail(fileLocator)
contents = open(name).read()
vtcontent = base64.b64encode(contents)
finally:
os.unlink(name)
#if not vistrail.db_version:
# vistrail.db_version = currentVersion
node.set('vtcontent',vtcontent)
xmlstring = ElementTree.tostring(node)
file_ = open(filename,'w')
file_.write(xmlstring)
file_.close()
result = True
return result
def exportMashup(filename, vtcontroller, mashuptrail, mashupversion,
etype):
"""exportMashup(filename: str, vtcontroller: VistrailController,
mashuptrail: Mashuptrail, type: int) -> bool
where etype is
0: include full tree
1: include only workflow and mashup identified by version
2: as a link, it will point to a local file.
"""
result = False
if vtcontroller is not None and mashuptrail is not None:
locator = vtcontroller.locator
version = mashuptrail.vtVersion
node = ElementTree.Element('vtlink')
if isinstance(locator, DBLocator):
node.set('host', str(locator.host))
node.set('port', str(locator.port))
node.set('database', str(locator.db))
node.set('vtid', str(locator.obj_id))
else:
node.set('filename', str(locator.name))
node.set('version', str(version))
node.set('execute', "True")
node.set('forceDB', "False")
node.set('showSpreadsheetOnly', "True")
node.set('mashuptrail', str(mashuptrail.id))
node.set('mashupVersion', str(mashupversion))
if etype in [0, 1]:
if etype == 1: #minimal
pip = vtcontroller.vistrail.getPipeline(version)
vistrail = Vistrail()
id_remap = {}
action = vistrails.core.db.action.create_paste_action(
pip, vistrail.idScope, id_remap)
vistrail.add_action(action, 0L, 0)
tag = vtcontroller.vistrail.get_tag(version)
if tag is None:
tag = "Imported workflow"
vistrail.addTag(tag, action.id)
node.set('version', str(action.id))
id_scope = IdScope(1L)
newmashuptrail = Mashuptrail(
MashupsManager.getNewMashuptrailId(), action.id,
id_scope)
maction = mashuptrail.actionMap[mashupversion]
mtag = mashuptrail.getTagForActionId(mashupversion)
newmashup = copy.copy(maction.mashup)
newmashup.remapPipelineObjects(id_remap)
currVersion = newmashuptrail.addVersion(
newmashuptrail.getLatestVersion(), newmashup,
maction.user, maction.date)
newmashuptrail.currentVersion = currVersion
newmashuptrail.changeTag(currVersion, mtag, maction.user,
maction.date)
newvtcontroller = BaseVistrailController(vistrail, None)
MashupsManager.addMashuptrailtoVistrailController(
newvtcontroller, newmashuptrail)
node.set('mashuptrail', str(newmashuptrail.id))
node.set('mashupVersion',
str(newmashuptrail.currentVersion))
else:
vistrail = vtcontroller.vistrail
newvtcontroller = MashupsManager.copyBaseVistrailController(
vtcontroller)
#create temporary file
(fd, name) = tempfile.mkstemp(prefix='vt_tmp', suffix='.vt')
os.close(fd)
try:
fileLocator = FileLocator(name)
newvtcontroller.write_vistrail(fileLocator)
contents = open(name).read()
vtcontent = base64.b64encode(contents)
finally:
os.unlink(name)
#if not vistrail.db_version:
# vistrail.db_version = currentVersion
node.set('vtcontent', vtcontent)
xmlstring = ElementTree.tostring(node)
file_ = open(filename, 'w')
file_.write(xmlstring)
file_.close()
result = True
return result
