def compute(self):
url = URL(drivername=self.get_input('protocol'),
username=self.force_get_input('user', None),
password=self.force_get_input('password', None),
host=self.force_get_input('host', None),
port=self.force_get_input('port', None),
database=self.get_input('db_name'))
try:
engine = create_engine(url)
except ImportError, e:
driver = url.drivername
installed = False
if driver == 'sqlite':
raise ModuleError(self,
"Python was built without sqlite3 support")
elif (driver == 'mysql'
or driver == 'drizzle'): # drizzle is a variant of MySQL
installed = install({
'pip': 'mysql-python',
'linux-debian': 'python-mysqldb',
'linux-ubuntu': 'python-mysqldb',
'linux-fedora': 'MySQL-python'
})
elif (driver == 'postgresql'
or driver == 'postgre'): # deprecated alias
installed = install({
'pip': 'psycopg2',
'linux-debian': 'python-psycopg2',
'linux-ubuntu': 'python-psycopg2',
'linux-fedora': 'python-psycopg2'
})
elif driver == 'firebird':
installed = install({
'pip': 'fdb',
'linux-fedora': 'python-fdb'
})
elif driver == 'mssql' or driver == 'sybase':
installed = install({
'pip': 'pyodbc',
'linux-debian': 'python-pyodbc',
'linux-ubuntu': 'python-pyodbc',
'linux-fedora': 'pyodbc'
})
elif driver == 'oracle':
installed = install({'pip': 'cx_Oracle'})
else:
raise ModuleError(
self, "SQLAlchemy couldn't connect: %s" %
debug.format_exception(e))
if not installed:
raise ModuleError(self, "Failed to install required driver")
try:
engine = create_engine(url)
except Exception, e:
raise ModuleError(
self, "Couldn't connect to the database: %s" %
debug.format_exception(e))
def compute(self):
url = URL(drivername=self.get_input('protocol'),
username=self.force_get_input('user', None),
password=self.force_get_input('password', None),
host=self.force_get_input('host', None),
port=self.force_get_input('port', None),
database=self.get_input('db_name'))
try:
engine = create_engine(url)
except ImportError, e:
driver = url.drivername
installed = False
if driver == 'sqlite':
raise ModuleError(self,
"Python was built without sqlite3 support")
elif (driver == 'mysql' or
driver == 'drizzle'): # drizzle is a variant of MySQL
installed = install({
'pip': 'mysql-python',
'linux-debian': 'python-mysqldb',
'linux-ubuntu': 'python-mysqldb',
'linux-fedora': 'MySQL-python'})
elif (driver == 'postgresql' or
driver == 'postgre'): # deprecated alias
installed = install({
'pip': 'psycopg2',
'linux-debian':'python-psycopg2',
'linux-ubuntu':'python-psycopg2',
'linux-fedora':'python-psycopg2'})
elif driver == 'firebird':
installed = install({
'pip': 'fdb',
'linux-fedora':'python-fdb'})
elif driver == 'mssql' or driver == 'sybase':
installed = install({
'pip': 'pyodbc',
'linux-debian':'python-pyodbc',
'linux-ubuntu':'python-pyodbc',
'linux-fedora':'pyodbc'})
elif driver == 'oracle':
installed = install({
'pip': 'cx_Oracle'})
else:
raise ModuleError(self,
"SQLAlchemy couldn't connect: %s" %
debug.format_exception(e))
if not installed:
raise ModuleError(self,
"Failed to install required driver")
try:
engine = create_engine(url)
except Exception, e:
raise ModuleError(self,
"Couldn't connect to the database: %s" %
debug.format_exception(e))
def compute(self):
cached = False
if self.has_input('cacheResults'):
cached = self.get_input('cacheResults')
self.is_cacheable = lambda: cached
connection = self.get_input('connection')
inputs = dict((k, self.get_input(k)) for k in self.inputPorts.iterkeys()
if k not in ('source', 'connection', 'cacheResults'))
s = urllib.unquote(str(self.get_input('source')))
try:
transaction = connection.begin()
results = connection.execute(s, inputs)
try:
rows = results.fetchall()
except Exception:
self.set_output('result', None)
self.set_output('resultSet', None)
else:
# results.returns_rows is True
# We don't use 'if return_rows' because this attribute didn't
# use to exist
table = TableObject.from_dicts(rows, results.keys())
self.set_output('result', table)
self.set_output('resultSet', rows)
transaction.commit()
except SQLAlchemyError, e:
raise ModuleError(self, debug.format_exception(e))
def download(self, response):
try:
dl_size = 0
CHUNKSIZE = 4096
f2 = open(self.local_filename, 'wb')
while True:
if self.size_header is not None:
self.module.logging.update_progress(
self.module,
dl_size * 1.0/self.size_header)
chunk = response.read(CHUNKSIZE)
if not chunk:
break
dl_size += len(chunk)
f2.write(chunk)
f2.close()
response.close()
except Exception, e:
try:
os.unlink(self.local_filename)
except OSError:
pass
raise ModuleError(
self.module,
"Error retrieving URL: %s" % debug.format_exception(e))
def compute(self):
# create dict of inputs
cacheable = False
if self.has_input('cacheable'):
cacheable = self.get_input('cacheable')
self.is_cacheable = lambda *args, **kwargs: cacheable
params = {}
mname = self.wsmethod.qname[0]
for name in self.wsmethod.inputs:
name = str(name)
if self.has_input(name):
params[name] = self.get_input(name)
if params[name].__class__.__name__ == 'UberClass':
params[name] = params[name].value
params[name] = self.service.makeDictType(params[name])
try:
#import logging
#logging.basicConfig(level=logging.INFO)
#logging.getLogger('suds.client').setLevel(logging.DEBUG)
#print "params:", str(params)[:400]
#self.service.service.set_options(retxml = True)
#result = getattr(self.service.service.service, mname)(**params)
#print "result:", str(result)[:400]
#self.service.service.set_options(retxml = False)
result = getattr(self.service.service.service,
mname)(**params)
except Exception, e:
debug.unexpected_exception(e)
raise ModuleError(
self, "Error invoking method %s: %s" %
(mname, debug.format_exception(e)))
def run_parameter_exploration(
locator,
pe_id,
extra_info={},
reason="Console Mode Parameter Exploration Execution"):
"""run_parameter_exploration(w_list: (locator, version),
pe_id: str/int,
reason: str) -> (pe_id, [error msg])
Run parameter exploration in w, and returns an interpreter result object.
version can be a tag name or a version id.
"""
if is_running_gui():
from vistrails.gui.vistrail_controller import VistrailController as \
GUIVistrailController
try:
(v, abstractions, thumbnails, mashups) = load_vistrail(locator)
controller = GUIVistrailController(v, locator, abstractions,
thumbnails, mashups)
try:
pe_id = int(pe_id)
pe = controller.vistrail.get_paramexp(pe_id)
except ValueError:
pe = controller.vistrail.get_named_paramexp(pe_id)
controller.change_selected_version(pe.action_id)
controller.executeParameterExploration(pe,
extra_info=extra_info,
showProgress=False)
except Exception, e:
return (locator, pe_id, debug.format_exception(e),
debug.format_exc())
def compute(self):
cached = False
if self.has_input('cacheResults'):
cached = self.get_input('cacheResults')
self.is_cacheable = lambda: cached
connection = self.get_input('connection')
inputs = dict((k, self.get_input(k))
for k in self.inputPorts.iterkeys()
if k not in ('source', 'connection', 'cacheResults'))
s = urllib.unquote(str(self.get_input('source')))
try:
transaction = connection.begin()
results = connection.execute(s, inputs)
try:
rows = results.fetchall()
except Exception:
self.set_output('result', None)
self.set_output('resultSet', None)
else:
# results.returns_rows is True
# We don't use 'if return_rows' because this attribute didn't
# use to exist
table = TableObject.from_dicts(rows, results.keys())
self.set_output('result', table)
self.set_output('resultSet', rows)
transaction.commit()
except SQLAlchemyError, e:
raise ModuleError(self, debug.format_exception(e))
def compute(self):
# create dict of inputs
cacheable = False
if self.has_input('cacheable'):
cacheable = self.get_input('cacheable')
self.is_cacheable = lambda *args, **kwargs: cacheable
params = {}
mname = self.wsmethod.qname[0]
for name in self.wsmethod.inputs:
name = str(name)
if self.has_input(name):
params[name] = self.get_input(name)
if params[name].__class__.__name__ == 'UberClass':
params[name] = params[name].value
params[name] = self.service.makeDictType(params[name])
try:
#import logging
#logging.basicConfig(level=logging.INFO)
#logging.getLogger('suds.client').setLevel(logging.DEBUG)
#print "params:", str(params)[:400]
#self.service.service.set_options(retxml = True)
#result = getattr(self.service.service.service, mname)(**params)
#print "result:", str(result)[:400]
#self.service.service.set_options(retxml = False)
result = getattr(self.service.service.service, mname)(**params)
except Exception, e:
debug.unexpected_exception(e)
raise ModuleError(self, "Error invoking method %s: %s" % (
mname, debug.format_exception(e)))
def get_vt_graph(vt_list, tree_info, pdf=False):
"""get_vt_graph(vt_list: list of locator, tree_info:str)
Load all vistrails in vt_list and dump their tree to tree_info.
"""
result = []
if is_running_gui():
from vistrails.gui.vistrail_controller import VistrailController as \
GUIVistrailController
for locator in vt_list:
try:
(v, abstractions , thumbnails, mashups) = load_vistrail(locator)
controller = GUIVistrailController(v, locator, abstractions,
thumbnails, mashups)
if tree_info is not None:
from vistrails.gui.version_view import QVersionTreeView
version_view = QVersionTreeView()
version_view.scene().setupScene(controller)
if pdf:
base_fname = "graph_%s.pdf" % locator.short_filename
filename = os.path.join(tree_info, base_fname)
version_view.scene().saveToPDF(filename)
else:
base_fname = "graph_%s.png" % locator.short_filename
filename = os.path.join(tree_info, base_fname)
version_view.scene().saveToPNG(filename)
del version_view
result.append((True, ""))
except Exception, e:
result.append((False, debug.format_exception(e)))
def run_parameter_exploration(locator, pe_id, extra_info = {},
reason="Console Mode Parameter Exploration Execution"):
"""run_parameter_exploration(w_list: (locator, version),
pe_id: str/int,
reason: str) -> (pe_id, [error msg])
Run parameter exploration in w, and returns an interpreter result object.
version can be a tag name or a version id.
"""
if is_running_gui():
from vistrails.gui.vistrail_controller import VistrailController as \
GUIVistrailController
try:
(v, abstractions , thumbnails, mashups) = load_vistrail(locator)
controller = GUIVistrailController(v, locator, abstractions,
thumbnails, mashups)
try:
pe_id = int(pe_id)
pe = controller.vistrail.get_paramexp(pe_id)
except ValueError:
pe = controller.vistrail.get_named_paramexp(pe_id)
controller.change_selected_version(pe.action_id)
controller.executeParameterExploration(pe, extra_info=extra_info,
showProgress=False)
except Exception, e:
return (locator, pe_id,
debug.format_exception(e), debug.format_exc())
def get_vt_graph(vt_list, tree_info, pdf=False):
"""get_vt_graph(vt_list: list of locator, tree_info:str)
Load all vistrails in vt_list and dump their tree to tree_info.
"""
result = []
if is_running_gui():
from vistrails.gui.vistrail_controller import VistrailController as \
GUIVistrailController
for locator in vt_list:
try:
(v, abstractions, thumbnails, mashups) = load_vistrail(locator)
controller = GUIVistrailController(v, locator, abstractions,
thumbnails, mashups)
if tree_info is not None:
from vistrails.gui.version_view import QVersionTreeView
version_view = QVersionTreeView()
version_view.scene().setupScene(controller)
if pdf:
base_fname = "graph_%s.pdf" % locator.short_filename
filename = os.path.join(tree_info, base_fname)
version_view.scene().saveToPDF(filename)
else:
base_fname = "graph_%s.png" % locator.short_filename
filename = os.path.join(tree_info, base_fname)
version_view.scene().saveToPNG(filename)
del version_view
result.append((True, ""))
except Exception, e:
result.append((False, debug.format_exception(e)))
def evaluate(i):
try:
v = d['value'](i)
if v is None:
return self._ptype.default_value
return v
except Exception, e:
return debug.format_exception(e)
def evaluate(i):
try:
v = d['value'](i)
if v is None:
return module.default_value
return v
except Exception, e:
debug.unexpected_exception(e)
return debug.format_exception(e)
def evaluate(i):
try:
v = d['value'](i)
if v == None:
return module.default_value
return v
except Exception, e:
debug.unexpected_exception(e)
return debug.format_exception(e)
def saveVistrailFileHook(self, vistrail, tmp_dir):
if hasattr(self._init_module, 'saveVistrailFileHook'):
try:
self._init_module.saveVistrailFileHook(vistrail, tmp_dir)
except Exception, e:
debug.unexpected_exception(e)
debug.critical("Got exception in %s's saveVistrailFileHook(): "
"%s\n%s" % (self.name,
debug.format_exception(e),
traceback.format_exc()))
def saveVistrailFileHook(self, vistrail, tmp_dir):
if hasattr(self._init_module, 'saveVistrailFileHook'):
try:
self._init_module.saveVistrailFileHook(vistrail, tmp_dir)
except Exception, e:
debug.unexpected_exception(e)
debug.critical("Got exception in %s's saveVistrailFileHook(): "
"%s\n%s" %
(self.name, debug.format_exception(e),
traceback.format_exc()))
def can_handle_vt_file(self, name):
""" Asks package if it can handle a file inside a zipped vt file
"""
try:
return (hasattr(self.init_module, 'can_handle_vt_file') and
self.init_module.can_handle_vt_file(name))
except Exception, e:
debug.unexpected_exception(e)
debug.critical("Got exception calling %s's can_handle_vt_file: "
"%s\n%s" % (self.name,
debug.format_exception(e),
traceback.format_exc()))
return False
def can_handle_identifier(self, identifier):
""" Asks package if it can handle this package
"""
try:
return (hasattr(self.init_module, 'can_handle_identifier') and
self.init_module.can_handle_identifier(identifier))
except Exception, e:
debug.unexpected_exception(e)
debug.critical("Got exception calling %s's can_handle_identifier: "
"%s\n%s" % (self.name,
debug.format_exception(e),
traceback.format_exc()))
return False
def runAndGetCellEvents(self, useDefaultValues=False):
spreadsheetController.setEchoMode(True)
#will run to get Spreadsheet Cell events
cellEvents = []
errors = []
try:
(res, errors) = self.run(useDefaultValues)
if res:
cellEvents = spreadsheetController.getEchoCellEvents()
except Exception, e:
import traceback
debug.unexpected_exception(e)
print "Executing pipeline failed:", debug.format_exception(e), traceback.format_exc()
def menu_items(self):
try:
callable_ = self._module.menu_items
except AttributeError:
return None
else:
try:
return callable_()
except Exception, e:
debug.unexpected_exception(e)
debug.critical("Couldn't load menu items for %s: %s\n%s" % (
self.name, debug.format_exception(e),
traceback.format_exc()))
def menu_items(self):
try:
callable_ = self._module.menu_items
except AttributeError:
return None
else:
try:
return callable_()
except Exception, e:
debug.unexpected_exception(e)
debug.critical("Couldn't load menu items for %s: %s\n%s" %
(self.name, debug.format_exception(e),
traceback.format_exc()))
def can_handle_vt_file(self, name):
""" Asks package if it can handle a file inside a zipped vt file
"""
try:
return (hasattr(self.init_module, 'can_handle_vt_file')
and self.init_module.can_handle_vt_file(name))
except Exception, e:
debug.unexpected_exception(e)
debug.critical(
"Got exception calling %s's can_handle_vt_file: "
"%s\n%s" %
(self.name, debug.format_exception(e), traceback.format_exc()))
return False
def can_handle_identifier(self, identifier):
""" Asks package if it can handle this package
"""
try:
return (hasattr(self.init_module, 'can_handle_identifier')
and self.init_module.can_handle_identifier(identifier))
except Exception, e:
debug.unexpected_exception(e)
debug.critical(
"Got exception calling %s's can_handle_identifier: "
"%s\n%s" %
(self.name, debug.format_exception(e), traceback.format_exc()))
return False
def dependencies(self):
deps = []
try:
callable_ = self._module.package_dependencies
except AttributeError:
pass
else:
try:
deps = callable_()
except Exception, e:
debug.critical("Couldn't get dependencies of %s: %s\n%s" %
(self.name, debug.format_exception(e),
traceback.format_exc()))
deps = []
def get_wf_graph(w_list, output_dir, pdf=False):
"""run_and_get_results(w_list: list of (locator, version),
output_dir:str, pdf:bool)
Load all workflows in wf_list and dump their graph to output_dir.
"""
result = []
if is_running_gui():
from vistrails.gui.vistrail_controller import VistrailController as \
GUIVistrailController
for locator, workflow in w_list:
try:
(v, abstractions, thumbnails, mashups) = load_vistrail(locator)
controller = GUIVistrailController(v,
locator,
abstractions,
thumbnails,
mashups,
auto_save=False)
# FIXME TE: why is this needed
controller.current_pipeline_view.set_controller(controller)
version = None
if isinstance(workflow, basestring):
version = v.get_version_number(workflow)
elif isinstance(workflow, (int, long)):
version = workflow
elif workflow is None:
version = controller.get_latest_version_in_graph()
else:
msg = "Invalid version tag or number: %s" % workflow
raise VistrailsInternalError(msg)
controller.change_selected_version(version)
if controller.current_pipeline is not None:
controller.updatePipelineScene()
if pdf:
base_fname = "%s_%s_pipeline.pdf" % \
(locator.short_filename, version)
filename = os.path.join(output_dir, base_fname)
controller.current_pipeline_scene.saveToPDF(filename)
else:
base_fname = "%s_%s_pipeline.png" % \
(locator.short_filename, version)
filename = os.path.join(output_dir, base_fname)
controller.current_pipeline_scene.saveToPNG(filename)
result.append((True, ""))
except Exception, e:
result.append((False, debug.format_exception(e)))
def dependencies(self):
deps = []
try:
callable_ = self._module.package_dependencies
except AttributeError:
pass
else:
try:
deps = callable_()
except Exception, e:
debug.critical(
"Couldn't get dependencies of %s: %s\n%s" % (
self.name, debug.format_exception(e),
traceback.format_exc()))
deps = []
def test(self):
try:
errs = []
filename = os.path.join(EXAMPLES_PATH, vtfile)
locator = FileLocator(os.path.abspath(filename))
(v, abstractions, thumbnails, mashups) = load_vistrail(locator)
errs = vistrails.core.console_mode.run(
[(locator, version)],
update_vistrail=False,
extra_info={'compare_thumbnails': compare_thumbnails})
if len(errs) > 0:
for err in errs:
print(" *** Error in %s:%s:%s -- %s" % err)
self.fail(str(err))
except Exception, e:
self.fail(debug.format_exception(e))
def finalize(self):
if not self._initialized:
return
debug.log("Finalizing %s" % self.name)
try:
callable_ = self._module.finalize
except AttributeError:
pass
else:
try:
callable_()
except Exception, e:
debug.unexpected_exception(e)
debug.critical("Couldn't finalize %s: %s\n%s" %
(self.name, debug.format_exception(e),
traceback.format_exc()))
def test(self):
try:
errs = []
filename = os.path.join(EXAMPLES_PATH, vtfile)
locator = FileLocator(os.path.abspath(filename))
(v, abstractions, thumbnails, mashups) = load_vistrail(locator)
errs = vistrails.core.console_mode.run(
[(locator, version)],
update_vistrail=False,
extra_info={'compare_thumbnails': compare_thumbnails})
if len(errs) > 0:
for err in errs:
print(" *** Error in %s:%s:%s -- %s" % err)
self.fail(str(err))
except Exception, e:
self.fail(debug.format_exception(e))
def finalize(self):
if not self._initialized:
return
debug.log("Finalizing %s" % self.name)
try:
callable_ = self._module.finalize
except AttributeError:
pass
else:
try:
callable_()
except Exception, e:
debug.unexpected_exception(e)
debug.critical("Couldn't finalize %s: %s\n%s" % (
self.name, debug.format_exception(e),
traceback.format_exc()))
def get_wf_graph(w_list, output_dir, pdf=False):
"""run_and_get_results(w_list: list of (locator, version),
output_dir:str, pdf:bool)
Load all workflows in wf_list and dump their graph to output_dir.
"""
result = []
if is_running_gui():
from vistrails.gui.vistrail_controller import VistrailController as \
GUIVistrailController
for locator, workflow in w_list:
try:
(v, abstractions , thumbnails, mashups) = load_vistrail(locator)
controller = GUIVistrailController(v, locator, abstractions,
thumbnails, mashups,
auto_save=False)
# FIXME TE: why is this needed
controller.current_pipeline_view.set_controller(controller)
version = None
if isinstance(workflow, basestring):
version = v.get_version_number(workflow)
elif isinstance(workflow, (int, long)):
version = workflow
elif workflow is None:
version = controller.get_latest_version_in_graph()
else:
msg = "Invalid version tag or number: %s" % workflow
raise VistrailsInternalError(msg)
controller.change_selected_version(version)
if controller.current_pipeline is not None:
controller.updatePipelineScene()
if pdf:
base_fname = "%s_%s_pipeline.pdf" % \
(locator.short_filename, version)
filename = os.path.join(output_dir, base_fname)
controller.current_pipeline_scene.saveToPDF(filename)
else:
base_fname = "%s_%s_pipeline.png" % \
(locator.short_filename, version)
filename = os.path.join(output_dir, base_fname)
controller.current_pipeline_scene.saveToPNG(filename)
result.append((True, ""))
except Exception, e:
result.append((False, debug.format_exception(e)))
def contextMenuEvent(self, event):
"""Just dispatches the menu event to the widget item"""
item = self.itemAt(event.pos())
if item:
# find top level
p = item
while p.parent():
p = p.parent()
# get package identifier
assert isinstance(p, QPackageTreeWidgetItem)
identifier = p.identifier
registry = get_module_registry()
package = registry.packages[identifier]
try:
if package.has_context_menu():
if isinstance(item, QPackageTreeWidgetItem):
text = None
elif isinstance(item, QNamespaceTreeWidgetItem):
return # no context menu for namespaces
elif isinstance(item, QModuleTreeWidgetItem):
text = item.descriptor.name
if item.descriptor.namespace:
text = '%s|%s' % (item.descriptor.namespace, text)
else:
assert False, "fell through"
menu_items = package.context_menu(text)
if menu_items:
menu = QtGui.QMenu(self)
for text, callback in menu_items:
act = QtGui.QAction(text, self)
act.setStatusTip(text)
QtCore.QObject.connect(act,
QtCore.SIGNAL("triggered()"),
callback)
menu.addAction(act)
menu.exec_(event.globalPos())
return
except Exception, e:
debug.unexpected_exception(e)
debug.warning("Got exception trying to display %s's "
"context menu in the palette: %s\n%s" % (
package.name,
debug.format_exception(e),
traceback.format_exc()))
item.contextMenuEvent(event, self)
def contextMenuEvent(self, event):
"""Just dispatches the menu event to the widget item"""
item = self.itemAt(event.pos())
if item:
# find top level
p = item
while p.parent():
p = p.parent()
# get package identifier
assert isinstance(p, QPackageTreeWidgetItem)
identifier = p.identifier
registry = get_module_registry()
package = registry.packages[identifier]
try:
if package.has_context_menu():
if isinstance(item, QPackageTreeWidgetItem):
text = None
elif isinstance(item, QNamespaceTreeWidgetItem):
return # no context menu for namespaces
elif isinstance(item, QModuleTreeWidgetItem):
text = item.descriptor.name
if item.descriptor.namespace:
text = '%s|%s' % (item.descriptor.namespace, text)
else:
assert False, "fell through"
menu_items = package.context_menu(text)
if menu_items:
menu = QtGui.QMenu(self)
for text, callback in menu_items:
act = QtGui.QAction(text, self)
act.setStatusTip(text)
QtCore.QObject.connect(
act, QtCore.SIGNAL("triggered()"), callback)
menu.addAction(act)
menu.exec_(event.globalPos())
return
except Exception, e:
debug.unexpected_exception(e)
debug.warning("Got exception trying to display %s's "
"context menu in the palette: %s\n%s" %
(package.name, debug.format_exception(e),
traceback.format_exc()))
item.contextMenuEvent(event, self)
def _open_ssh(self, username, password, hostname, portnum, path):
paramiko = py_import('paramiko', {
'pip': 'paramiko',
'linux-debian': 'python-paramiko',
'linux-ubuntu': 'python-paramiko',
'linux-fedora': 'python-paramiko'})
scp = py_import('scp', {
'pip': 'scp'})
local_filename = os.path.join(package_directory,
cache_filename(self.url))
ssh = paramiko.SSHClient()
ssh.load_system_host_keys()
try:
ssh.connect(hostname, port=portnum,
username=username, password=password)
except paramiko.SSHException, e:
raise ModuleError(self.module, debug.format_exception(e))
def call_it(function, p):
# Translate between VisTrails objects and VTK objects
if p is None:
# None indicates a call with no parameters
params = []
elif isinstance(p, tuple):
# A tuple indicates a call with many parameters
params = list(p)
else:
# Otherwise, it's a single parameter
params = [p]
# Unwraps VTK objects
for i in xrange(len(params)):
if hasattr(params[i], 'vtkInstance'):
params[i] = params[i].vtkInstance
try:
self.call_input_function(function, params)
except Exception, e:
raise ModuleError(
self, "VTK Exception: %s" % debug.format_exception(e))
def contextMenuEvent(self, event):
# Just dispatches the menu event to the widget item
item = self.itemAt(event.pos())
if item:
# find top level
p = item
while p.parent():
p = p.parent()
# get package identifier
assert isinstance(p, QPackageTreeWidgetItem)
identifier = p.identifier
registry = get_module_registry()
package = registry.packages[identifier]
try:
if package.has_contextMenuName():
name = package.contextMenuName(item.text(0))
if name:
act = QtGui.QAction(name, self)
act.setStatusTip(name)
def callMenu():
if package.has_callContextMenu():
name = package.callContextMenu(item.text(0))
QtCore.QObject.connect(act,
QtCore.SIGNAL("triggered()"),
callMenu)
menu = QtGui.QMenu(self)
menu.addAction(act)
menu.exec_(event.globalPos())
return
except Exception, e:
debug.unexpected_exception(e)
debug.warning("Got exception trying to display %s's "
"context menu in the palette: %s\n%s" % (
package.name,
debug.format_exception(e),
traceback.format_exc()))
item.contextMenuEvent(event, self)
def call_it(function, p):
# Translate between VisTrails objects and VTK objects
if p is None:
# None indicates a call with no parameters
params = []
elif isinstance(p, tuple):
# A tuple indicates a call with many parameters
params = list(p)
else:
# Otherwise, it's a single parameter
params = [p]
# Unwraps VTK objects
for i in xrange(len(params)):
if hasattr(params[i], 'vtkInstance'):
params[i] = params[i].vtkInstance
try:
self.call_input_function(function, params)
except Exception, e:
raise ModuleError(
self,
"VTK Exception: %s" % debug.format_exception(e))
def execute(self):
""" Tries to download a file from url.
Returns the path to the local file.
"""
self.local_filename = os.path.join(package_directory,
cache_filename(self.url))
# Before download
self.pre_download()
# Send request
try:
response = self.send_request()
except urllib2.URLError, e:
if self.is_in_local_cache:
debug.warning("A network error occurred. DownloadFile will "
"use a cached version of the file")
return self.local_filename
else:
raise ModuleError(
self.module,
"Network error: %s" % debug.format_exception(e))
def cleanup(self):
"""cleanup() -> None
Cleans up the file pool, by removing all temporary files and
the directory they existed in. Module developers should never
call this directly.
"""
if not os.path.isdir(self.directory):
# cleanup has already happened
return
try:
for root, dirs, files in os.walk(self.directory, topdown=False):
for name in files:
os.remove(os.path.join(root, name))
for name in dirs:
os.rmdir(os.path.join(root, name))
os.rmdir(self.directory)
except OSError, e:
debug.unexpected_exception(e)
raise VistrailsInternalError("Can't remove %s: %s" %
(self.directory,
debug.format_exception(e)))
def setup_pipeline(self, pipeline, **kwargs):
"""setup_pipeline(controller, pipeline, locator, currentVersion,
view, aliases, **kwargs)
Matches a pipeline with the persistent pipeline and creates
instances of modules that aren't in the cache.
"""
def fetch(name, default):
return kwargs.pop(name, default)
controller = fetch('controller', None)
locator = fetch('locator', None)
current_version = fetch('current_version', None)
view = fetch('view', DummyView())
vistrail_variables = fetch('vistrail_variables', None)
aliases = fetch('aliases', None)
params = fetch('params', None)
extra_info = fetch('extra_info', None)
logger = fetch('logger', DummyLogController)
sinks = fetch('sinks', None)
reason = fetch('reason', None)
actions = fetch('actions', None)
done_summon_hooks = fetch('done_summon_hooks', [])
module_executed_hook = fetch('module_executed_hook', [])
stop_on_error = fetch('stop_on_error', True)
parent_exec = fetch('parent_exec', None)
reg = get_module_registry()
if len(kwargs) > 0:
raise VistrailsInternalError('Wrong parameters passed '
'to setup_pipeline: %s' % kwargs)
def create_null():
"""Creates a Null value"""
getter = reg.get_descriptor_by_name
descriptor = getter(basic_pkg, 'Null')
return descriptor.module()
def create_constant(param, module):
"""Creates a Constant from a parameter spec"""
getter = reg.get_descriptor_by_name
desc = getter(param.identifier, param.type, param.namespace)
constant = desc.module()
constant.id = module.id
# if param.evaluatedStrValue:
# constant.setValue(param.evaluatedStrValue)
if param.strValue != '':
constant.setValue(param.strValue)
else:
constant.setValue( \
constant.translate_to_string(constant.default_value))
return constant
### BEGIN METHOD ###
# if self.debugger:
# self.debugger.update()
to_delete = []
errors = {}
if controller is not None:
# Controller is none for sub_modules
controller.validate(pipeline)
else:
pipeline.validate()
self.resolve_aliases(pipeline, aliases)
if vistrail_variables:
self.resolve_variables(vistrail_variables, pipeline)
self.update_params(pipeline, params)
(tmp_to_persistent_module_map,
conn_map,
module_added_set,
conn_added_set) = self.add_to_persistent_pipeline(pipeline)
# Create the new objects
for i in module_added_set:
persistent_id = tmp_to_persistent_module_map[i]
module = self._persistent_pipeline.modules[persistent_id]
obj = self._objects[persistent_id] = module.summon()
obj.interpreter = self
obj.id = persistent_id
obj.signature = module._signature
# Checking if output should be stored
if module.has_annotation_with_key('annotate_output'):
annotate_output = module.get_annotation_by_key('annotate_output')
#print annotate_output
if annotate_output:
obj.annotate_output = True
for f in module.functions:
connector = None
if len(f.params) == 0:
connector = ModuleConnector(create_null(), 'value',
f.get_spec('output'))
elif len(f.params) == 1:
p = f.params[0]
try:
constant = create_constant(p, module)
connector = ModuleConnector(constant, 'value',
f.get_spec('output'))
except Exception, e:
debug.unexpected_exception(e)
err = ModuleError(
module,
"Uncaught exception creating Constant from "
"%r: %s" % (
p.strValue,
debug.format_exception(e)))
errors[i] = err
to_delete.append(obj.id)
else:
tupleModule = vistrails.core.interpreter.base.InternalTuple()
tupleModule.length = len(f.params)
for (j,p) in enumerate(f.params):
try:
constant = create_constant(p, module)
constant.update()
connector = ModuleConnector(constant, 'value',
f.get_spec('output'))
tupleModule.set_input_port(j, connector)
except Exception, e:
debug.unexpected_exception(e)
err = ModuleError(
module,
"Uncaught exception creating Constant "
"from %r: %s" % (
p.strValue,
debug.format_exception(e)))
errors[i] = err
to_delete.append(obj.id)
connector = ModuleConnector(tupleModule, 'value',
f.get_spec('output'))
if connector:
obj.set_input_port(f.name, connector, is_method=True)
raise ModuleError(
self, "Error parsing configuration env: %s" %
(debug.format_exception(e)))
if 'options' in self.conf and 'env' in self.conf['options']:
try:
for var in self.conf['options']['env'].split(";"):
key, value = var.split('=')
key = key.strip()
value = value.strip()
if key:
env[key] = value
except Exception, e: # pragma: no cover
raise ModuleError(
self, "Error parsing module env: %s" %
(debug.format_exception(e)))
if 'options' in self.conf and 'env_port' in self.conf['options']:
for e in self.force_get_input_list('env'):
try:
for var in e.split(';'):
if not var:
continue
key, value = var.split('=')
key = key.strip()
value = value.strip()
if key:
env[key] = value
except Exception, e: # pragma: no cover
raise ModuleError(
self, "Error parsing env port: %s" %
def compute(self):
""" 1. read inputs
2. call with inputs
3. set outputs
"""
# add all arguments as an unordered list
args = [self.conf['command']]
file_std = 'options' in self.conf and 'std_using_files' in self.conf[
'options']
fail_with_cmd = 'options' in self.conf and 'fail_with_cmd' in self.conf[
'options']
setOutput = [] # (name, File) - set File contents as output for name
open_files = []
stdin = None
kwargs = {}
for type, name, klass, options in self.conf['args']:
type = type.lower()
klass = klass.lower()
if "constant" == type:
flag = 'flag' in options and options['flag']
if flag:
args.append(flag)
if name:
# if flag==name we assume user tried to name a constant
if not name == flag:
args.append('%s%s' % (options.get('prefix', ''), name))
elif "input" == type:
# handle multiple inputs
values = self.force_get_input_list(name)
if values and 'list' == klass:
values = values[0]
klass = options['type'].lower() \
if 'type' in options else 'string'
for value in values:
if 'flag' == klass:
if not value:
continue
if 'flag' in options and options['flag']:
value = options['flag']
else:
# use name as flag
value = name
elif klass in ('file', 'directory', 'path'):
value = value.name
# check for flag and append file name
if not 'flag' == klass and 'flag' in options:
args.append(options['flag'])
value = '%s%s' % (options.get('prefix', ''), value)
args.append(value)
elif "output" == type:
# output must be a filename but we may convert the result to a string
# create new file
file = self.interpreter.filePool.create_file(
suffix=options.get('suffix', DEFAULTFILESUFFIX))
fname = file.name
if 'prefix' in options:
fname = options['prefix'] + fname
if 'flag' in options:
args.append(options['flag'])
args.append(fname)
if "file" == klass:
self.set_output(name, file)
elif "string" == klass:
setOutput.append((name, file))
else:
raise ValueError
elif "inputoutput" == type:
# handle single file that is both input and output
value = self.get_input(name)
# create copy of infile to operate on
outfile = self.interpreter.filePool.create_file(
suffix=options.get('suffix', DEFAULTFILESUFFIX))
try:
shutil.copyfile(value.name, outfile.name)
except IOError, e: # pragma: no cover
raise ModuleError(
self, "Error copying file '%s': %s" %
(value.name, debug.format_exception(e)))
value = '%s%s' % (options.get('prefix', ''), outfile.name)
# check for flag and append file name
if 'flag' in options:
args.append(options['flag'])
args.append(value)
self.set_output(name, outfile)
def _add_tool(path):
# first create classes
tool_name = os.path.basename(path)
if isinstance(tool_name, unicode):
tool_name = tool_name.encode('utf-8')
if not tool_name.endswith(SUFFIX): # pragma: no cover
return
(tool_name, _) = os.path.splitext(tool_name)
if tool_name in cl_tools: # pragma: no cover
debug.critical("Package CLTools already added: '%s'" % tool_name)
try:
conf = json.load(open(path))
except ValueError as exc: # pragma: no cover
debug.critical("Package CLTools could not parse '%s'" % path, exc)
return
def compute(self):
""" 1. read inputs
2. call with inputs
3. set outputs
"""
# add all arguments as an unordered list
args = [self.conf['command']]
file_std = 'options' in self.conf and 'std_using_files' in self.conf[
'options']
fail_with_cmd = 'options' in self.conf and 'fail_with_cmd' in self.conf[
'options']
setOutput = [] # (name, File) - set File contents as output for name
open_files = []
stdin = None
kwargs = {}
for type, name, klass, options in self.conf['args']:
type = type.lower()
klass = klass.lower()
if "constant" == type:
flag = 'flag' in options and options['flag']
if flag:
args.append(flag)
if name:
# if flag==name we assume user tried to name a constant
if not name == flag:
args.append('%s%s' % (options.get('prefix', ''), name))
elif "input" == type:
# handle multiple inputs
values = self.force_get_input_list(name)
if values and 'list' == klass:
values = values[0]
klass = options['type'].lower() \
if 'type' in options else 'string'
for value in values:
if 'flag' == klass:
if not value:
continue
if 'flag' in options and options['flag']:
value = options['flag']
else:
# use name as flag
value = name
elif klass in ('file', 'directory', 'path'):
value = value.name
# check for flag and append file name
if not 'flag' == klass and 'flag' in options:
args.append(options['flag'])
value = '%s%s' % (options.get('prefix', ''), value)
args.append(value)
elif "output" == type:
# output must be a filename but we may convert the result to a string
# create new file
file = self.interpreter.filePool.create_file(
suffix=options.get('suffix', DEFAULTFILESUFFIX))
fname = file.name
if 'prefix' in options:
fname = options['prefix'] + fname
if 'flag' in options:
args.append(options['flag'])
args.append(fname)
if "file" == klass:
self.set_output(name, file)
elif "string" == klass:
setOutput.append((name, file))
else:
raise ValueError
elif "inputoutput" == type:
# handle single file that is both input and output
value = self.get_input(name)
# create copy of infile to operate on
outfile = self.interpreter.filePool.create_file(
suffix=options.get('suffix', DEFAULTFILESUFFIX))
try:
shutil.copyfile(value.name, outfile.name)
except IOError, e: # pragma: no cover
raise ModuleError(
self, "Error copying file '%s': %s" %
(value.name, debug.format_exception(e)))
value = '%s%s' % (options.get('prefix', ''), outfile.name)
# check for flag and append file name
if 'flag' in options:
args.append(options['flag'])
args.append(value)
self.set_output(name, outfile)
if "stdin" in self.conf:
name, type, options = self.conf["stdin"]
type = type.lower()
if self.has_input(name):
value = self.get_input(name)
if "file" == type:
if file_std:
f = open(value.name, 'rb')
else:
f = open(value.name, 'rb')
stdin = f.read()
f.close()
elif "string" == type:
if file_std:
file = self.interpreter.filePool.create_file()
f = open(file.name, 'wb')
f.write(value)
f.close()
f = open(file.name, 'rb')
else:
stdin = value
else: # pragma: no cover
raise ValueError
if file_std:
open_files.append(f)
kwargs['stdin'] = f.fileno()
else:
kwargs['stdin'] = subprocess.PIPE
if "stdout" in self.conf:
if file_std:
name, type, options = self.conf["stdout"]
type = type.lower()
file = self.interpreter.filePool.create_file(
suffix=DEFAULTFILESUFFIX)
if "file" == type:
self.set_output(name, file)
elif "string" == type:
setOutput.append((name, file))
else: # pragma: no cover
raise ValueError
f = open(file.name, 'wb')
open_files.append(f)
kwargs['stdout'] = f.fileno()
else:
kwargs['stdout'] = subprocess.PIPE
if "stderr" in self.conf:
if file_std:
name, type, options = self.conf["stderr"]
type = type.lower()
file = self.interpreter.filePool.create_file(
suffix=DEFAULTFILESUFFIX)
if "file" == type:
self.set_output(name, file)
elif "string" == type:
setOutput.append((name, file))
else: # pragma: no cover
raise ValueError
f = open(file.name, 'wb')
open_files.append(f)
kwargs['stderr'] = f.fileno()
else:
kwargs['stderr'] = subprocess.PIPE
if fail_with_cmd:
return_code = 0
else:
return_code = self.conf.get('return_code', None)
env = {}
# 0. add defaults
# 1. add from configuration
# 2. add from module env
# 3. add from env port
if configuration.check('env'):
try:
for var in configuration.env.split(";"):
key, value = var.split('=')
key = key.strip()
value = value.strip()
if key:
env[key] = value
except Exception, e: # pragma: no cover
raise ModuleError(
self, "Error parsing configuration env: %s" %
(debug.format_exception(e)))
def setup_pipeline(self, pipeline, **kwargs):
"""setup_pipeline(controller, pipeline, locator, currentVersion,
view, aliases, **kwargs)
Matches a pipeline with the persistent pipeline and creates
instances of modules that aren't in the cache.
"""
def fetch(name, default):
return kwargs.pop(name, default)
controller = fetch('controller', None)
locator = fetch('locator', None)
current_version = fetch('current_version', None)
view = fetch('view', DummyView())
vistrail_variables = fetch('vistrail_variables', None)
aliases = fetch('aliases', None)
params = fetch('params', None)
extra_info = fetch('extra_info', None)
logger = fetch('logger', DummyLogController)
sinks = fetch('sinks', None)
reason = fetch('reason', None)
actions = fetch('actions', None)
done_summon_hooks = fetch('done_summon_hooks', [])
module_executed_hook = fetch('module_executed_hook', [])
stop_on_error = fetch('stop_on_error', True)
parent_exec = fetch('parent_exec', None)
job_monitor = fetch('job_monitor', None)
reg = get_module_registry()
if len(kwargs) > 0:
raise VistrailsInternalError('Wrong parameters passed '
'to setup_pipeline: %s' % kwargs)
def create_null():
"""Creates a Null value"""
getter = reg.get_descriptor_by_name
descriptor = getter(basic_pkg, 'Null')
return descriptor.module()
def create_constant(param, module):
"""Creates a Constant from a parameter spec"""
getter = reg.get_descriptor_by_name
desc = getter(param.identifier, param.type, param.namespace)
constant = desc.module()
constant.id = module.id
# if param.evaluatedStrValue:
# constant.setValue(param.evaluatedStrValue)
if param.strValue != '':
constant.setValue(param.strValue)
else:
constant.setValue(
constant.translate_to_string(constant.default_value))
return constant
### BEGIN METHOD ###
# if self.debugger:
# self.debugger.update()
to_delete = []
errors = {}
if controller is not None:
# Controller is none for sub_modules
controller.validate(pipeline)
else:
pipeline.validate()
self.resolve_aliases(pipeline, aliases)
if vistrail_variables:
self.resolve_variables(vistrail_variables, pipeline)
self.update_params(pipeline, params)
(tmp_to_persistent_module_map, conn_map, module_added_set,
conn_added_set) = self.add_to_persistent_pipeline(pipeline)
# Create the new objects
for i in module_added_set:
persistent_id = tmp_to_persistent_module_map[i]
module = self._persistent_pipeline.modules[persistent_id]
obj = self._objects[persistent_id] = module.summon()
obj.interpreter = self
obj.id = persistent_id
obj.signature = module._signature
# Checking if output should be stored
if module.has_annotation_with_key('annotate_output'):
annotate_output = module.get_annotation_by_key(
'annotate_output')
#print annotate_output
if annotate_output:
obj.annotate_output = True
for f in module.functions:
connector = None
if len(f.params) == 0:
connector = ModuleConnector(create_null(), 'value',
f.get_spec('output'))
elif len(f.params) == 1:
p = f.params[0]
try:
constant = create_constant(p, module)
connector = ModuleConnector(constant, 'value',
f.get_spec('output'))
except Exception, e:
debug.unexpected_exception(e)
err = ModuleError(
module,
"Uncaught exception creating Constant from "
"%r: %s" % (p.strValue, debug.format_exception(e)))
errors[i] = err
to_delete.append(obj.id)
else:
tupleModule = vistrails.core.interpreter.base.InternalTuple(
)
tupleModule.length = len(f.params)
for (j, p) in enumerate(f.params):
try:
constant = create_constant(p, module)
constant.update()
connector = ModuleConnector(
constant, 'value', f.get_spec('output'))
tupleModule.set_input_port(j, connector)
except Exception, e:
debug.unexpected_exception(e)
err = ModuleError(
module, "Uncaught exception creating Constant "
"from %r: %s" %
(p.strValue, debug.format_exception(e)))
errors[i] = err
to_delete.append(obj.id)
connector = ModuleConnector(tupleModule, 'value',
f.get_spec('output'))
if connector:
obj.set_input_port(f.name, connector, is_method=True)
raise ModuleError(self,
"Error parsing configuration env: %s" % (
debug.format_exception(e)))
if 'options' in self.conf and 'env' in self.conf['options']:
try:
for var in self.conf['options']['env'].split(";"):
key, value = var.split('=')
key = key.strip()
value = value.strip()
if key:
env[key] = value
except Exception, e: # pragma: no cover
raise ModuleError(self,
"Error parsing module env: %s" % (
debug.format_exception(e)))
if 'options' in self.conf and 'env_port' in self.conf['options']:
for e in self.force_get_input_list('env'):
try:
for var in e.split(';'):
if not var:
continue
key, value = var.split('=')
key = key.strip()
value = value.strip()
if key:
env[key] = value
except Exception, e: # pragma: no cover
raise ModuleError(self,
"Error parsing env port: %s" % (
def compute(self):
""" 1. read inputs
2. call with inputs
3. set outputs
"""
# add all arguments as an unordered list
args = [self.conf['command']]
file_std = 'options' in self.conf and 'std_using_files' in self.conf['options']
fail_with_cmd = 'options' in self.conf and 'fail_with_cmd' in self.conf['options']
setOutput = [] # (name, File) - set File contents as output for name
open_files = []
stdin = None
kwargs = {}
for type, name, klass, options in self.conf['args']:
type = type.lower()
klass = klass.lower()
if "constant" == type:
flag = 'flag' in options and options['flag']
if flag:
args.append(flag)
if name:
# if flag==name we assume user tried to name a constant
if not name == flag:
args.append('%s%s' % (options.get('prefix', ''), name))
elif "input" == type:
# handle multiple inputs
values = self.force_get_input_list(name)
if values and 'list' == klass:
values = values[0]
klass = options['type'].lower() \
if 'type' in options else 'string'
for value in values:
if 'flag' == klass:
if not value:
continue
if 'flag' in options and options['flag']:
value = options['flag']
else:
# use name as flag
value = name
elif klass in ('file', 'directory', 'path'):
value = value.name
# check for flag and append file name
if not 'flag' == klass and 'flag' in options:
args.append(options['flag'])
value = '%s%s' % (options.get('prefix', ''),
value)
args.append(value)
elif "output" == type:
# output must be a filename but we may convert the result to a string
# create new file
file = self.interpreter.filePool.create_file(
suffix=options.get('suffix', DEFAULTFILESUFFIX))
fname = file.name
if 'prefix' in options:
fname = options['prefix'] + fname
if 'flag' in options:
args.append(options['flag'])
args.append(fname)
if "file" == klass:
self.set_output(name, file)
elif "string" == klass:
setOutput.append((name, file))
else:
raise ValueError
elif "inputoutput" == type:
# handle single file that is both input and output
value = self.get_input(name)
# create copy of infile to operate on
outfile = self.interpreter.filePool.create_file(
suffix=options.get('suffix', DEFAULTFILESUFFIX))
try:
shutil.copyfile(value.name, outfile.name)
except IOError, e: # pragma: no cover
raise ModuleError(self,
"Error copying file '%s': %s" %
(value.name, debug.format_exception(e)))
value = '%s%s' % (options.get('prefix', ''), outfile.name)
# check for flag and append file name
if 'flag' in options:
args.append(options['flag'])
args.append(value)
self.set_output(name, outfile)
def _add_tool(path):
# first create classes
tool_name = os.path.basename(path)
if isinstance(tool_name, unicode):
tool_name = tool_name.encode('utf-8')
if not tool_name.endswith(SUFFIX): # pragma: no cover
return
(tool_name, _) = os.path.splitext(tool_name)
if tool_name in cl_tools: # pragma: no cover
debug.critical("Package CLTools already added: '%s'" % tool_name)
try:
conf = json.load(open(path))
except ValueError as exc: # pragma: no cover
debug.critical("Package CLTools could not parse '%s'" % path, exc)
return
def compute(self):
""" 1. read inputs
2. call with inputs
3. set outputs
"""
# add all arguments as an unordered list
args = [self.conf['command']]
file_std = 'options' in self.conf and 'std_using_files' in self.conf['options']
fail_with_cmd = 'options' in self.conf and 'fail_with_cmd' in self.conf['options']
setOutput = [] # (name, File) - set File contents as output for name
open_files = []
stdin = None
kwargs = {}
for type, name, klass, options in self.conf['args']:
type = type.lower()
klass = klass.lower()
if "constant" == type:
flag = 'flag' in options and options['flag']
if flag:
args.append(flag)
if name:
# if flag==name we assume user tried to name a constant
if not name == flag:
args.append('%s%s' % (options.get('prefix', ''), name))
elif "input" == type:
# handle multiple inputs
values = self.force_get_input_list(name)
if values and 'list' == klass:
values = values[0]
klass = options['type'].lower() \
if 'type' in options else 'string'
for value in values:
if 'flag' == klass:
if not value:
continue
if 'flag' in options and options['flag']:
value = options['flag']
else:
# use name as flag
value = name
elif klass in ('file', 'directory', 'path'):
value = value.name
# check for flag and append file name
if not 'flag' == klass and 'flag' in options:
args.append(options['flag'])
value = '%s%s' % (options.get('prefix', ''),
value)
args.append(value)
elif "output" == type:
# output must be a filename but we may convert the result to a string
# create new file
file = self.interpreter.filePool.create_file(
suffix=options.get('suffix', DEFAULTFILESUFFIX))
fname = file.name
if 'prefix' in options:
fname = options['prefix'] + fname
if 'flag' in options:
args.append(options['flag'])
args.append(fname)
if "file" == klass:
self.set_output(name, file)
elif "string" == klass:
setOutput.append((name, file))
else:
raise ValueError
elif "inputoutput" == type:
# handle single file that is both input and output
value = self.get_input(name)
# create copy of infile to operate on
outfile = self.interpreter.filePool.create_file(
suffix=options.get('suffix', DEFAULTFILESUFFIX))
try:
shutil.copyfile(value.name, outfile.name)
except IOError, e: # pragma: no cover
raise ModuleError(self,
"Error copying file '%s': %s" %
(value.name, debug.format_exception(e)))
value = '%s%s' % (options.get('prefix', ''), outfile.name)
# check for flag and append file name
if 'flag' in options:
args.append(options['flag'])
args.append(value)
self.set_output(name, outfile)
if "stdin" in self.conf:
name, type, options = self.conf["stdin"]
type = type.lower()
if self.has_input(name):
value = self.get_input(name)
if "file" == type:
if file_std:
f = open(value.name, 'rb')
else:
f = open(value.name, 'rb')
stdin = f.read()
f.close()
elif "string" == type:
if file_std:
file = self.interpreter.filePool.create_file()
f = open(file.name, 'wb')
f.write(value)
f.close()
f = open(file.name, 'rb')
else:
stdin = value
else: # pragma: no cover
raise ValueError
if file_std:
open_files.append(f)
kwargs['stdin'] = f.fileno()
else:
kwargs['stdin'] = subprocess.PIPE
if "stdout" in self.conf:
if file_std:
name, type, options = self.conf["stdout"]
type = type.lower()
file = self.interpreter.filePool.create_file(
suffix=DEFAULTFILESUFFIX)
if "file" == type:
self.set_output(name, file)
elif "string" == type:
setOutput.append((name, file))
else: # pragma: no cover
raise ValueError
f = open(file.name, 'wb')
open_files.append(f)
kwargs['stdout'] = f.fileno()
else:
kwargs['stdout'] = subprocess.PIPE
if "stderr" in self.conf:
if file_std:
name, type, options = self.conf["stderr"]
type = type.lower()
file = self.interpreter.filePool.create_file(
suffix=DEFAULTFILESUFFIX)
if "file" == type:
self.set_output(name, file)
elif "string" == type:
setOutput.append((name, file))
else: # pragma: no cover
raise ValueError
f = open(file.name, 'wb')
open_files.append(f)
kwargs['stderr'] = f.fileno()
else:
kwargs['stderr'] = subprocess.PIPE
if fail_with_cmd:
return_code = 0
else:
return_code = self.conf.get('return_code', None)
env = {}
# 0. add defaults
# 1. add from configuration
# 2. add from module env
# 3. add from env port
if configuration.check('env'):
try:
for var in configuration.env.split(";"):
key, value = var.split('=')
key = key.strip()
value = value.strip()
if key:
env[key] = value
except Exception, e: # pragma: no cover
raise ModuleError(self,
"Error parsing configuration env: %s" % (
debug.format_exception(e)))
def updateFunctionPort(self):
"""
Function to be used inside the updateUsptream method of the Map module. It
updates the module connected to the FunctionPort port, executing it in
parallel.
"""
nameInput = self.get_input('InputPort')
nameOutput = self.get_input('OutputPort')
rawInputList = self.get_input('InputList')
# Create inputList to always have iterable elements
# to simplify code
if len(nameInput) == 1:
element_is_iter = False
inputList = [[element] for element in rawInputList]
else:
element_is_iter = True
inputList = rawInputList
workflows = []
module = None
vtType = None
# iterating through the connectors
for connector in self.inputPorts.get('FunctionPort'):
module = connector.obj
# pipeline
original_pipeline = connector.obj.moduleInfo['pipeline']
# module
module_id = connector.obj.moduleInfo['moduleId']
vtType = original_pipeline.modules[module_id].vtType
# serialize the module for each value in the list
for i, element in enumerate(inputList):
if element_is_iter:
self.element = element
else:
self.element = element[0]
# checking type and setting input in the module
self.typeChecking(connector.obj, nameInput, inputList)
self.setInputValues(connector.obj, nameInput, element, i)
pipeline_db_module = original_pipeline.modules[
module_id].do_copy()
# transforming a subworkflow in a group
# TODO: should we also transform inner subworkflows?
if pipeline_db_module.is_abstraction():
group = Group(id=pipeline_db_module.id,
cache=pipeline_db_module.cache,
location=pipeline_db_module.location,
functions=pipeline_db_module.functions,
annotations=pipeline_db_module.annotations)
source_port_specs = pipeline_db_module.sourcePorts()
dest_port_specs = pipeline_db_module.destinationPorts()
for source_port_spec in source_port_specs:
group.add_port_spec(source_port_spec)
for dest_port_spec in dest_port_specs:
group.add_port_spec(dest_port_spec)
group.pipeline = pipeline_db_module.pipeline
pipeline_db_module = group
# getting highest id between functions to guarantee unique ids
# TODO: can get current IdScope here?
if pipeline_db_module.functions:
high_id = max(function.db_id
for function in pipeline_db_module.functions)
else:
high_id = 0
# adding function and parameter to module in pipeline
# TODO: 'pos' should not be always 0 here
id_scope = IdScope(beginId=long(high_id + 1))
for elementValue, inputPort in izip(element, nameInput):
p_spec = pipeline_db_module.get_port_spec(
inputPort, 'input')
descrs = p_spec.descriptors()
if len(descrs) != 1:
raise ModuleError(
self, "Tuple input ports are not supported")
if not issubclass(descrs[0].module, Constant):
raise ModuleError(
self, "Module inputs should be Constant types")
type = p_spec.sigstring[1:-1]
mod_function = ModuleFunction(id=id_scope.getNewId(
ModuleFunction.vtType),
pos=0,
name=inputPort)
mod_param = ModuleParam(id=0L,
pos=0,
type=type,
val=elementValue)
mod_function.add_parameter(mod_param)
pipeline_db_module.add_function(mod_function)
# serializing module
wf = self.serialize_module(pipeline_db_module)
workflows.append(wf)
# getting first connector, ignoring the rest
break
# IPython stuff
try:
rc = get_client()
except Exception, error:
raise ModuleError(
self, "Exception while loading IPython: %s" %
debug.format_exception(error))
def updateFunctionPort(self):
"""
Function to be used inside the updateUsptream method of the Map module. It
updates the module connected to the FunctionPort port, executing it in
parallel.
"""
nameInput = self.get_input('InputPort')
nameOutput = self.get_input('OutputPort')
rawInputList = self.get_input('InputList')
# Create inputList to always have iterable elements
# to simplify code
if len(nameInput) == 1:
element_is_iter = False
inputList = [[element] for element in rawInputList]
else:
element_is_iter = True
inputList = rawInputList
workflows = []
module = None
vtType = None
# iterating through the connectors
for connector in self.inputPorts.get('FunctionPort'):
module = connector.obj
# pipeline
original_pipeline = connector.obj.moduleInfo['pipeline']
# module
module_id = connector.obj.moduleInfo['moduleId']
vtType = original_pipeline.modules[module_id].vtType
# serialize the module for each value in the list
for i, element in enumerate(inputList):
if element_is_iter:
self.element = element
else:
self.element = element[0]
# checking type and setting input in the module
self.typeChecking(connector.obj, nameInput, inputList)
self.setInputValues(connector.obj, nameInput, element, i)
pipeline_db_module = original_pipeline.modules[module_id].do_copy()
# transforming a subworkflow in a group
# TODO: should we also transform inner subworkflows?
if pipeline_db_module.is_abstraction():
group = Group(id=pipeline_db_module.id,
cache=pipeline_db_module.cache,
location=pipeline_db_module.location,
functions=pipeline_db_module.functions,
annotations=pipeline_db_module.annotations)
source_port_specs = pipeline_db_module.sourcePorts()
dest_port_specs = pipeline_db_module.destinationPorts()
for source_port_spec in source_port_specs:
group.add_port_spec(source_port_spec)
for dest_port_spec in dest_port_specs:
group.add_port_spec(dest_port_spec)
group.pipeline = pipeline_db_module.pipeline
pipeline_db_module = group
# getting highest id between functions to guarantee unique ids
# TODO: can get current IdScope here?
if pipeline_db_module.functions:
high_id = max(function.db_id
for function in pipeline_db_module.functions)
else:
high_id = 0
# adding function and parameter to module in pipeline
# TODO: 'pos' should not be always 0 here
id_scope = IdScope(beginId=long(high_id+1))
for elementValue, inputPort in izip(element, nameInput):
p_spec = pipeline_db_module.get_port_spec(inputPort, 'input')
descrs = p_spec.descriptors()
if len(descrs) != 1:
raise ModuleError(
self,
"Tuple input ports are not supported")
if not issubclass(descrs[0].module, Constant):
raise ModuleError(
self,
"Module inputs should be Constant types")
type = p_spec.sigstring[1:-1]
mod_function = ModuleFunction(id=id_scope.getNewId(ModuleFunction.vtType),
pos=0,
name=inputPort)
mod_param = ModuleParam(id=0L,
pos=0,
type=type,
val=elementValue)
mod_function.add_parameter(mod_param)
pipeline_db_module.add_function(mod_function)
# serializing module
wf = self.serialize_module(pipeline_db_module)
workflows.append(wf)
# getting first connector, ignoring the rest
break
# IPython stuff
try:
rc = get_client()
except Exception, error:
raise ModuleError(self, "Exception while loading IPython: %s" %
debug.format_exception(error))
print filename
locator = vistrails.core.db.locator.FileLocator(
os.path.abspath(filename))
(v, abstractions, thumbnails,
mashups) = vistrails.core.db.io.load_vistrail(locator)
w_list = []
for version, tag in v.get_tagMap().iteritems():
if tag not in VT_EXAMPLES[vtfile]:
w_list.append((locator, version))
nworkflows += 1
if len(w_list) > 0:
errs = vistrails.core.console_mode.run(w_list,
update_vistrail=False)
summary[vtfile] = errs
except Exception, e:
errs.append((vtfile, "None", "None", debug.format_exception(e)))
summary[vtfile] = errs
nvtfiles += 1
print "-" * 79
print "Summary of Examples: %s workflows in %s vistrail files" % (
nworkflows, nvtfiles)
print ""
errors = False
for vtfile, errs in summary.iteritems():
print vtfile
if len(errs) > 0:
for err in errs:
print(" *** Error in %s:%s:%s -- %s" % err)
errors = True
else:
errs = []
filename = os.path.join(EXAMPLES_PATH,
vtfile)
print filename
locator = vistrails.core.db.locator.FileLocator(os.path.abspath(filename))
(v, abstractions, thumbnails, mashups) = vistrails.core.db.io.load_vistrail(locator)
w_list = []
for version,tag in v.get_tagMap().iteritems():
if tag not in VT_EXAMPLES[vtfile]:
w_list.append((locator,version))
nworkflows += 1
if len(w_list) > 0:
errs = vistrails.core.console_mode.run(w_list, update_vistrail=False)
summary[vtfile] = errs
except Exception, e:
errs.append((vtfile,"None", "None", debug.format_exception(e)))
summary[vtfile] = errs
nvtfiles += 1
print "-" * 79
print "Summary of Examples: %s workflows in %s vistrail files" % (
nworkflows, nvtfiles)
print ""
errors = False
for vtfile, errs in summary.iteritems():
print vtfile
if len(errs) > 0:
for err in errs:
print(" *** Error in %s:%s:%s -- %s" % err)
errors = True
else:
