def install_default_startupxml_if_needed():
fname = os.path.join(self.temp_configuration.dotVistrails,
'startup.xml')
root_dir = system.vistrails_root_directory()
origin = os.path.join(root_dir, 'core','resources',
'default_vistrails_startup_xml')
def skip():
if os.path.isfile(fname):
try:
d = self.startup_dom()
v = str(d.getElementsByTagName('startup')[0].attributes['version'].value)
return LooseVersion('0.1') <= LooseVersion(v)
except Exception:
return False
else:
return False
if skip():
return
try:
shutil.copyfile(origin, fname)
debug.log('Succeeded!')
except Exception, e:
debug.critical("""Failed to copy default configuration
file to %s. This could be an indication of a
permissions problem. Please make sure '%s' is writable."""
% (fname,
self.temp_configuration.dotVistrails), e)
class JobMixin(NotCacheable):
""" Mixin for suspendable modules.
This provides the base behavior for modules that submit jobs by handling
the serialization & JobMonitor interaction for you.
The module developer needs only implement the following methods:
job_read_inputs()
job_set_results()
job_start()
job_get_handle()
job_finish()
"""
def compute(self):
""" Base behavior for job-submitting modules.
This provides the base code and calls the methods that the module
developer should provide.
"""
debug.log("%s compute() starting\n"
"signature = %r" % (self.__class__.__name__, self.signature))
jm = self.job_monitor()
cache = jm.getCache(self.signature)
if cache is not None:
# Result is available from cache
jm.setCache(self.signature, cache.parameters)
debug.log("Cached results found; calling job_set_results()")
self.job_set_results(cache.parameters)
return
else:
debug.log("Cache miss")
job = jm.getJob(self.signature)
if job is None:
debug.log("Job doesn't exist")
params = self.job_read_inputs()
params = self.job_start(params)
else:
debug.log("Got job from JobMonitor")
params = job.parameters
jm.addJob(self.signature, params, self.job_name())
# Might raise ModuleSuspended
debug.log("Calling checkJob()")
try:
jm.checkJob(self, self.signature, self.job_get_handle(params))
except ModuleSuspended, e:
debug.log("checkJob() raised ModuleSuspended, job handle is %r" %
e.handle)
raise
# Didn't raise: job is finished
debug.log("Calling job_finish()")
params = self.job_finish(params)
debug.log("Filling cache")
jm.setCache(self.signature, params)
debug.log("Calling job_set_results()")
self.job_set_results(params)
def get_connection(self):
if self._conn_id is not None \
and DBLocator.connections.has_key(self._conn_id):
connection = DBLocator.connections[self._conn_id]
if io.ping_db_connection(connection):
return connection
else:
if self._conn_id is None:
if DBLocator.cache_connections.has_key(self._hash):
connection = DBLocator.cache_connections[self._hash]
if io.ping_db_connection(connection):
debug.log("Reusing cached connection")
return connection
if len(DBLocator.connections.keys()) == 0:
self._conn_id = 1
else:
self._conn_id = max(DBLocator.connections.keys()) + 1
config = {'host': self._host,
'port': self._port,
'db': self._db,
'user': self._user,
'passwd': self._passwd}
#print "config:", config
connection = io.open_db_connection(config)
DBLocator.connections[self._conn_id] = connection
DBLocator.cache_connections[self._hash] = connection
return connection
def reload_scripts(initial=False, name=None):
reg = vistrails.core.modules.module_registry.get_module_registry()
if not initial:
from vistrails.core.interpreter.cached import CachedInterpreter
CachedInterpreter.clear_package(identifiers.identifier)
if name is None:
remove_all_scripts()
else:
del cl_tools[name]
reg.delete_module(identifiers.identifier, name)
if "CLTools" == identifiers.name:
# this is the original package
location = os.path.join(vistrails.core.system.current_dot_vistrails(),
"CLTools")
# make sure dir exist
if not os.path.isdir(location): # pragma: no cover # pragma: no branch
try:
debug.log("Creating CLTools directory...")
os.mkdir(location)
except Exception, e:
debug.critical(
"Could not create CLTools directory. Make "
"sure '%s' does not exist and parent directory "
"is writable" % location, e)
sys.exit(1)
def register_operations(reg, pkg, namespace, exclude=set()):
modules = set()
for name in dir(pkg):
if name in exclude:
continue
op = getattr(pkg, name)
if isinstance(op, types.ModuleType) or name.startswith('_'):
continue
if not callable(op):
continue
if op.__doc__ is None:
debug.log("Object has no __doc__: %r" % op)
continue
args = guess_args(op, op.__doc__, op_name=name)
input_ports = [(arg, type_)
for arg, descr, type_ in args]
reg.add_module(type(name, (AutoOperation,),
{'args': args, 'op': (op,),
'_input_ports': input_ports,
'__doc__': op.__doc__}),
namespace=namespace)
modules.add(name)
return modules
def create_startupxml_if_needed(self):
needs_create = True
fname = self.get_startup_xml_fname()
if os.path.isfile(fname):
try:
tree = ElementTree.parse(fname)
startup_version = \
vistrails.db.services.io.get_version_for_xml(tree.getroot())
version_list = version_string_to_list(startup_version)
if version_list >= [0,1]:
needs_create = False
except:
debug.warning("Unable to read startup.xml file, "
"creating a new one")
if needs_create:
root_dir = system.vistrails_root_directory()
origin = os.path.join(root_dir, 'core','resources',
'default_vistrails_startup_xml')
try:
shutil.copyfile(origin, fname)
debug.log('Succeeded!')
self.first_run = True
except:
debug.critical("""Failed to copy default configuration
file to %s. This could be an indication of a
permissions problem. Please make sure '%s' is writable."""
% (fname, self._dot_vistrails))
raise
def install_package(self, codepath):
debug.log("package found!")
# read manifest
try:
f = open(os.path.join(self._path, codepath, 'MANIFEST'))
except IOError, e:
raise PackageRepository.InvalidPackage("Package is missing manifest.")
def install_package(self, codepath):
debug.log("package found!")
# read manifest
try:
f = urllib2.urlopen(self._path + '/' + codepath + '/MANIFEST')
except urllib2.HTTPError, e:
raise PackageRepository.InvalidPackage("Package is missing manifest.")
def get_connection(self):
if self._conn_id is not None \
and DBLocator.connections.has_key(self._conn_id):
connection = DBLocator.connections[self._conn_id]
if io.ping_db_connection(connection):
return connection
else:
if self._conn_id is None:
if DBLocator.cache_connections.has_key(self._hash):
connection = DBLocator.cache_connections[self._hash]
if io.ping_db_connection(connection):
debug.log("Reusing cached connection")
return connection
if len(DBLocator.connections.keys()) == 0:
self._conn_id = 1
else:
self._conn_id = max(DBLocator.connections.keys()) + 1
config = {'host': self._host,
'port': self._port,
'db': self._db,
'user': self._user,
'passwd': self._passwd}
#print "config:", config
connection = io.open_db_connection(config)
DBLocator.connections[self._conn_id] = connection
DBLocator.cache_connections[self._hash] = connection
return connection
def reload_scripts(initial=False, name=None):
reg = vistrails.core.modules.module_registry.get_module_registry()
if not initial:
from vistrails.core.interpreter.cached import CachedInterpreter
CachedInterpreter.clear_package(identifiers.identifier)
if name is None:
remove_all_scripts()
else:
del cl_tools[name]
reg.delete_module(identifiers.identifier, name)
if "CLTools" == identifiers.name:
# this is the original package
location = os.path.join(vistrails.core.system.current_dot_vistrails(),
"CLTools")
# make sure dir exist
if not os.path.isdir(location): # pragma: no cover # pragma: no branch
try:
debug.log("Creating CLTools directory...")
os.mkdir(location)
except Exception, e:
debug.critical("Could not create CLTools directory. Make "
"sure '%s' does not exist and parent directory "
"is writable" % location,
e)
sys.exit(1)
def __init__(self, address):
""" Process WSDL and add all Types and Methods
"""
self.address = address
self.signature = toSignature(self.address)
self.wsdlHash = '-1'
self.modules = []
self.package = None
debug.log("Installing Web Service from WSDL: %s"% address)
options = dict(cachingpolicy=1, cache=package_cache)
proxy_types = ['http']
for t in proxy_types:
key = 'proxy_%s'%t
if configuration.check(key):
proxy = getattr(configuration, key)
debug.log("Using proxy: %s" % proxy)
if len(proxy):
options['proxy'] = {t:proxy}
try:
self.service = suds.client.Client(address, **options)
self.backUpCache()
except Exception, e:
self.service = None
# We may be offline and the cache may have expired,
# try to use backup
if self.restoreFromBackup():
try:
self.service = suds.client.Client(address, **options)
except Exception, e:
self.service = None
debug.critical("Could not load WSDL: %s" % address,
str(e) + '\n' + str(traceback.format_exc()))
def initialize(*args, **keywords):
if "CLTools" == identifiers.name:
# this is the original package
location = os.path.join(vistrails.core.system.current_dot_vistrails(),
"CLTools")
# make sure dir exist
if not os.path.isdir(location):
try:
debug.log("Creating CLTools directory...")
os.mkdir(location)
except:
debug.critical("""Could not create CLTools directory. Make
sure '%s' does not exist and parent directory is writable""" % location)
sys.exit(1)
else:
# this is a standalone package so modules are placed in this directory
location = os.path.dirname(__file__)
reg = vistrails.core.modules.module_registry.get_module_registry()
reg.add_module(CLTools, abstract=True)
for path in os.listdir(location):
if path.endswith(SUFFIX):
try:
add_tool(os.path.join(location, path))
except Exception as exc:
import traceback
debug.critical("Package CLTools failed to create module "
"from '%s': %s" % (os.path.join(location, path), exc),
traceback.format_exc())
def reload_scripts():
remove_all_scripts()
if "CLTools" == identifiers.name:
# this is the original package
location = os.path.join(vistrails.core.system.current_dot_vistrails(),
"CLTools")
# make sure dir exist
if not os.path.isdir(location):
try:
debug.log("Creating CLTools directory...")
os.mkdir(location)
except:
debug.critical("""Could not create CLTools directory. Make
sure '%s' does not exist and parent directory is writable""" % location)
sys.exit(1)
else:
# this is a standalone package so modules are placed in this directory
location = os.path.dirname(__file__)
for path in os.listdir(location):
if path.endswith(SUFFIX):
try:
add_tool(os.path.join(location, path))
except Exception as exc:
import traceback
debug.critical("Package CLTools failed to create module "
"from '%s': %s" % (os.path.join(location, path), exc),
traceback.format_exc())
from vistrails.core.interpreter.cached import CachedInterpreter
CachedInterpreter.clear_package(identifiers.identifier)
from vistrails.gui.vistrails_window import _app
_app.invalidate_pipelines()
def install_package(self, codepath):
debug.log("package found!")
# read manifest
try:
f = open(os.path.join(self._path, codepath, "MANIFEST"))
except IOError, e:
raise InvalidPackage("Package is missing manifest.")
def initialize(*args, **keywords):
""" initialize() -> None
Package-entry to initialize the package
"""
import vistrails.core.application
if not vistrails.core.application.is_running_gui():
raise RuntimeError, "GUI is not running. The Spreadsheet package requires the GUI"
# initialize widgets
debug.log('Loading Spreadsheet widgets...')
global basicWidgets
if basicWidgets == None:
basicWidgets = addWidget(
'vistrails.packages.spreadsheet.basic_widgets')
importWidgetModules(basicWidgets)
# Create application if there is no one available
global app
app = QtCore.QCoreApplication.instance()
if app == None:
app = QtGui.QApplication(sys.argv)
if hasattr(app, 'builderWindow'):
global spreadsheetWindow
spreadsheetWindow = spreadsheetController.findSpreadsheetWindow(
show=False)
def install_package(self, codepath):
debug.log("package found!")
# read manifest
try:
f = urllib2.urlopen(self._path + "/" + codepath + "/MANIFEST")
except urllib2.HTTPError, e:
raise InvalidPackage("Package is missing manifest.")
def create_startupxml_if_needed(self):
needs_create = True
fname = self.get_startup_xml_fname()
if os.path.isfile(fname):
try:
tree = ElementTree.parse(fname)
startup_version = \
vistrails.db.services.io.get_version_for_xml(tree.getroot())
version_list = version_string_to_list(startup_version)
if version_list >= [0, 1]:
needs_create = False
except:
debug.warning("Unable to read startup.xml file, "
"creating a new one")
if needs_create:
root_dir = system.vistrails_root_directory()
origin = os.path.join(root_dir, 'core', 'resources',
'default_vistrails_startup_xml')
try:
shutil.copyfile(origin, fname)
debug.log('Succeeded!')
self.first_run = True
except:
debug.critical("""Failed to copy default configuration
file to %s. This could be an indication of a
permissions problem. Please make sure '%s' is writable.""" %
(fname, self._dot_vistrails))
raise
def guess_args(obj, doc, op_name=None):
args = []
for line in read_args(doc):
if not ':' in line:
debug.log("Malformated argument in doc: %r" % obj)
continue
arg, descr = line.split(':', 1)
descr = descr.strip()
if arg == 'shape':
type_ = '(basic:List)'
elif (op_name is not None and
(op_name == 'assign' or op_name.startswith('assign_')) and
arg == 'ref'):
type_ = Variable
elif (_re_arg_bool.search(descr) is not None):
type_ = '(basic:Boolean)'
elif (_re_arg_int.search(descr) is not None):
type_ = '(basic:Integer)'
elif descr.lower().startswith("A list "):
type_ = '(basic:List)'
elif arg == 'dtype' or arg == 'name':
type_ = '(basic:String)'
else:
type_ = TFOperation
args.append((arg, descr, type_))
if not args:
debug.log("Didn't find 'Args:' in doc; skipping: %r" %
obj)
return args
def emit(self, record):
msg = "tej: %s" % self.format(record)
if record.levelno >= logging.CRITICAL:
debug.critical(msg)
elif record.levelno >= logging.WARNING:
debug.warning(msg)
elif record.levelno >= logging.INFO:
debug.log(msg)
else:
debug.debug(msg)
def temp_mouse_hover_changed(self, on):
""" temp_mouse_hover_changed(on: int) -> None
"""
debug.log("thumbs_temp_mouse_hover_changed")
value = bool(on)
if self._mouse_hover_cb.text() == "No (for this session only)":
value = not bool(on)
self._temp_configuration.thumbs.mouseHover = value
def temp_autosave_changed(self, on):
""" temp_autosave_changed(on: int) -> None
"""
debug.log("thumbs_temp_auto_save_changed")
value = bool(on)
if self._autosave_cb.text() == "No (for this session only)":
value = not bool(on)
self._temp_configuration.thumbs.autoSave = value
def temp_use_cache_changed(self, on):
""" temp_use_cache_changed(on: int) -> None
"""
debug.log("temp_use_cache_changed")
value = bool(on)
if self._use_cache_cb.text() == "No (for this session only)":
value = not bool(on)
self._temp_configuration.useCache = value
def temp_db_connect_changed(self, on):
""" temp_db_connect_changed(on: int) -> None
"""
debug.log("temp_db_connect_changed")
value = bool(on)
if self._db_connect_cb.text() == "No (for this session only)":
value = not bool(on)
self._temp_configuration.dbDefault = value
def emit(self, record):
msg = "tej: %s" % self.format(record)
if record.levelno >= logging.CRITICAL:
debug.critical(msg)
elif record.levelno >= logging.WARNING:
debug.warning(msg)
elif record.levelno >= logging.INFO:
debug.log(msg)
else:
debug.debug(msg)
def initialize(*args, **keywords):
reg = vistrails.core.modules.module_registry.get_module_registry()
basic = vistrails.core.modules.basic_modules
reg.add_module(DownloadFile)
reg.add_input_port(DownloadFile, "url", (basic.String, 'URL'))
reg.add_input_port(DownloadFile, 'insecure',
(basic.Boolean, "Allow invalid SSL certificates"),
optional=True, defaults="['False']")
reg.add_output_port(DownloadFile, "file",
(basic.File, 'local File object'))
reg.add_output_port(DownloadFile, "local_filename",
(basic.String, 'local filename'), optional=True)
reg.add_module(HTTPDirectory)
reg.add_input_port(HTTPDirectory, 'url', (basic.String, "URL"))
reg.add_input_port(HTTPDirectory, 'insecure',
(basic.Boolean, "Allow invalid SSL certificates"),
optional=True, defaults="['False']")
reg.add_output_port(HTTPDirectory, 'directory',
(basic.Directory, "local Directory object"))
reg.add_output_port(HTTPDirectory, 'local_path',
(basic.String, "local path"), optional=True)
reg.add_module(RepoSync)
reg.add_input_port(RepoSync, "file", (basic.File, 'File'))
reg.add_input_port(RepoSync, "checksum",
(basic.String, 'Checksum'), optional=True)
reg.add_output_port(RepoSync, "file", (basic.File,
'Repository Synced File object'))
reg.add_output_port(RepoSync, "checksum",
(basic.String, 'Checksum'), optional=True)
reg.add_module(URLEncode)
reg.add_input_port(URLEncode, "string", basic.String)
reg.add_output_port(URLEncode, "encoded", basic.String)
reg.add_module(URLDecode)
reg.add_input_port(URLDecode, "encoded", basic.String)
reg.add_output_port(URLDecode, "string", basic.String)
global package_directory
dotVistrails = current_dot_vistrails()
package_directory = os.path.join(dotVistrails, "HTTP")
if not os.path.isdir(package_directory):
try:
debug.log("Creating HTTP package directory: %s" % package_directory)
os.mkdir(package_directory)
except Exception, e:
raise RuntimeError("Failed to create cache directory: %s" %
package_directory, e)
def __init__(self, address):
""" Process WSDL and add all Types and Methods
"""
self.address = address
self.signature = toSignature(self.address)
self.wsdlHash = '-1'
self.modules = []
self.package = None
debug.log("Installing Web Service from WSDL: %s"% address)
options = dict(cachingpolicy=1, cache=package_cache)
proxy_types = ['http']
for t in proxy_types:
key = 'proxy_%s'%t
if configuration.check(key):
proxy = getattr(configuration, key)
debug.log("Using proxy: %s" % proxy)
if len(proxy):
options['proxy'] = {t:proxy}
try:
self.service = suds.client.Client(address, **options)
self.backUpCache()
except Exception:
self.service = None
# We may be offline and the cache may have expired,
# try to use backup
if self.restoreFromBackup():
try:
self.service = suds.client.Client(address, **options)
except Exception:
self.service = None
debug.critical("Could not load WSDL: %s" % address,
traceback.format_exc())
else:
debug.critical("Could not load WSDL: %s" % address,
traceback.format_exc())
if self.service:
try:
self.createPackage()
self.setTypes()
self.setMethods()
self.createTypeClasses()
self.createMethodClasses()
except Exception:
debug.critical("Could not create Web Service: %s" % address,
traceback.format_exc())
self.service = None
if self.wsdlHash == '-1':
# create empty package so that it can be reloaded/deleted
self.createFailedPackage()
def __init__(self, address):
""" Process WSDL and add all Types and Methods
"""
self.address = address
self.signature = toSignature(self.address)
self.wsdlHash = '-1'
self.modules = []
self.package = None
debug.log("Installing Web Service from WSDL: %s" % address)
options = dict(cachingpolicy=1, cache=package_cache)
proxy_types = ['http']
for t in proxy_types:
key = 'proxy_%s' % t
if configuration.check(key):
proxy = getattr(configuration, key)
debug.log("Using proxy: %s" % proxy)
if len(proxy):
options['proxy'] = {t: proxy}
try:
self.service = suds.client.Client(address, **options)
self.backUpCache()
except Exception:
self.service = None
# We may be offline and the cache may have expired,
# try to use backup
if self.restoreFromBackup():
try:
self.service = suds.client.Client(address, **options)
except Exception:
self.service = None
debug.critical("Could not load WSDL: %s" % address,
traceback.format_exc())
else:
debug.critical("Could not load WSDL: %s" % address,
traceback.format_exc())
if self.service:
try:
self.createPackage()
self.setTypes()
self.setMethods()
self.createTypeClasses()
self.createMethodClasses()
except Exception:
debug.critical("Could not create Web Service: %s" % address,
traceback.format_exc())
self.service = None
if self.wsdlHash == '-1':
# create empty package so that it can be reloaded/deleted
self.createFailedPackage()
def addTag(self, version_name, version_number):
"""addTag(version_name, version_number) -> None
Adds new tag to vistrail
"""
if version_name == '':
return None
if self.has_tag(version_number):
debug.log("Version is already tagged")
raise VersionAlreadyTagged()
if self.has_tag_str(version_name):
debug.log("Tag already exists")
raise TagExists()
self.set_tag(version_number, version_name)
def addTag(self, version_name, version_number):
"""addTag(version_name, version_number) -> None
Adds new tag to vistrail
"""
if version_name == '':
return None
if self.has_tag(version_number):
debug.log("Version is already tagged")
raise VersionAlreadyTagged()
if self.has_tag_str(version_name):
debug.log("Tag already exists")
raise TagExists()
self.set_tag(version_number, version_name)
def finalize(self):
if not self._initialized:
return
debug.log("Finalizing %s" % self.name)
try:
callable_ = self._module.finalize
except AttributeError:
pass
else:
try:
callable_()
except Exception, e:
debug.critical("Couldn't finalize %s: %s: %s" % (
self.name, type(e).__name__, ', '.join(e.args)))
def create_default_directory(self):
if os.path.lexists(self.temp_configuration.dotVistrails):
return
debug.log('Will try to create default directory')
try:
os.mkdir(self.temp_configuration.dotVistrails)
debug.log('Succeeded!')
except:
debug.critical("""Failed to create initialization directory.
This could be an indication of a permissions problem.
Make sure parent directory of '%s' is writable."""
% self.temp_configuration.dotVistrails)
sys.exit(1)
def initialize(*args, **keywords):
""" initialize() -> None
Package-entry to initialize the package
"""
import vistrails.core.application
if not vistrails.core.application.is_running_gui():
raise RuntimeError, "GUI is not running. The Spreadsheet package requires the GUI"
# initialize widgets
debug.log('Loading Spreadsheet widgets...')
global basicWidgets
if basicWidgets==None:
basicWidgets = addWidget('vistrails.packages.spreadsheet.basic_widgets')
importWidgetModules(basicWidgets)
def get_repository():
global _repository
if _repository:
return _repository
import vistrails.core.configuration
conf = vistrails.core.configuration.get_vistrails_configuration()
if conf.check('repositoryHTTPURL'):
_repository = HTTPPackageRepository(conf.repositoryHTTPURL)
debug.log("Using HTTP Package Repository @ %s" % conf.repositoryHTTPURL)
elif conf.check('repositoryLocalPath'):
_repository = LocalPackageRepository(conf.repositoryLocalPath)
debug.log("Using Local Repository @ %s" % conf.repositoryLocalPath)
else:
_repository = None
return _repository
def get_repository():
global _repository
if _repository:
return _repository
import vistrails.core.configuration
conf = vistrails.core.configuration.get_vistrails_configuration()
if conf.check('repositoryHTTPURL'):
_repository = HTTPPackageRepository(conf.repositoryHTTPURL)
debug.log("Using HTTP Package Repository @ %s" % conf.repositoryHTTPURL)
elif conf.check('repositoryLocalPath'):
_repository = LocalPackageRepository(conf.repositoryLocalPath)
debug.log("Using Local Repository @ %s" % conf.repositoryLocalPath)
else:
_repository = None
return _repository
def finalize(self):
if not self._initialized:
return
debug.log("Finalizing %s" % self.name)
try:
callable_ = self._module.finalize
except AttributeError:
pass
else:
try:
callable_()
except Exception, e:
debug.critical(
"Couldn't finalize %s: %s: %s" %
(self.name, type(e).__name__, ', '.join(e.args)))
def initialize(*args, **keywords):
""" initialize() -> None
Package-entry to initialize the package
"""
import vistrails.core.application
if not vistrails.core.application.is_running_gui():
raise RuntimeError, "GUI is not running. The Spreadsheet package requires the GUI"
# initialize widgets
debug.log('Loading Spreadsheet widgets...')
global basicWidgets
if basicWidgets==None:
basicWidgets = addWidget('vistrails.packages.spreadsheet.basic_widgets')
importWidgetModules(basicWidgets)
def reload_scripts(initial=False):
if not initial:
remove_all_scripts()
if "CLTools" == identifiers.name:
# this is the original package
location = os.path.join(vistrails.core.system.current_dot_vistrails(),
"CLTools")
# make sure dir exist
if not os.path.isdir(location): # pragma: no cover # pragma: no branch
try:
debug.log("Creating CLTools directory...")
os.mkdir(location)
except Exception, e:
debug.critical("""Could not create CLTools directory. Make
sure '%s' does not exist and parent directory is writable""" % location, e)
sys.exit(1)
def install_default_startup():
debug.log('Will try to create default startup script')
try:
root_dir = system.vistrails_root_directory()
default_file = os.path.join(root_dir,'core','resources',
'default_vistrails_startup')
user_file = os.path.join(self.temp_configuration.dotVistrails,
'startup.py')
shutil.copyfile(default_file,user_file)
debug.log('Succeeded!')
except:
debug.critical("""Failed to copy default file %s.
This could be an indication of a permissions problem.
Make sure directory '%s' is writable"""
% (user_file,self.temp_configuration.dotVistrails))
sys.exit(1)
def finalize(self):
if not self._initialized:
return
debug.log("Finalizing %s" % self.name)
try:
callable_ = self._module.finalize
except AttributeError:
pass
else:
try:
callable_()
except Exception, e:
debug.unexpected_exception(e)
debug.critical("Couldn't finalize %s: %s\n%s" % (
self.name, debug.format_exception(e),
traceback.format_exc()))
def changeTag(self, version_name, version_number):
"""changeTag(version_name, version_number) -> None
Changes the old tag of version_number to version_name in the
vistrail. If version_name is empty, this version will be
untagged.
"""
if not self.has_tag(version_number):
debug.log("Version is not tagged")
raise VersionNotTagged()
if self.get_tag(version_number) == version_name:
return None
if self.has_tag_str(version_name):
debug.log("Tag already exists")
raise TagExists()
self.set_tag(version_number, version_name)
def finalize(self):
if not self._initialized:
return
debug.log("Finalizing %s" % self.name)
try:
callable_ = self._module.finalize
except AttributeError:
pass
else:
try:
callable_()
except Exception, e:
debug.unexpected_exception(e)
debug.critical("Couldn't finalize %s: %s\n%s" %
(self.name, debug.format_exception(e),
traceback.format_exc()))
def changeTag(self, version_name, version_number):
"""changeTag(version_name, version_number) -> None
Changes the old tag of version_number to version_name in the
vistrail. If version_name is empty, this version will be
untagged.
"""
if not self.has_tag(version_number):
debug.log("Version is not tagged")
raise VersionNotTagged()
if self.get_tag(version_number) == version_name:
return None
if self.has_tag_str(version_name):
debug.log("Tag already exists")
raise TagExists()
self.set_tag(version_number, version_name)
def data_sync(self):
""" downloads/uploads/uses the local file depending on availability """
self.checksum_lookup()
# local file not on repository, so upload
if not self.on_server and os.path.isfile(self.in_file.name):
cookiejar = None
if is_running_gui():
import vistrails.gui.repository
cookiejar = vistrails.gui.repository.QRepositoryDialog.cookiejar
if cookiejar:
register_openers(cookiejar=cookiejar)
params = {'dataset_file': open(self.in_file.name, 'rb'),
'name': self.in_file.name.split('/')[-1],
'origin': 'vistrails',
'checksum': self.checksum}
upload_url = "%s/datasets/upload/" % self.base_url
datagen, headers = multipart_encode(params)
request = urllib2.Request(upload_url, datagen, headers)
try:
result = urllib2.urlopen(request)
if result.code != 200:
debug.warning("Failed to upload data to the "
"repository")
# make temporarily uncachable
self.is_cacheable = self.invalidate_cache
else:
debug.log("Push to repository was successful")
# make sure module caches
self.is_cacheable = self.validate_cache
except Exception, e:
debug.warning("Failed to upload data to the repository")
# make temporarily uncachable
self.is_cacheable = self.invalidate_cache
debug.warning('RepoSync uploaded %s to the repository' % \
self.in_file.name)
else:
debug.warning("You must be logged into the web repository in "
"order to upload data. No data was synced")
# make temporarily uncachable
self.is_cacheable = self.invalidate_cache
# use local data
self.set_output("file", self.in_file)
def run(self, *args):
"""run(*args), runs ImageMagick's 'convert' on a shell, passing all
arguments to the program."""
path = None
if configuration.check("path"):
path = configuration.path
if path:
cmd = os.path.join(path, "convert")
else:
cmd = "convert"
cmd = [cmd] + list(args)
cmdline = list2cmdline(cmd)
if not configuration.quiet:
debug.log(cmdline)
r = os.system(cmdline)
if r != 0:
raise ModuleError(self, "system call failed: %r" % cmdline)
def initialize(*args, **keywords):
# import core.packagemanager
global webServicesDict
global package_cache
#Create a directory for the SUDSWebServices package
location = os.path.join(vistrails.core.system.current_dot_vistrails(),
"SUDSWebServices")
if not os.path.isdir(location):
try:
debug.log("Creating SUDS cache directory...")
os.mkdir(location)
except OSError, e:
debug.critical(
"""Could not create SUDS cache directory. Make sure
'%s' does not exist and parent directory is writable""" % location, e)
sys.exit(1)
def initialize(*args, **keywords):
# import core.packagemanager
global webServicesDict
global package_cache
#Create a directory for the SUDSWebServices package
location = os.path.join(vistrails.core.system.current_dot_vistrails(),
"SUDSWebServices")
if not os.path.isdir(location):
try:
debug.log("Creating SUDS cache directory...")
os.mkdir(location)
except OSError, e:
debug.critical(
"""Could not create SUDS cache directory. Make sure
'%s' does not exist and parent directory is writable""" % location, e)
sys.exit(1)
def setup_debug(self):
if (self.temp_configuration.has('debugLevel') and
self.temp_configuration.debugLevel != -1):
verbose = self.temp_configuration.debugLevel
if verbose < 0:
msg = ("""Don't know how to set verboseness level to %s - "
"setting to the lowest one I know of: 0""" % verbose)
debug.critical(msg)
verbose = 0
if verbose > 2:
msg = ("""Don't know how to set verboseness level to %s - "
"setting to the highest one I know of: 2""" % verbose)
debug.critical(msg)
verbose = 2
dbg = debug.DebugPrint.getInstance()
levels = [dbg.WARNING, dbg.INFO, dbg.DEBUG]
dbg.set_message_level(levels[verbose])
debug.log("Set verboseness level to %s" % verbose)
def setup_debug(self):
if (self.temp_configuration.has('debugLevel')
and self.temp_configuration.debugLevel != -1):
verbose = self.temp_configuration.debugLevel
if verbose < 0:
msg = ("""Don't know how to set verboseness level to %s - "
"setting to the lowest one I know of: 0""" % verbose)
debug.critical(msg)
verbose = self.temp_configuration.debugLevel = 0
if verbose > 2:
msg = ("""Don't know how to set verboseness level to %s - "
"setting to the highest one I know of: 2""" % verbose)
debug.critical(msg)
verbose = self.temp_configuration.debugLevel = 2
dbg = debug.DebugPrint.getInstance()
levels = [dbg.WARNING, dbg.INFO, dbg.DEBUG]
dbg.set_message_level(levels[verbose])
debug.log("Set verboseness level to %s" % verbose)
def addWidget(packagePath):
""" addWidget(packagePath: str) -> package
Add a new widget type to the spreadsheet registry supplying a
basic set of spreadsheet widgets
"""
try:
registry = get_module_registry()
widget = importReturnLast(packagePath)
if hasattr(widget, 'widgetName'):
widgetName = widget.widgetName()
else:
widgetName = packagePath
widget.registerWidget(registry, basic_modules, basicWidgets)
spreadsheetRegistry.registerPackage(widget, packagePath)
debug.log(' ==> Successfully import <%s>' % widgetName)
except Exception, e:
debug.log(' ==> Ignored package <%s>' % packagePath, e)
widget = None
def run(self, *args):
"""run(*args), runs ImageMagick's 'convert' on a shell, passing all
arguments to the program.
"""
path = None
if configuration.check('path'):
path = configuration.path
if path:
cmd = os.path.join(path, 'convert')
else:
cmd = 'convert'
cmd = [cmd] + list(args)
cmdline = list2cmdline(cmd)
if not configuration.quiet:
debug.log(cmdline)
r = os.system(cmdline)
if r != 0:
raise ModuleError(self, "system call failed: %r" % cmdline)
def create_dot_vistrails_if_necessary(self):
if os.path.exists(self._dot_vistrails):
if not os.path.isdir(self._dot_vistrails):
raise ValueError('The .vistrails directory cannot be used or '
'created because the specified path "%s" is '
'a file not a directory.' %
self._dot_vistrails)
else:
return
debug.log('Will try to create default directory')
try:
os.mkdir(self._dot_vistrails)
debug.log('Succeeded!')
except:
debug.critical("""Failed to create initialization directory.
This could be an indication of a permissions problem.
Make sure parent directory of '%s' is writable.""" %
self._dot_vistrails)
raise
def compute(self):
globalproppath = self.getInputFromPort(
'GlobalPropertyPath') if self.hasInputFromPort(
'GlobalPropertyPath') else "/parameters"
localpropsuffix = self.getInputFromPort(
'LocalPropertySuffix') if self.hasInputFromPort(
'LocalPropertySuffix') else "/parameters"
resroot = self.getInputFromPort('ResultPath') if self.hasInputFromPort(
'ResultPath') else "/timesteps/"
loader = Hdf5Loader()
self.getInputFromPort('ResultFiles')
if self.hasInputFromPort('ResultFiles'):
files = [
f.props["filename"]
for f in self.getInputFromPort('ResultFiles')
]
datasets = []
if self.hasInputFromPort('Measurements'):
#loop over files
for f in files:
try:
#open the file and open the results root group
h5file = h5.archive(f, 'r')
#enumerate the subgroups
L = h5file.list_children(resroot)
#Create an iterator of length the number of subgroups
stepper = [i + 1 for i in range(len(L))]
#Read in global props
globalprops = loader.GetProperties([f], globalproppath)
for d in stepper:
#Get the measurements from the numbered subgroups
locdata=loader.ReadMeasurementFromFile([f],proppath=resroot+str(d)+localpropsuffix, \
respath=resroot+str(d)+'/results', measurements=self.getInputFromPort('Measurements'))
#Append the global props to the local props
for i in range(len(locdata[0])):
locdata[0][i].props.update(globalprops[0].props)
#Extend the total dataset with this data
datasets.extend(locdata)
except Exception, e:
debug.log(traceback.format_exc())
def package_dependencies():
import vistrails.core.packagemanager
manager = vistrails.core.packagemanager.get_package_manager()
reg = vistrails.core.modules.module_registry.get_module_registry()
conf = get_vistrails_configuration()
abstraction_dir = get_vistrails_directory("subworkflowsDir")
if abstraction_dir is None:
debug.log("Subworkflows directory unset, cannot add any abstractions")
return []
p = re.compile(r".*\.xml")
all_packages = set()
for abstraction in os.listdir(abstraction_dir):
if p.match(abstraction):
abs_fname = os.path.join(abstraction_dir, abstraction)
vistrail = read_vistrail(abs_fname)
try:
dependencies = get_abstraction_dependencies(vistrail)
except vistrails.core.modules.module_registry.MissingPackage, e:
dependencies = {e._identifier: set()}
add_abstraction = True
inter_depends = []
for package, depends in dependencies.iteritems():
if package != identifier:
if not manager.has_package(package):
add_abstraction = False
break
else:
inter_depends.append(depends)
if add_abstraction:
# print 'adding', abstraction[:-4]
all_packages.update(p for p in dependencies.iterkeys()
if p != identifier)
my_vistrails[abstraction[:-4]] = \
(vistrail, abs_fname, inter_depends)
else:
debug.critical(
("Subworkflow '%s' is missing packages it " + "depends on")
% abstraction)
def dispatch_request(controller, module_id, current_pipeline):
pm = get_package_manager()
if module_id not in current_pipeline.modules:
# It is possible that some other upgrade request has
# already removed the invalid module of this request. In
# that case, disregard the request.
debug.log("module %s already handled. skipping" % module_id)
return []
invalid_module = current_pipeline.modules[module_id]
pkg = pm.get_package(invalid_module.package)
if hasattr(pkg.module, 'handle_module_upgrade_request'):
f = pkg.module.handle_module_upgrade_request
return f(controller, module_id, current_pipeline)
elif hasattr(pkg.module, '_upgrades'):
return UpgradeWorkflowHandler.remap_module(controller, module_id,
current_pipeline,
pkg.module._upgrades)
else:
debug.log('Package "%s" cannot handle upgrade request. '
'VisTrails will attempt automatic upgrade.' % \
pkg.identifier)
auto_upgrade = UpgradeWorkflowHandler.attempt_automatic_upgrade
return auto_upgrade(controller, current_pipeline, module_id)
def register_optimizers(reg):
modules = set()
for name in dir(tensorflow.train):
if name == 'Optimizer' or not name.endswith('Optimizer'):
continue
class_ = getattr(tensorflow.train, name)
if class_.__doc__ is None:
debug.log("Object has no __doc__: %s" % class_)
continue
args = guess_args(class_, class_.__init__.__doc__)
input_ports = [(arg, type_)
for arg, descr, type_ in args]
reg.add_module(type(name, (AutoOptimizer,),
{'args': args, 'class_': (class_,),
'_input_ports': input_ports,
'__doc__': class_.__doc__}),
namespace='train|optimizer')
modules.add(name)
return modules
def __init__(self, database=None):
if database is None:
self.database = ':memory:'
else:
self.database = database
self.entities = {}
self.deleted_entities = {}
self.temp_entities = {}
self.workspaces = {}
self.currentWorkspace = 'Default'
self.listeners = [] # listens for entity creation removal
self.max_id = 0
if not os.path.exists(self.database):
debug.log("'%s' does not exist. Trying to create" % self.database)
self.conn = sqlite3.connect(self.database)
try:
cur = self.conn.cursor()
for s in schema:
cur.execute(s)
self.conn.commit()
except Exception, e:
debug.critical("Could not create vistrail index schema", e)
L = h5file.list_children(resroot)
#Create an iterator of length the number of subgroups
stepper = [i + 1 for i in range(len(L))]
#Read in global props
globalprops = loader.GetProperties([f], globalproppath)
for d in stepper:
#Get the measurements from the numbered subgroups
locdata=loader.ReadMeasurementFromFile([f],proppath=resroot+str(d)+localpropsuffix, \
respath=resroot+str(d)+'/results', measurements=None)
#Append the global props to the local props
for i in range(len(locdata[0])):
locdata[0][i].props.update(globalprops[0].props)
#Extend the total dataset with this data
datasets.extend(locdata)
except Exception, e:
debug.log(traceback.format_exc())
self.setResult('data', datasets)
class LoadBinningAnalysis(Module):
"""Load the Binning Analysis from hdf5 files. Description of input ports:
@ResultFiles: The hdf5-files.
@Measurements: List of observables to load
@PropertyPath: Hdf5-path to the parameters stored. Default: /parameters
@ResultPath: Hdf5-path to the observables stored. Default: None"""
my_input_ports = [
PortDescriptor('ResultFiles', ResultFiles),
PortDescriptor('Measurements', basic.List),
PortDescriptor('PropertyPath', basic.String),
PortDescriptor('ResultPath', basic.String)
