def get_module(value, signature):
"""
Creates a module for value, in order to do the type checking.
"""
from vistrails.core.modules.basic_modules import Boolean, String, Integer, Float, List
if isinstance(value, Constant):
return type(value)
elif isinstance(value, bool):
return Boolean
elif isinstance(value, str):
return String
elif isinstance(value, int):
return Integer
elif isinstance(value, float):
return Float
elif isinstance(value, list):
return List
elif isinstance(value, tuple):
v_modules = ()
for element in xrange(len(value)):
v_modules += (get_module(value[element], signature[element]))
return v_modules
else:
from vistrails.core import debug
debug.warning("Could not identify the type of the list element.")
debug.warning("Type checking is not going to be done inside Map module.")
return None
def get_module(value, signature):
"""
Creates a module for value, in order to do the type checking.
"""
from vistrails.core.modules.basic_modules import Boolean, String, Integer, Float, List
if isinstance(value, Constant):
return type(value)
elif isinstance(value, bool):
return Boolean
elif isinstance(value, str):
return String
elif isinstance(value, int):
return Integer
elif isinstance(value, float):
return Float
elif isinstance(value, list):
return List
elif isinstance(value, tuple):
v_modules = ()
for element in xrange(len(value)):
v_modules += (get_module(value[element], signature[element]))
return v_modules
else:
from vistrails.core import debug
debug.warning("Could not identify the type of the list element.")
debug.warning(
"Type checking is not going to be done inside Map module.")
return None
def compute(self):
table = self.get_input('table')
delimiter = self.get_input('delimiter')
fileobj = self.interpreter.filePool.create_file(suffix='.csv')
fname = fileobj.name
with open(fname, 'w') as fp:
write_header = self.force_get_input('write_header')
if write_header is not False:
if table.names is None:
if write_header is True: # pragma: no cover
raise ModuleError(
self, "write_header is set but the table doesn't "
"have column names")
else:
fp.write(delimiter.join(table.names) + '\n')
cols = [iter(table.get_column(i)) for i in xrange(table.columns)]
if not cols:
raise ModuleError(self, "Table has no columns")
line = 0
for l in izip(*cols):
fp.write(delimiter.join(str(e) for e in l) + '\n')
line += 1
rows = table.rows
if line != rows: # pragma: no cover
debug.warning("WriteCSV wrote %d lines instead of expected "
"%d" % (line, rows))
self.set_output('file', fileobj)
def identifier_is_available(self, identifier):
"""Searchs for an available (but disabled) package.
If found, returns succesfully loaded, uninitialized package.
There can be multiple package versions for a single identifier. If so,
return the version that passes requirements, or the latest version.
"""
matches = []
for codepath in self.available_package_names_list():
pkg = self.get_available_package(codepath)
try:
pkg.load()
if pkg.identifier == identifier:
matches.append(pkg)
elif identifier in pkg.old_identifiers:
matches.append(pkg)
if (hasattr(pkg._module, "can_handle_identifier") and
pkg._module.can_handle_identifier(identifier)):
matches.append(pkg)
except (pkg.LoadFailed, pkg.InitializationFailed,
MissingRequirement):
pass
except Exception, e:
debug.warning(
"Error loading package <codepath %s>" % pkg.codepath,
e)
def create_startupxml_if_needed(self):
needs_create = True
fname = self.get_startup_xml_fname()
if os.path.isfile(fname):
try:
tree = ElementTree.parse(fname)
startup_version = \
vistrails.db.services.io.get_version_for_xml(tree.getroot())
version_list = version_string_to_list(startup_version)
if version_list >= [0,1]:
needs_create = False
except:
debug.warning("Unable to read startup.xml file, "
"creating a new one")
if needs_create:
root_dir = system.vistrails_root_directory()
origin = os.path.join(root_dir, 'core','resources',
'default_vistrails_startup_xml')
try:
shutil.copyfile(origin, fname)
debug.log('Succeeded!')
self.first_run = True
except:
debug.critical("""Failed to copy default configuration
file to %s. This could be an indication of a
permissions problem. Please make sure '%s' is writable."""
% (fname, self._dot_vistrails))
raise
def compute(self):
table = self.getInputFromPort('table')
delimiter = self.getInputFromPort('delimiter')
fileobj = self.interpreter.filePool.create_file(suffix='.csv')
fname = fileobj.name
with open(fname, 'w') as fp:
write_header = self.forceGetInputFromPort('write_header')
if write_header is not False:
if table.names is None:
if write_header is True: # pragma: no cover
raise ModuleError(
self,
"write_header is set but the table doesn't "
"have column names")
if not table.columns:
raise ModuleError(
self,
"Table has no columns")
nb_lines = self.write(fname, table, delimiter,
write_header is not False)
rows = table.rows
if nb_lines != rows: # pragma: no cover
debug.warning("WriteCSV wrote %d lines instead of expected "
"%d" % (nb_lines, rows))
self.setResult('file', fileobj)
def identifier_is_available(self, identifier):
"""Searchs for an available (but disabled) package.
If found, returns succesfully loaded, uninitialized package.
There can be multiple package versions for a single identifier. If so,
return the version that passes requirements, or the latest version.
"""
matches = []
for codepath in self.available_package_names_list():
pkg = self.get_available_package(codepath)
try:
pkg.load()
if pkg.identifier == identifier:
matches.append(pkg)
elif identifier in pkg.old_identifiers:
matches.append(pkg)
if (hasattr(pkg._module, "can_handle_identifier")
and pkg._module.can_handle_identifier(identifier)):
matches.append(pkg)
except (pkg.LoadFailed, pkg.InitializationFailed,
MissingRequirement):
pass
except Exception, e:
debug.warning(
"Error loading package <codepath %s>" % pkg.codepath, e)
def get_server_news():
global _server_news
if _server_news is not None:
return _server_news
dot_vistrails = os.path.expanduser('~/.vistrails')
if not os.path.exists(dot_vistrails):
os.mkdir(dot_vistrails)
file_name = os.path.join(dot_vistrails, 'server_news.json')
file_exists = os.path.exists(file_name)
headers = {}
if file_exists:
mtime = email.utils.formatdate(os.path.getmtime(file_name),
usegmt=True)
headers['If-Modified-Since'] = mtime
try:
resp = requests.get(
'https://reprozip-stats.poly.edu/vistrails_news/%s' %
vistrails_version(), headers=headers,
timeout=2 if file_exists else 10,
stream=True, verify=get_ca_certificate())
resp.raise_for_status()
if resp.status_code == 304:
raise requests.HTTPError(
'304 File is up to date, no data returned',
response=resp)
except requests.RequestException, e:
if not e.response or e.response.status_code != 304:
debug.warning("Can't download server news", e)
def compute(self):
table = self.get_input('table')
delimiter = self.get_input('delimiter')
fileobj = self.interpreter.filePool.create_file(suffix='.csv')
fname = fileobj.name
with open(fname, 'w') as fp:
write_header = self.force_get_input('write_header')
if write_header is not False:
if table.names is None:
if write_header is True: # pragma: no cover
raise ModuleError(
self, "write_header is set but the table doesn't "
"have column names")
if not table.columns:
raise ModuleError(self, "Table has no columns")
nb_lines = self.write(fname, table, delimiter, write_header
is not False)
rows = table.rows
if nb_lines != rows: # pragma: no cover
debug.warning("WriteCSV wrote %d lines instead of expected "
"%d" % (nb_lines, rows))
self.set_output('file', fileobj)
def get_module(value, signature):
"""
Creates a module for value, in order to do the type checking.
"""
if isinstance(value, Constant):
return type(value)
elif isinstance(value, bool):
return Boolean
elif isinstance(value, str):
return String
elif isinstance(value, int):
return Integer
elif isinstance(value, float):
return Float
elif isinstance(value, list):
return List
elif isinstance(value, tuple):
v_modules = ()
for element in xrange(len(value)):
v_modules += (get_module(value[element], signature[element]),)
return v_modules
else:
debug.warning("Could not identify the type of the list element.")
debug.warning("Type checking is not going to be done inside"
"FoldWithModule module.")
return None
def open(self):
retry = True
while retry:
config = {'host': self.host,
'port': self.port,
'user': self.user}
# unfortunately keywords are not standard across libraries
if self.protocol == 'mysql':
config['db'] = self.db_name
if self.password is not None:
config['passwd'] = self.password
elif self.protocol == 'postgresql':
config['database'] = self.db_name
if self.password is not None:
config['password'] = self.password
try:
self.conn = self.get_db_lib().connect(**config)
break
except self.get_db_lib().Error, e:
debug.warning(str(e))
if (e[0] == 1045 or self.get_db_lib().OperationalError
and self.password is None):
passwd_dlg = QPasswordEntry()
if passwd_dlg.exec_():
self.password = passwd_dlg.get_password()
else:
retry = False
else:
raise ModuleError(self, str(e))
def updateVersion(self, versionNumber):
""" updateVersion(versionNumber: int) -> None
Update the property page of the version
"""
self.versionNumber = versionNumber
self.versionNotes.updateVersion(versionNumber)
self.versionThumbs.updateVersion(versionNumber)
self.versionMashups.updateVersion(versionNumber)
if self.controller:
if self.use_custom_colors:
custom_color = self.controller.vistrail.get_action_annotation(
versionNumber, custom_color_key)
if custom_color is not None:
try:
custom_color = parse_custom_color(custom_color.value)
custom_color = QtGui.QColor(*custom_color)
except ValueError, e:
debug.warning("Version %r has invalid color "
"annotation (%s)" % (versionNumber, e))
custom_color = None
self.customColor.setColor(custom_color)
if self.controller.vistrail.actionMap.has_key(versionNumber):
action = self.controller.vistrail.actionMap[versionNumber]
name = self.controller.vistrail.getVersionName(versionNumber)
self.tagEdit.setText(name)
self.userEdit.setText(action.user)
self.dateEdit.setText(action.date)
self.idEdit.setText(unicode(action.id))
self.tagEdit.setEnabled(True)
return
else:
self.tagEdit.setEnabled(False)
self.tagReset.setEnabled(False)
def compute_evaluation_order(self, aliases):
# Build the dependencies graph
dp = {}
for alias, (atype, (base, exp)) in aliases.items():
edges = []
for e in exp:
edges += self.get_name_dependencies()
dp[alias] = edges
# Topological Sort to find the order to compute aliases
# Just a slow implementation, O(n^3)...
unordered = copy.copy(list(aliases.keys()))
ordered = []
while unordered:
added = []
for i in xrange(len(unordered)):
ok = True
u = unordered[i]
for j in xrange(len(unordered)):
if i != j:
for v in dp[unordered[j]]:
if u == v:
ok = False
break
if not ok: break
if ok: added.append(i)
if not added:
debug.warning('Looping dependencies detected!')
break
for i in reversed(added):
ordered.append(unordered[i])
del unordered[i]
return ordered
def py_import(module_name, dependency_dictionary, store_in_config=False):
"""Tries to import a python module, installing if necessary.
If the import doesn't succeed, we guess which system we are running on and
install the corresponding package from the dictionary. We then run the
import again.
If the installation fails, we won't try to install that same module again
for the session.
"""
try:
result = _vanilla_import(module_name)
return result
except ImportError:
if not getattr(get_vistrails_configuration(), 'installBundles'):
raise
if module_name in _previously_failed_pkgs:
raise PyImportException("Import of Python module '%s' failed again, "
"not triggering installation" % module_name)
if store_in_config:
ignored_packages_list = getattr(get_vistrails_configuration(),
'bundleDeclinedList',
None)
if ignored_packages_list:
ignored_packages = set(ignored_packages_list.split(';'))
else:
ignored_packages = set()
if module_name in ignored_packages:
raise PyImportException("Import of Python module '%s' failed "
"again, installation disabled by "
"configuration" % module_name)
debug.warning("Import of python module '%s' failed. "
"Will try to install bundle." % module_name)
success = vistrails.core.bundles.installbundle.install(
dependency_dictionary)
if store_in_config:
if bool(success):
ignored_packages.discard(module_name)
else:
ignored_packages.add(module_name)
setattr(get_vistrails_configuration(),
'bundleDeclinedList',
';'.join(sorted(ignored_packages)))
setattr(get_vistrails_persistent_configuration(),
'bundleDeclinedList',
';'.join(sorted(ignored_packages)))
if not success:
_previously_failed_pkgs.add(module_name)
raise PyImportException("Installation of Python module '%s' failed." %
module_name)
try:
result = _vanilla_import(module_name)
return result
except ImportError, e:
_previously_failed_pkgs.add(module_name)
raise PyImportBug("Installation of package '%s' succeeded, but import "
"still fails." % module_name)
def compute_evaluation_order(self, aliases):
# Build the dependencies graph
dp = {}
for alias,(atype,(base,exp)) in aliases.items():
edges = []
for e in exp:
edges += self.get_name_dependencies()
dp[alias] = edges
# Topological Sort to find the order to compute aliases
# Just a slow implementation, O(n^3)...
unordered = copy.copy(list(aliases.keys()))
ordered = []
while unordered:
added = []
for i in xrange(len(unordered)):
ok = True
u = unordered[i]
for j in xrange(len(unordered)):
if i!=j:
for v in dp[unordered[j]]:
if u==v:
ok = False
break
if not ok: break
if ok: added.append(i)
if not added:
debug.warning('Looping dependencies detected!')
break
for i in reversed(added):
ordered.append(unordered[i])
del unordered[i]
return ordered
def install(dependency_dictionary):
"""Tries to install a bundle after a py_import() failed.."""
distro = guess_system()
files = (dependency_dictionary.get(distro) or
dependency_dictionary.get('pip'))
if not files:
debug.warning("No source for bundle on this platform")
return None
can_install = (('pip' in dependency_dictionary and pip_installed) or
distro in dependency_dictionary)
if can_install:
action = show_question(
files,
distro in dependency_dictionary,
'pip' in dependency_dictionary)
if action == 'distro':
callable_ = getattr(vistrails.gui.bundles.installbundle,
distro.replace('-', '_') + '_install')
return callable_(files)
elif action == 'pip':
if not pip_installed:
debug.warning("Attempted to use pip, but it is not installed.")
return False
return pip_install(dependency_dictionary.get('pip'))
else:
return False
def create_startupxml_if_needed(self):
needs_create = True
fname = self.get_startup_xml_fname()
if os.path.isfile(fname):
try:
tree = ElementTree.parse(fname)
startup_version = \
vistrails.db.services.io.get_version_for_xml(tree.getroot())
version_list = version_string_to_list(startup_version)
if version_list >= [0, 1]:
needs_create = False
except:
debug.warning("Unable to read startup.xml file, "
"creating a new one")
if needs_create:
root_dir = system.vistrails_root_directory()
origin = os.path.join(root_dir, 'core', 'resources',
'default_vistrails_startup_xml')
try:
shutil.copyfile(origin, fname)
debug.log('Succeeded!')
self.first_run = True
except:
debug.critical("""Failed to copy default configuration
file to %s. This could be an indication of a
permissions problem. Please make sure '%s' is writable.""" %
(fname, self._dot_vistrails))
raise
def py_import(module_name, dependency_dictionary):
"""Tries to import a python module, installing if necessary.
If the import doesn't succeed, we guess which system we are running on and
install the corresponding package from the dictionary. We then run the
import again.
If the installation fails, we won't try to install that same module again
for the session.
"""
try:
result = _vanilla_import(module_name)
return result
except ImportError:
if not getattr(get_vistrails_configuration(), 'installBundles'):
raise
if module_name in _previously_failed_pkgs:
raise PyImportException("Import of Python module '%s' failed again, "
"not triggering installation" % module_name)
debug.warning("Import of python module '%s' failed. "
"Will try to install bundle." % module_name)
success = vistrails.core.bundles.installbundle.install(dependency_dictionary)
if not success:
_previously_failed_pkgs.add(module_name)
raise PyImportException("Installation of Python module '%s' failed." %
module_name)
try:
result = _vanilla_import(module_name)
return result
except ImportError, e:
_previously_failed_pkgs.add(module_name)
raise PyImportBug("Installation of package '%s' succeeded, but import "
"still fails." % module_name)
def adjust_version_colors(self, controller):
""" adjust_version_colors(controller: VistrailController) -> None
Based on the controller to set version colors
"""
currentUser = controller.vistrail.getUser()
ranks = {}
ourMaxRank = 0
otherMaxRank = 0
am = controller.vistrail.actionMap
for nodeId in sorted(self.versions.keys()):
if nodeId!=0:
nodeUser = am[nodeId].user
if nodeUser==currentUser:
ranks[nodeId] = ourMaxRank
ourMaxRank += 1
else:
ranks[nodeId] = otherMaxRank
otherMaxRank += 1
for (nodeId, item) in self.versions.iteritems():
if nodeId == 0:
item.setGhosted(True)
continue
nodeUser = am[nodeId].user
if controller.search and nodeId!=0:
action = am[nodeId]
if getattr(get_vistrails_configuration(), 'hideUpgrades',
True):
# Use upgraded version to match
action = am[controller.vistrail.get_upgrade(nodeId, False)]
ghosted = not controller.search.match(controller, action)
else:
ghosted = False
#item.setGhosted(ghosted) # we won't set it now so we can check if
# the state changed in update_color
if nodeUser==currentUser:
max_rank = ourMaxRank
else:
max_rank = otherMaxRank
# max_rank = ourMaxRank if nodeUser==currentUser else otherMaxRank
configuration = get_vistrails_configuration()
if configuration.check('customVersionColors'):
custom_color = controller.vistrail.get_action_annotation(
nodeId,
custom_color_key)
if custom_color is not None:
try:
custom_color = parse_custom_color(custom_color.value)
except ValueError, e:
debug.warning("Version %r has invalid color annotation "
"(%s)" % (nodeId, e))
custom_color = None
else:
custom_color = None
####
item.update_color(nodeUser==currentUser,
ranks[nodeId],
max_rank, ghosted, custom_color)
def download(self, url):
"""download(url:string) -> (result: int, downloaded_file: File,
local_filename:string)
Tries to download a file from url. It returns a tuple with:
result: 0 -> success
1 -> couldn't download the file, but found a cached version
2 -> failed (in this case downloaded_file will contain the
error message)
downloaded_file: The downloaded file or the error message in case it
failed
local_filename: the path to the local_filename
"""
self._parse_url(url)
opener = urllib2.build_opener()
local_filename = self._local_filename(url)
# Get ETag from disk
try:
with open(local_filename + '.etag') as etag_file:
etag = etag_file.read()
except IOError:
etag = None
try:
request = urllib2.Request(url)
if etag is not None:
request.add_header(
'If-None-Match',
etag)
try:
mtime = email.utils.formatdate(os.path.getmtime(local_filename),
usegmt=True)
request.add_header(
'If-Modified-Since',
mtime)
except OSError:
pass
f1 = opener.open(request)
except urllib2.URLError, e:
if isinstance(e, urllib2.HTTPError) and e.code == 304:
# Not modified
result = vistrails.core.modules.basic_modules.File()
result.name = local_filename
return (0, result, local_filename)
if self._file_is_in_local_cache(local_filename):
debug.warning('A network error occurred. HTTPFile will use a '
'cached version of the file')
result = vistrails.core.modules.basic_modules.File()
result.name = local_filename
return (1, result, local_filename)
else:
return (2, (str(e)), local_filename)
def compute(self):
localpath = self.force_get_input('path')
hasquery = self.has_input('metadata')
hashash = self.has_input('hash')
file_store = get_default_store()
if hashash:
if localpath or hasquery:
raise ModuleError(self,
"Don't set other ports if 'hash' is set")
h = self.get_input('hash')._hash
self._set_result(file_store.get(h))
elif hasquery:
# Do the query
metadata = self.get_input_list('metadata')
metadata = dict(m.metadata for m in metadata)
# Find the most recent match
best = None
for entry in file_store.query(metadata):
if best is None or (KEY_TIME in entry.metadata and
entry[KEY_TIME] > best[KEY_TIME]):
best = entry
if best is not None:
self.check_path_type(best.filename)
if localpath and os.path.exists(localpath.name):
path = localpath.name
self.check_path_type(path)
if best is not None:
# Compare
if hash_path(path) != best['hash']:
# Record new version of external file
use_local = True
else:
# Recorded version is up to date
use_local = False
else:
# No external file: use recorded version
use_local = True
if use_local:
data = dict(metadata)
data[KEY_TYPE] = TYPE_INPUT
data[KEY_TIME] = datetime.strftime(datetime.utcnow(),
'%Y-%m-%d %H:%M:%S')
best = file_store.add(path, data)
self.annotate({'added_file': best['hash']})
elif localpath:
debug.warning("Local file does not exist: %s" % localpath)
if best is None:
raise ModuleError(self, "Query returned no file")
self._set_result(best)
else:
raise ModuleError(self,
"Missing input: set either 'metadata' "
"(optionally with path) or hash")
def compute(self):
localpath = self.force_get_input('path')
hasquery = self.has_input('metadata')
hashash = self.has_input('hash')
file_store = get_default_store()
if hashash:
if localpath or hasquery:
raise ModuleError(self,
"Don't set other ports if 'hash' is set")
h = self.get_input('hash')._hash
self._set_result(file_store.get(h))
elif hasquery:
# Do the query
metadata = self.get_input_list('metadata')
metadata = dict(m.metadata for m in metadata)
# Find the most recent match
best = None
for entry in file_store.query(metadata):
if best is None or (KEY_TIME in entry.metadata
and entry[KEY_TIME] > best[KEY_TIME]):
best = entry
if best is not None:
self.check_path_type(best.filename)
if localpath and os.path.exists(localpath.name):
path = localpath.name
self.check_path_type(path)
if best is not None:
# Compare
if hash_path(path) != best['hash']:
# Record new version of external file
use_local = True
else:
# Recorded version is up to date
use_local = False
else:
# No external file: use recorded version
use_local = True
if use_local:
data = dict(metadata)
data[KEY_TYPE] = TYPE_INPUT
data[KEY_TIME] = datetime.strftime(datetime.utcnow(),
'%Y-%m-%d %H:%M:%S')
best = file_store.add(path, data)
self.annotate({'added_file': best['hash']})
elif localpath:
debug.warning("Local file does not exist: %s" % localpath)
if best is None:
raise ModuleError(self, "Query returned no file")
self._set_result(best)
else:
raise ModuleError(
self, "Missing input: set either 'metadata' "
"(optionally with path) or hash")
def package_requirements():
from vistrails.core.requirements import require_python_module, python_module_exists
require_python_module(
"vtk", {"linux-debian": "python-vtk", "linux-ubuntu": "python-vtk", "linux-fedora": "vtk-python"}
)
if not python_module_exists("PyQt4"):
from vistrails.core import debug
debug.warning("PyQt4 is not available. There will be no interaction " "between VTK and the spreadsheet.")
def emit(self, record):
msg = "tej: %s" % self.format(record)
if record.levelno >= logging.CRITICAL:
debug.critical(msg)
elif record.levelno >= logging.WARNING:
debug.warning(msg)
elif record.levelno >= logging.INFO:
debug.log(msg)
else:
debug.debug(msg)
def emit(self, record):
msg = "tej: %s" % self.format(record)
if record.levelno >= logging.CRITICAL:
debug.critical(msg)
elif record.levelno >= logging.WARNING:
debug.warning(msg)
elif record.levelno >= logging.INFO:
debug.log(msg)
else:
debug.debug(msg)
def adjust_version_colors(self, controller):
""" adjust_version_colors(controller: VistrailController) -> None
Based on the controller to set version colors
"""
currentUser = controller.vistrail.getUser()
ranks = {}
ourMaxRank = 0
otherMaxRank = 0
am = controller.vistrail.actionMap
for nodeId in sorted(self.versions.keys()):
if nodeId!=0:
nodeUser = am[nodeId].user
if nodeUser==currentUser:
ranks[nodeId] = ourMaxRank
ourMaxRank += 1
else:
ranks[nodeId] = otherMaxRank
otherMaxRank += 1
for (nodeId, item) in self.versions.iteritems():
if nodeId == 0:
item.setGhosted(True)
continue
nodeUser = am[nodeId].user
if controller.search and nodeId!=0:
ghosted = not controller.search.match(controller.vistrail,
am[nodeId])
else:
ghosted = False
#item.setGhosted(ghosted) # we won't set it now so we can check if
# the state changed in update_color
if nodeUser==currentUser:
max_rank = ourMaxRank
else:
max_rank = otherMaxRank
# max_rank = ourMaxRank if nodeUser==currentUser else otherMaxRank
configuration = get_vistrails_configuration()
if configuration.check('enableCustomVersionColors'):
custom_color = controller.vistrail.get_action_annotation(
nodeId,
custom_color_key)
if custom_color is not None:
try:
custom_color = parse_custom_color(custom_color.value)
except ValueError, e:
debug.warning("Version %r has invalid color annotation "
"(%s)" % (nodeId, e))
custom_color = None
else:
custom_color = None
####
item.update_color(nodeUser==currentUser,
ranks[nodeId],
max_rank, ghosted, custom_color)
def _stage(self, filename):
fullpath = os.path.join(self.repo.path, filename)
if os.path.islink(fullpath):
debug.warning("Warning: not staging symbolic link %s" % os.path.basename(filename))
elif os.path.isdir(fullpath):
for f in os.listdir(fullpath):
self._stage(os.path.join(filename, f))
else:
if os.path.sep != '/':
filename = filename.replace(os.path.sep, '/')
self.repo.stage(filename)
def get_indent(s):
indent = 0
for c in s:
if c == ' ':
indent += 1
elif c == '\t':
debug.warning("Found a tab in Google docstring!")
indent += 4
else:
break
return indent
def _stage(self, filename):
fullpath = os.path.join(self.repo.path, filename)
if os.path.islink(fullpath):
debug.warning("Warning: not staging symbolic link %s" % os.path.basename(filename))
elif os.path.isdir(fullpath):
for f in os.listdir(fullpath):
self._stage(os.path.join(filename, f))
else:
if os.path.sep != '/':
filename = filename.replace(os.path.sep, '/')
self.repo.stage(filename)
def package_requirements():
from vistrails.core.requirements import require_python_module, \
python_module_exists
require_python_module('vtk', {
'linux-debian': 'python-vtk',
'linux-ubuntu': 'python-vtk',
'linux-fedora': 'vtk-python'})
if not python_module_exists('PyQt4'):
from vistrails.core import debug
debug.warning('PyQt4 is not available. There will be no interaction '
'between VTK and the spreadsheet.')
def update_module_execs(old_obj, translate_dict):
new_module_execs = []
for obj in old_obj.db_items:
if obj.vtType == 'module_exec':
new_module_execs.append(DBModuleExec.update_version(obj,
translate_dict))
elif obj.vtType == 'group_exec':
# cannot handle group execs
debug.warning('Cannot translate group exec')
elif obj.vtType == 'loop_exec':
debug.warning('Cannot translate loop exec')
return new_module_execs
def update_module_execs(old_obj, translate_dict):
new_module_execs = []
for obj in old_obj.db_items:
if obj.vtType == 'module_exec':
new_module_execs.append(
DBModuleExec.update_version(obj, translate_dict))
elif obj.vtType == 'group_exec':
# cannot handle group execs
debug.warning('Cannot translate group exec')
elif obj.vtType == 'loop_exec':
debug.warning('Cannot translate loop exec')
return new_module_execs
def _delete_files(dirname):
"""delete_files(dirname: str) -> None
Deletes all files inside dirname
"""
try:
for root, dirs, files in os.walk(dirname):
for fname in files:
os.unlink(os.path.join(root,fname))
except OSError, e:
debug.warning("Error when removing thumbnails: %s"%str(e))
def update_exec_items(which, old_obj, translate_dict, catch_loops=False):
new_items = []
for obj in old_obj.db_item_execs:
if catch_loops and obj.vtType == 'loop_exec':
debug.warning("Cannot translate loop_exec inside of loop_exec")
if obj.vtType == which:
if obj.vtType == 'module_exec':
new_items.append(DBModuleExec.update_version(obj, translate_dict))
elif obj.vtType == 'group_exec':
new_items.append(DBGroupExec.update_version(obj, translate_dict))
elif obj.vtType == 'loop_exec':
new_items.append(DBLoopExec.update_version(obj, translate_dict))
return new_items
def get_latest_vistrails_version():
"""get_latest_vistrails_version() -> string - Returns latest vistrails
release version as queried from vistrails.org"""
version = ""
version_url = "http://www.vistrails.org/download/download.php?id=release_version.txt"
try:
request = urllib2.Request(version_url)
get_latest_version = urllib2.urlopen(request)
version = get_latest_version.read().strip()
except urllib2.HTTPError, err:
debug.warning("Unable to check for updates: %s" % str(err))
return version
def remove_lru(self,n=1):
elements = self.elements.values()
elements.sort(key=lambda obj: obj.time)
num = min(n,len(elements))
debug.debug("Will remove %s elements from cache..."%num)
debug.debug("Cache has %s elements and %s bytes"%(len(elements),
self.size()))
for elem in itertools.islice(elements, num):
try:
del self.elements[elem.name]
os.unlink(elem.abs_name)
except os.error, e:
debug.warning("Could not remove file %s" % elem.abs_name, e)
def remove_lru(self, n=1):
elements = self.elements.values()
elements.sort(key=lambda obj: obj.time)
num = min(n, len(elements))
debug.debug("Will remove %s elements from cache..." % num)
debug.debug("Cache has %s elements and %s bytes" %
(len(elements), self.size()))
for elem in itertools.islice(elements, num):
try:
del self.elements[elem.name]
os.unlink(elem.abs_name)
except os.error, e:
debug.warning("Could not remove file %s" % elem.abs_name, e)
def update_exec_items(which, old_obj, translate_dict, catch_loops=False):
new_items = []
for obj in old_obj.db_item_execs:
if catch_loops and obj.vtType == 'loop_exec':
debug.warning("Cannot translate loop_exec inside of loop_exec")
if obj.vtType == which:
if obj.vtType == 'module_exec':
new_items.append(DBModuleExec.update_version(obj, translate_dict))
elif obj.vtType == 'group_exec':
new_items.append(DBGroupExec.update_version(obj, translate_dict))
elif obj.vtType == 'loop_exec':
new_items.append(DBLoopExec.update_version(obj, translate_dict))
return new_items
def create_directory(self, dir_name):
if not os.path.isdir(dir_name):
debug.warning('Will try to create directory "%s"' % dir_name)
try:
os.mkdir(dir_name)
return True
except Exception, e:
msg = ("Failed to create directory: '%s'."
"This could be an indication of a permissions problem."
"Make sure directory '%s' in writable." %
(str(e), dir_name))
debug.critical(msg)
sys.exit(1)
def create_directory(self, dir_name):
if not os.path.isdir(dir_name):
debug.warning('Will try to create directory "%s"' % dir_name)
try:
os.mkdir(dir_name)
return True
except Exception, e:
msg = ("Failed to create directory: '%s'."
"This could be an indication of a permissions problem."
"Make sure directory '%s' in writable." %
(str(e), dir_name))
debug.critical(msg)
sys.exit(1)
def get_latest_vistrails_version():
"""get_latest_vistrails_version() -> string - Returns latest vistrails
release version as queried from vistrails.org"""
version = ''
version_url = \
"http://www.vistrails.org/download/download.php?id=release_version.txt"
try:
request = urllib2.Request(version_url)
get_latest_version = urllib2.urlopen(request)
version = get_latest_version.read().strip()
except urllib2.HTTPError, err:
debug.warning("Unable to check for updates: %s" % str(err))
return version
def _delete_files(dirname):
"""delete_files(dirname: str) -> None
Deletes all files inside dirname
"""
if dirname is None:
return
try:
for root, dirs, files in os.walk(dirname):
for fname in files:
os.unlink(os.path.join(root, fname))
except OSError, e:
debug.warning("Error when removing thumbnails", e)
def _is_outdated(self):
local_time = datetime.utcfromtimestamp(
os.path.getmtime(self.local_filename))
try:
remote_time = strptime(self.mod_header, "%a, %d %b %Y %H:%M:%S %Z")
except ValueError:
try:
remote_time = strptime(self.mod_header,
"%a, %d %B %Y %H:%M:%S %Z")
except ValueError:
# unable to parse last-modified header, download file again
debug.warning("Unable to parse Last-Modified header, "
"downloading file")
return True
return remote_time > local_time
def registerSelf():
""" registerSelf() -> None
Registry module with the registry
"""
registry = get_module_registry()
registry.add_module(VTKViewCell)
registry.add_input_port(VTKViewCell, "Location", CellLocation)
from vistrails.core import debug
for (port,module) in [("SetRenderView",'vtkRenderView')]:
try:
registry.add_input_port(VTKViewCell, port,
'(%s:%s)' % (vtk_pkg_identifier, module))
except Exception, e:
debug.warning("Got an exception adding VTKViewCell's %s input "
"port" % port, e)
def _is_outdated(self, remoteHeader, localFile):
"""Checks whether local file is outdated."""
local_time = \
datetime.datetime.utcfromtimestamp(os.path.getmtime(localFile))
try:
remote_time = strptime(remoteHeader, "%a, %d %b %Y %H:%M:%S %Z")
except ValueError:
try:
remote_time = strptime(remoteHeader, "%a, %d %B %Y %H:%M:%S %Z")
except ValueError:
# unable to parse last-modified header, download file again
debug.warning("Unable to parse Last-Modified header"
", downloading file")
return True
return remote_time > local_time
def has_context_menu(self):
if hasattr(self._init_module, 'context_menu'):
return True
name = hasattr(self._init_module, 'contextMenuName')
callback = hasattr(self._init_module, 'callContextMenu')
if name and callback:
return True
elif name or callback:
if not self._warned_contextmenu_notboth:
debug.warning(
"In package '%s', only one of contextMenuName and "
"callContextMenu is provided; the context menu will "
"not be shown" % self.identifier)
self._warned_contextmenu_notboth = True
return False
def has_context_menu(self):
if hasattr(self._init_module, 'context_menu'):
return True
name = hasattr(self._init_module, 'contextMenuName')
callback = hasattr(self._init_module, 'callContextMenu')
if name and callback:
return True
elif name or callback:
if not self._warned_contextmenu_notboth:
debug.warning(
"In package '%s', only one of contextMenuName and "
"callContextMenu is provided; the context menu will "
"not be shown" % self.identifier)
self._warned_contextmenu_notboth = True
return False
def data_sync(self):
""" downloads/uploads/uses the local file depending on availability """
self.checksum_lookup()
# local file not on repository, so upload
if not self.on_server and os.path.isfile(self.in_file.name):
cookiejar = vistrails.gui.repository.QRepositoryDialog.cookiejar
if cookiejar:
register_openers(cookiejar=cookiejar)
params = {
'dataset_file': open(self.in_file.name, 'rb'),
'name': self.in_file.name.split('/')[-1],
'origin': 'vistrails',
'checksum': self.checksum
}
upload_url = "%s/datasets/upload/" % self.base_url
datagen, headers = multipart_encode(params)
request = urllib2.Request(upload_url, datagen, headers)
try:
result = urllib2.urlopen(request)
if result.code != 200:
show_warning("Upload Failure",
"Data failed to upload to repository")
# make temporarily uncachable
self.is_cacheable = self.invalidate_cache
else:
debug.warning("Push to repository was successful")
# make sure module caches
self.is_cacheable = self.validate_cache
except Exception, e:
show_warning("Upload Failure",
"Data failed to upload to repository")
# make temporarily uncachable
self.is_cacheable = self.invalidate_cache
debug.warning('RepoSync uploaded %s to the repository' % \
self.in_file.name)
else:
show_warning("Please login", ("You must be logged into the web"
" repository in order to upload "
"data. No data was synced"))
# make temporarily uncachable
self.is_cacheable = self.invalidate_cache
# use local data
self.set_output("file", self.in_file)
def compute(self):
# if server, grab local file using checksum id
if self.is_server:
self.check_input('checksum')
self.checksum = self.get_input("checksum")
# get file path
path_url = "%s/datasets/path/%s/"%(self.base_url, self.checksum)
dataset_path_request = urllib2.urlopen(url=path_url)
dataset_path = dataset_path_request.read()
if os.path.isfile(dataset_path):
out_file = PathObject(dataset_path)
self.set_output("file", out_file)
else: # is client
self.check_input('file')
self.in_file = self.get_input("file")
if os.path.isfile(self.in_file.name):
# do size check
size = os.path.getsize(self.in_file.name)
if size > 26214400:
debug.warning("File is too large (>25MB): unable to sync "
"with web repository")
self.set_output("file", self.in_file)
else:
# compute checksum
f = open(self.in_file.name, 'r')
self.checksum = sha_hash()
block = 1
while block:
block = f.read(128)
self.checksum.update(block)
f.close()
self.checksum = self.checksum.hexdigest()
# upload/download file
self.data_sync()
# set checksum param in module
if not self.has_input('checksum'):
self.change_parameter('checksum', [self.checksum])
else:
# local file not present
if self.has_input('checksum'):
self.checksum = self.get_input("checksum")
# download file
self.data_sync()
def updateVersion(self, versionNumber):
""" updateVersion(versionNumber: int) -> None
Update the property page of the version
"""
self.versionNumber = versionNumber
self.versionNotes.updateVersion(versionNumber)
self.versionThumbs.updateVersion(versionNumber)
self.versionMashups.updateVersion(versionNumber)
if self.controller:
vistrail = self.controller.vistrail
if self.use_custom_colors:
custom_color = vistrail.get_action_annotation(
versionNumber, custom_color_key)
if custom_color is not None:
try:
custom_color = parse_custom_color(custom_color.value)
custom_color = QtGui.QColor(*custom_color)
except ValueError, e:
debug.warning("Version %r has invalid color "
"annotation (%s)" % (versionNumber, e))
custom_color = None
self.customColor.setColor(custom_color)
if vistrail.actionMap.has_key(versionNumber):
# Follow upgrades forward to find tag
tag = self.controller.get_tag(versionNumber)
if getattr(get_vistrails_configuration(), 'hideUpgrades',
True):
base_ver = vistrail.get_base_upgrade_version(versionNumber)
else:
base_ver = versionNumber
action = vistrail.actionMap[base_ver]
self.tagEdit.setText(tag)
self.userEdit.setText(action.user)
self.dateEdit.setText(action.date)
if base_ver != versionNumber:
version_text = '%s (%s)' % (versionNumber, base_ver)
else:
version_text = '%s' % base_ver
self.idEdit.setText(version_text)
self.tagEdit.setEnabled(True)
return
else:
self.tagEdit.setEnabled(False)
self.tagReset.setEnabled(False)
def _is_outdated(self):
local_time = datetime.utcfromtimestamp(
os.path.getmtime(self.local_filename))
try:
remote_time = strptime(self.mod_header,
"%a, %d %b %Y %H:%M:%S %Z")
except ValueError:
try:
remote_time = strptime(self.mod_header,
"%a, %d %B %Y %H:%M:%S %Z")
except ValueError:
# unable to parse last-modified header, download file again
debug.warning("Unable to parse Last-Modified header, "
"downloading file")
return True
return remote_time > local_time
def package_requirements():
from vistrails.core.requirements import require_python_module, \
python_module_exists
require_python_module(
'vtk', {
'linux-debian': 'python-vtk',
'linux-ubuntu': 'python-vtk',
'linux-fedora': 'vtk-python'
})
from vistrails.gui.requirements import require_pyqt4_api2
try:
require_pyqt4_api2()
except MissingRequirement:
debug.warning('PyQt4 is not available. There will be no interaction '
'between VTK and the spreadsheet.')
def move_cache_directory(self, sourcedir, destdir):
"""change_cache_directory(sourcedir: str, dest_dir: str) -> None"
Moves files from sourcedir to destdir
"""
if os.path.exists(destdir):
for entry in self.elements.itervalues():
try:
srcname = entry.abs_name
dstname = os.path.join(destdir, entry.name)
shutil.move(srcname, dstname)
entry.abs_name = dstname
except shutil.Error, e:
debug.warning(
"Could not move thumbnail from %s to %s" %
(sourcedir, destdir), e)
def contextMenuEvent(self, event):
"""Just dispatches the menu event to the widget item"""
item = self.itemAt(event.pos())
if item:
# find top level
p = item
while p.parent():
p = p.parent()
# get package identifier
assert isinstance(p, QPackageTreeWidgetItem)
identifier = p.identifier
registry = get_module_registry()
package = registry.packages[identifier]
try:
if package.has_context_menu():
if isinstance(item, QPackageTreeWidgetItem):
text = None
elif isinstance(item, QNamespaceTreeWidgetItem):
return # no context menu for namespaces
elif isinstance(item, QModuleTreeWidgetItem):
text = item.descriptor.name
if item.descriptor.namespace:
text = '%s|%s' % (item.descriptor.namespace, text)
else:
assert False, "fell through"
menu_items = package.context_menu(text)
if menu_items:
menu = QtGui.QMenu(self)
for text, callback in menu_items:
act = QtGui.QAction(text, self)
act.setStatusTip(text)
QtCore.QObject.connect(
act, QtCore.SIGNAL("triggered()"), callback)
menu.addAction(act)
menu.exec_(event.globalPos())
return
except Exception, e:
debug.unexpected_exception(e)
debug.warning("Got exception trying to display %s's "
"context menu in the palette: %s\n%s" %
(package.name, debug.format_exception(e),
traceback.format_exc()))
item.contextMenuEvent(event, self)
