def write(self):
info_list = self.ref_search.ref_widget.get_info_list()
if len(info_list) < 1:
return
elif len(info_list) == 1:
# save single file/dir
info = info_list[0]
name = info[2]
chosen_path = QtGui.QFileDialog.getSaveFileName(
self, 'Save...', name)
if not chosen_path:
return
# FIXME really should move this calls to a higher level so
# we don't need to instantiate a module
if info[1] is None:
version = "HEAD"
else:
version = info[1]
repo.get_current_repo().get_path(info[0], version, None,
chosen_path)
else:
# have multiple files/dirs
chosen_path = QtGui.QFileDialog.getExistingDirectory(
self, 'Save All to Directory...')
has_overwrite = False
# if untitled (no name, use the uuid)
for info in info_list:
if info[2]:
name = info[2]
else:
name = info[0]
full_path = os.path.join(chosen_path, name)
if os.path.exists(full_path):
has_overwrite = True
if has_overwrite:
question_str = "One or more of the paths already exist. " + \
"Overwrite?"
ret_val = \
QtGui.QMessageBox.question(self, "Overwrite", \
question_str, \
QtGui.QMessageBox.Cancel | \
QtGui.QMessageBox.No | \
QtGui.QMessageBox.Yes)
if ret_val != QtGui.QMessageBox.Yes:
return
for info in info_list:
if info[1] is None:
version = "HEAD"
else:
version = info[1]
if info[2]:
name = info[2]
else:
name = info[0]
full_path = os.path.join(chosen_path, name)
repo.get_current_repo().git_get_path(info[0], version, None,
full_path)
def finalize():
# delete all temporary files/directories used by zip
global db_access, _configuration_widget
for fname in repo.get_current_repo().temp_persist_files:
if os.path.isfile(fname):
os.remove(fname)
elif os.path.isdir(fname):
shutil.rmtree(fname)
db_access.finalize()
if _configuration_widget is not None:
_configuration_widget.deleteLater()
def finalize():
# delete all temporary files/directories used by zip
global db_access, _configuration_widget
for fname in repo.get_current_repo().temp_persist_files:
if os.path.isfile(fname):
os.remove(fname)
elif os.path.isdir(fname):
shutil.rmtree(fname)
db_access.finalize()
if _configuration_widget is not None:
_configuration_widget.deleteLater()
def persistent_module_hasher(pipeline, module, chm):
current_hash = Hasher.module_signature(module, chm)
ref = None
read_local = False
for function in module.functions:
if function.name == "ref":
ref = PersistentRef.translate_to_python(function.params[0].strValue)
if function.name == 'readLocal':
read_local = \
Boolean.translate_to_python(function.params[0].strValue)
if ref and not read_local and db_access.ref_exists(ref.id, ref.version):
if ref.version is None:
ref.version = repo.get_current_repo().get_latest_version(ref.id)
return Hasher.compound_signature([current_hash, str(ref.id),
str(ref.version)])
return current_hash
def persistent_module_hasher(pipeline, module, chm):
current_hash = Hasher.module_signature(module, chm)
ref = None
read_local = False
for function in module.functions:
if function.name == "ref":
ref = PersistentRef.translate_to_python(
function.params[0].strValue)
if function.name == 'readLocal':
read_local = \
Boolean.translate_to_python(function.params[0].strValue)
if ref and not read_local and db_access.ref_exists(ref.id, ref.version):
if ref.version is None:
ref.version = repo.get_current_repo().get_latest_version(ref.id)
return Hasher.compound_signature(
[current_hash, str(ref.id),
str(ref.version)])
return current_hash
def compute(self, is_input=None, path_type=None):
global db_access
if not self.hasInputFromPort('value') and \
not self.hasInputFromPort('ref'):
raise ModuleError(self, "Need to specify path or reference")
if self.persistent_path is not None:
debug_print('using persistent path')
ref = self.persistent_ref
path = self.persistent_path
elif self.hasInputFromPort('ref'):
ref = self.getInputFromPort('ref')
if ref.id is None:
ref.id = str(uuid.uuid1())
else:
# create a new reference
ref = PersistentRef()
ref.id = str(uuid.uuid1())
if self.hasInputFromPort('localPath'):
ref.local_path = self.getInputFromPort('localPath').name
if self.hasInputFromPort('readLocal'):
ref.local_read = self.getInputFromPort('readLocal')
if self.hasInputFromPort('writeLocal'):
ref.local_writeback = self.getInputFromPort('writeLocal')
if is_input is None:
is_input = False
if not self.hasInputFromPort('value'):
is_input = True
else:
if ref.local_path and ref.local_read:
debug_print('found local path with local read')
is_input = True
# FIXME: check if the signature is the signature of
# the value if so we know that it's an input...
# if just reference, pull path from repository (get latest
# version unless specified as specific version)
if self.persistent_path is None and not self.hasInputFromPort('value') \
and is_input and not (ref.local_path and ref.local_read):
_, suffix = os.path.splitext(ref.name)
if not db_access.ref_exists(ref.id, ref.version):
raise ModuleError(self, "Persistent entity '%s' does not "
"exist in the local repository." % ref.id)
if ref.version is None:
ref.version = repo.get_current_repo().get_latest_version(ref.id)
# get specific ref.uuid, ref.version combo
path = repo.get_current_repo().get_path(ref.id, ref.version,
out_suffix=suffix)
elif self.persistent_path is None:
# copy path to persistent directory with uuid as name
if is_input and ref.local_path and ref.local_read:
debug_print('using local_path')
path = ref.local_path
else:
path = self.getInputFromPort('value').name
# this is a static method so we need to add module ourselves
try:
new_hash = repo.get_current_repo().compute_hash(path)
except ModuleError, e:
e.module = self
raise e
rep_path = os.path.join(local_db, ref.id)
do_update = True
if os.path.exists(rep_path):
if os.path.isdir(rep_path):
actual_type = 'tree'
elif os.path.isfile(rep_path):
actual_type = 'blob'
else:
raise ModuleError(self, "Path is something not a file or "
"a directory")
if path_type is None:
path_type = actual_type
else:
if path_type != actual_type:
def show_type(t):
if t == 'tree': return "directory"
elif t == 'blob': return "file"
else: return '"%s"' % t
raise ModuleError(self, "Path is not a %s but a %s" % (
show_type(path_type),
show_type(actual_type)))
old_hash = repo.get_current_repo().get_hash(ref.id,
path_type=path_type)
debug_print('old_hash:', old_hash)
debug_print('new_hash:', new_hash)
if old_hash == new_hash:
do_update = False
if do_update:
debug_print('doing update')
if path_type == 'tree':
if (not os.path.exists(path) or
not os.listdir(path)):
raise ModuleError(self, "This directory is empty")
self.copypath(path, os.path.join(local_db, ref.id))
# commit (and add to) repository
# get commit id as version id
# persist object-hash, commit-version to repository
version = repo.get_current_repo().add_commit(ref.id)
ref.version = version
# write object-hash, commit-version to provenance
if is_input:
signature = new_hash
else:
signature = self.signature
db_access.write_database({'id': ref.id,
'name': ref.name,
'tags': ref.tags,
'user': current_user(),
'date_created': current_time(),
'date_modified': current_time(),
'content_hash': new_hash,
'version': version,
'signature': signature,
'type': path_type})
def updateUpstream(self, is_input=None, path_type=None):
global db_access
if is_input is None:
if not self.hasInputFromPort('value'):
is_input = True
else:
# FIXME: check if the signature is the signature of
# the value if so we know that it's an input...
is_input = False
self.persistent_ref = None
self.persistent_path = None
if is_input:
return super(PersistentPath, self).updateUpstream()
# can check updateUpstream
if not hasattr(self, 'signature'):
raise ModuleError(self, 'Module has no signature')
ref_exists = False
if not self.hasInputFromPort('ref'):
# create new reference with no name or tags
ref = PersistentRef()
ref.signature = self.signature
else:
# update single port
self.updateUpstreamPort('ref')
ref = self.getInputFromPort('ref')
if db_access.ref_exists(ref.id, ref.version):
ref_exists = True
if ref.version is None:
ref.version = \
repo.get_current_repo().get_latest_version(ref.id)
signature = db_access.get_signature(ref.id, ref.version)
if signature == self.signature:
# don't need to create a new version
self.persistent_ref = ref
# Note that ref_exists is True if the reference is a fixed
# reference; if it was assigned a new uuid, we can still
# reuse a reference with the same signature
if not ref_exists:
signature = self.signature
debug_print('searching for signature', signature)
sig_ref = db_access.search_by_signature(signature)
debug_print('sig_ref:', sig_ref)
if sig_ref:
debug_print('setting persistent_ref')
ref.id, ref.version, ref.name = sig_ref
self.persistent_ref = ref
# else:
# ref.id = uuid.uuid1()
# copy as normal
# don't copy if equal
# FIXME also need to check that the file actually exists here!
if self.persistent_ref is not None:
_, suffix = os.path.splitext(self.persistent_ref.name)
self.persistent_path = repo.get_current_repo().get_path(
self.persistent_ref.id,
self.persistent_ref.version,
out_suffix=suffix)
debug_print("FOUND persistent path")
debug_print(self.persistent_path)
debug_print(self.persistent_ref.local_path)
if self.persistent_ref is None or self.persistent_path is None:
debug_print("NOT FOUND persistent path")
super(PersistentPath, self).updateUpstream()
def write(self):
info_list = self.ref_search.ref_widget.get_info_list()
if len(info_list) < 1:
return
elif len(info_list) == 1:
# save single file/dir
info = info_list[0]
name = info[2]
chosen_path = QtGui.QFileDialog.getSaveFileName(
self,
'Save...',
name)
if not chosen_path:
return
# FIXME really should move this calls to a higher level so
# we don't need to instantiate a module
if info[1] is None:
version = "HEAD"
else:
version = info[1]
repo.get_current_repo().get_path(info[0], version, None,
chosen_path)
else:
# have multiple files/dirs
chosen_path = QtGui.QFileDialog.getExistingDirectory(
self,
'Save All to Directory...')
has_overwrite = False
# if untitled (no name, use the uuid)
for info in info_list:
if info[2]:
name = info[2]
else:
name = info[0]
full_path = os.path.join(chosen_path, name)
if os.path.exists(full_path):
has_overwrite = True
if has_overwrite:
question_str = "One or more of the paths already exist. " + \
"Overwrite?"
ret_val = \
QtGui.QMessageBox.question(self, "Overwrite", \
question_str, \
QtGui.QMessageBox.Cancel | \
QtGui.QMessageBox.No | \
QtGui.QMessageBox.Yes)
if ret_val != QtGui.QMessageBox.Yes:
return
for info in info_list:
if info[1] is None:
version = "HEAD"
else:
version = info[1]
if info[2]:
name = info[2]
else:
name = info[0]
full_path = os.path.join(chosen_path, name)
repo.get_current_repo().git_get_path(info[0], version, None,
full_path)
def compute(self, is_input=None, path_type=None):
global db_access
if not self.has_input('value') and \
not self.has_input('ref'):
raise ModuleError(self, "Need to specify path or reference")
if self.persistent_path is not None:
debug_print('using persistent path')
ref = self.persistent_ref
path = self.persistent_path
elif self.has_input('ref'):
ref = self.get_input('ref')
if ref.id is None:
ref.id = str(uuid.uuid1())
else:
# create a new reference
ref = PersistentRef()
ref.id = str(uuid.uuid1())
if self.has_input('localPath'):
ref.local_path = self.get_input('localPath').name
if self.has_input('readLocal'):
ref.local_read = self.get_input('readLocal')
if self.has_input('writeLocal'):
ref.local_writeback = self.get_input('writeLocal')
if is_input is None:
is_input = False
if not self.has_input('value'):
is_input = True
else:
if ref.local_path and ref.local_read:
debug_print('found local path with local read')
is_input = True
# FIXME: check if the signature is the signature of
# the value if so we know that it's an input...
# if just reference, pull path from repository (get latest
# version unless specified as specific version)
if self.persistent_path is None and not self.has_input('value') \
and is_input and not (ref.local_path and ref.local_read):
_, suffix = os.path.splitext(ref.name)
if not db_access.ref_exists(ref.id, ref.version):
raise ModuleError(
self, "Persistent entity '%s' does not "
"exist in the local repository." % ref.id)
if ref.version is None:
ref.version = repo.get_current_repo().get_latest_version(
ref.id)
# get specific ref.uuid, ref.version combo
path = repo.get_current_repo().get_path(ref.id,
ref.version,
out_suffix=suffix)
elif self.persistent_path is None:
# copy path to persistent directory with uuid as name
if is_input and ref.local_path and ref.local_read:
debug_print('using local_path')
path = ref.local_path
else:
path = self.get_input('value').name
# this is a static method so we need to add module ourselves
try:
new_hash = repo.get_current_repo().compute_hash(path)
except ModuleError, e:
e.module = self
raise e
rep_path = os.path.join(local_db, ref.id)
do_update = True
if os.path.exists(rep_path):
if os.path.isdir(rep_path):
actual_type = 'tree'
elif os.path.isfile(rep_path):
actual_type = 'blob'
else:
raise ModuleError(
self, "Path is something not a file or "
"a directory")
if path_type is None:
path_type = actual_type
else:
if path_type != actual_type:
def show_type(t):
if t == 'tree': return "directory"
elif t == 'blob': return "file"
else: return '"%s"' % t
raise ModuleError(
self, "Path is not a %s but a %s" %
(show_type(path_type), show_type(actual_type)))
old_hash = repo.get_current_repo().get_hash(
ref.id, path_type=path_type)
debug_print('old_hash:', old_hash)
debug_print('new_hash:', new_hash)
if old_hash == new_hash:
do_update = False
if do_update:
debug_print('doing update')
if path_type == 'tree':
if (not os.path.exists(path) or not os.listdir(path)):
raise ModuleError(self, "This directory is empty")
self.copypath(path, os.path.join(local_db, ref.id))
# commit (and add to) repository
# get commit id as version id
# persist object-hash, commit-version to repository
version = repo.get_current_repo().add_commit(ref.id)
ref.version = version
# write object-hash, commit-version to provenance
if is_input:
signature = new_hash
else:
signature = self.signature
db_access.write_database({
'id': ref.id,
'name': ref.name,
'tags': ref.tags,
'user': current_user(),
'date_created': current_time(),
'date_modified': current_time(),
'content_hash': new_hash,
'version': version,
'signature': signature,
'type': path_type
})
def update_upstream(self, is_input=None, path_type=None):
global db_access
if is_input is None:
if not self.has_input('value'):
is_input = True
else:
# FIXME: check if the signature is the signature of
# the value if so we know that it's an input...
is_input = False
self.persistent_ref = None
self.persistent_path = None
if is_input:
return super(PersistentPath, self).update_upstream()
# can check update_upstream
if not hasattr(self, 'signature'):
raise ModuleError(self, 'Module has no signature')
ref_exists = False
if not self.has_input('ref'):
# create new reference with no name or tags
ref = PersistentRef()
ref.signature = self.signature
else:
# update single port
self.update_upstream_port('ref')
ref = self.get_input('ref')
if db_access.ref_exists(ref.id, ref.version):
ref_exists = True
if ref.version is None:
ref.version = \
repo.get_current_repo().get_latest_version(ref.id)
signature = db_access.get_signature(ref.id, ref.version)
if signature == self.signature:
# don't need to create a new version
self.persistent_ref = ref
# Note that ref_exists is True if the reference is a fixed
# reference; if it was assigned a new uuid, we can still
# reuse a reference with the same signature
if not ref_exists:
signature = self.signature
debug_print('searching for signature', signature)
sig_ref = db_access.search_by_signature(signature)
debug_print('sig_ref:', sig_ref)
if sig_ref:
debug_print('setting persistent_ref')
ref.id, ref.version, ref.name = sig_ref
self.persistent_ref = ref
# else:
# ref.id = uuid.uuid1()
# copy as normal
# don't copy if equal
# FIXME also need to check that the file actually exists here!
if self.persistent_ref is not None:
_, suffix = os.path.splitext(self.persistent_ref.name)
self.persistent_path = repo.get_current_repo().get_path(
self.persistent_ref.id,
self.persistent_ref.version,
out_suffix=suffix)
debug_print("FOUND persistent path")
debug_print(self.persistent_path)
debug_print(self.persistent_ref.local_path)
if self.persistent_ref is None or self.persistent_path is None:
debug_print("NOT FOUND persistent path")
super(PersistentPath, self).update_upstream()
