def write(self): info_list = self.ref_search.ref_widget.get_info_list() if len(info_list) < 1: return elif len(info_list) == 1: # save single file/dir info = info_list[0] name = info[2] chosen_path = QtGui.QFileDialog.getSaveFileName( self, 'Save...', name) if not chosen_path: return # FIXME really should move this calls to a higher level so # we don't need to instantiate a module if info[1] is None: version = "HEAD" else: version = info[1] repo.get_current_repo().get_path(info[0], version, None, chosen_path) else: # have multiple files/dirs chosen_path = QtGui.QFileDialog.getExistingDirectory( self, 'Save All to Directory...') has_overwrite = False # if untitled (no name, use the uuid) for info in info_list: if info[2]: name = info[2] else: name = info[0] full_path = os.path.join(chosen_path, name) if os.path.exists(full_path): has_overwrite = True if has_overwrite: question_str = "One or more of the paths already exist. " + \ "Overwrite?" ret_val = \ QtGui.QMessageBox.question(self, "Overwrite", \ question_str, \ QtGui.QMessageBox.Cancel | \ QtGui.QMessageBox.No | \ QtGui.QMessageBox.Yes) if ret_val != QtGui.QMessageBox.Yes: return for info in info_list: if info[1] is None: version = "HEAD" else: version = info[1] if info[2]: name = info[2] else: name = info[0] full_path = os.path.join(chosen_path, name) repo.get_current_repo().git_get_path(info[0], version, None, full_path)
def finalize(): # delete all temporary files/directories used by zip global db_access, _configuration_widget for fname in repo.get_current_repo().temp_persist_files: if os.path.isfile(fname): os.remove(fname) elif os.path.isdir(fname): shutil.rmtree(fname) db_access.finalize() if _configuration_widget is not None: _configuration_widget.deleteLater()
def persistent_module_hasher(pipeline, module, chm): current_hash = Hasher.module_signature(module, chm) ref = None read_local = False for function in module.functions: if function.name == "ref": ref = PersistentRef.translate_to_python(function.params[0].strValue) if function.name == 'readLocal': read_local = \ Boolean.translate_to_python(function.params[0].strValue) if ref and not read_local and db_access.ref_exists(ref.id, ref.version): if ref.version is None: ref.version = repo.get_current_repo().get_latest_version(ref.id) return Hasher.compound_signature([current_hash, str(ref.id), str(ref.version)]) return current_hash
def persistent_module_hasher(pipeline, module, chm): current_hash = Hasher.module_signature(module, chm) ref = None read_local = False for function in module.functions: if function.name == "ref": ref = PersistentRef.translate_to_python( function.params[0].strValue) if function.name == 'readLocal': read_local = \ Boolean.translate_to_python(function.params[0].strValue) if ref and not read_local and db_access.ref_exists(ref.id, ref.version): if ref.version is None: ref.version = repo.get_current_repo().get_latest_version(ref.id) return Hasher.compound_signature( [current_hash, str(ref.id), str(ref.version)]) return current_hash
def compute(self, is_input=None, path_type=None): global db_access if not self.hasInputFromPort('value') and \ not self.hasInputFromPort('ref'): raise ModuleError(self, "Need to specify path or reference") if self.persistent_path is not None: debug_print('using persistent path') ref = self.persistent_ref path = self.persistent_path elif self.hasInputFromPort('ref'): ref = self.getInputFromPort('ref') if ref.id is None: ref.id = str(uuid.uuid1()) else: # create a new reference ref = PersistentRef() ref.id = str(uuid.uuid1()) if self.hasInputFromPort('localPath'): ref.local_path = self.getInputFromPort('localPath').name if self.hasInputFromPort('readLocal'): ref.local_read = self.getInputFromPort('readLocal') if self.hasInputFromPort('writeLocal'): ref.local_writeback = self.getInputFromPort('writeLocal') if is_input is None: is_input = False if not self.hasInputFromPort('value'): is_input = True else: if ref.local_path and ref.local_read: debug_print('found local path with local read') is_input = True # FIXME: check if the signature is the signature of # the value if so we know that it's an input... # if just reference, pull path from repository (get latest # version unless specified as specific version) if self.persistent_path is None and not self.hasInputFromPort('value') \ and is_input and not (ref.local_path and ref.local_read): _, suffix = os.path.splitext(ref.name) if not db_access.ref_exists(ref.id, ref.version): raise ModuleError(self, "Persistent entity '%s' does not " "exist in the local repository." % ref.id) if ref.version is None: ref.version = repo.get_current_repo().get_latest_version(ref.id) # get specific ref.uuid, ref.version combo path = repo.get_current_repo().get_path(ref.id, ref.version, out_suffix=suffix) elif self.persistent_path is None: # copy path to persistent directory with uuid as name if is_input and ref.local_path and ref.local_read: debug_print('using local_path') path = ref.local_path else: path = self.getInputFromPort('value').name # this is a static method so we need to add module ourselves try: new_hash = repo.get_current_repo().compute_hash(path) except ModuleError, e: e.module = self raise e rep_path = os.path.join(local_db, ref.id) do_update = True if os.path.exists(rep_path): if os.path.isdir(rep_path): actual_type = 'tree' elif os.path.isfile(rep_path): actual_type = 'blob' else: raise ModuleError(self, "Path is something not a file or " "a directory") if path_type is None: path_type = actual_type else: if path_type != actual_type: def show_type(t): if t == 'tree': return "directory" elif t == 'blob': return "file" else: return '"%s"' % t raise ModuleError(self, "Path is not a %s but a %s" % ( show_type(path_type), show_type(actual_type))) old_hash = repo.get_current_repo().get_hash(ref.id, path_type=path_type) debug_print('old_hash:', old_hash) debug_print('new_hash:', new_hash) if old_hash == new_hash: do_update = False if do_update: debug_print('doing update') if path_type == 'tree': if (not os.path.exists(path) or not os.listdir(path)): raise ModuleError(self, "This directory is empty") self.copypath(path, os.path.join(local_db, ref.id)) # commit (and add to) repository # get commit id as version id # persist object-hash, commit-version to repository version = repo.get_current_repo().add_commit(ref.id) ref.version = version # write object-hash, commit-version to provenance if is_input: signature = new_hash else: signature = self.signature db_access.write_database({'id': ref.id, 'name': ref.name, 'tags': ref.tags, 'user': current_user(), 'date_created': current_time(), 'date_modified': current_time(), 'content_hash': new_hash, 'version': version, 'signature': signature, 'type': path_type})
def updateUpstream(self, is_input=None, path_type=None): global db_access if is_input is None: if not self.hasInputFromPort('value'): is_input = True else: # FIXME: check if the signature is the signature of # the value if so we know that it's an input... is_input = False self.persistent_ref = None self.persistent_path = None if is_input: return super(PersistentPath, self).updateUpstream() # can check updateUpstream if not hasattr(self, 'signature'): raise ModuleError(self, 'Module has no signature') ref_exists = False if not self.hasInputFromPort('ref'): # create new reference with no name or tags ref = PersistentRef() ref.signature = self.signature else: # update single port self.updateUpstreamPort('ref') ref = self.getInputFromPort('ref') if db_access.ref_exists(ref.id, ref.version): ref_exists = True if ref.version is None: ref.version = \ repo.get_current_repo().get_latest_version(ref.id) signature = db_access.get_signature(ref.id, ref.version) if signature == self.signature: # don't need to create a new version self.persistent_ref = ref # Note that ref_exists is True if the reference is a fixed # reference; if it was assigned a new uuid, we can still # reuse a reference with the same signature if not ref_exists: signature = self.signature debug_print('searching for signature', signature) sig_ref = db_access.search_by_signature(signature) debug_print('sig_ref:', sig_ref) if sig_ref: debug_print('setting persistent_ref') ref.id, ref.version, ref.name = sig_ref self.persistent_ref = ref # else: # ref.id = uuid.uuid1() # copy as normal # don't copy if equal # FIXME also need to check that the file actually exists here! if self.persistent_ref is not None: _, suffix = os.path.splitext(self.persistent_ref.name) self.persistent_path = repo.get_current_repo().get_path( self.persistent_ref.id, self.persistent_ref.version, out_suffix=suffix) debug_print("FOUND persistent path") debug_print(self.persistent_path) debug_print(self.persistent_ref.local_path) if self.persistent_ref is None or self.persistent_path is None: debug_print("NOT FOUND persistent path") super(PersistentPath, self).updateUpstream()
def compute(self, is_input=None, path_type=None): global db_access if not self.has_input('value') and \ not self.has_input('ref'): raise ModuleError(self, "Need to specify path or reference") if self.persistent_path is not None: debug_print('using persistent path') ref = self.persistent_ref path = self.persistent_path elif self.has_input('ref'): ref = self.get_input('ref') if ref.id is None: ref.id = str(uuid.uuid1()) else: # create a new reference ref = PersistentRef() ref.id = str(uuid.uuid1()) if self.has_input('localPath'): ref.local_path = self.get_input('localPath').name if self.has_input('readLocal'): ref.local_read = self.get_input('readLocal') if self.has_input('writeLocal'): ref.local_writeback = self.get_input('writeLocal') if is_input is None: is_input = False if not self.has_input('value'): is_input = True else: if ref.local_path and ref.local_read: debug_print('found local path with local read') is_input = True # FIXME: check if the signature is the signature of # the value if so we know that it's an input... # if just reference, pull path from repository (get latest # version unless specified as specific version) if self.persistent_path is None and not self.has_input('value') \ and is_input and not (ref.local_path and ref.local_read): _, suffix = os.path.splitext(ref.name) if not db_access.ref_exists(ref.id, ref.version): raise ModuleError( self, "Persistent entity '%s' does not " "exist in the local repository." % ref.id) if ref.version is None: ref.version = repo.get_current_repo().get_latest_version( ref.id) # get specific ref.uuid, ref.version combo path = repo.get_current_repo().get_path(ref.id, ref.version, out_suffix=suffix) elif self.persistent_path is None: # copy path to persistent directory with uuid as name if is_input and ref.local_path and ref.local_read: debug_print('using local_path') path = ref.local_path else: path = self.get_input('value').name # this is a static method so we need to add module ourselves try: new_hash = repo.get_current_repo().compute_hash(path) except ModuleError, e: e.module = self raise e rep_path = os.path.join(local_db, ref.id) do_update = True if os.path.exists(rep_path): if os.path.isdir(rep_path): actual_type = 'tree' elif os.path.isfile(rep_path): actual_type = 'blob' else: raise ModuleError( self, "Path is something not a file or " "a directory") if path_type is None: path_type = actual_type else: if path_type != actual_type: def show_type(t): if t == 'tree': return "directory" elif t == 'blob': return "file" else: return '"%s"' % t raise ModuleError( self, "Path is not a %s but a %s" % (show_type(path_type), show_type(actual_type))) old_hash = repo.get_current_repo().get_hash( ref.id, path_type=path_type) debug_print('old_hash:', old_hash) debug_print('new_hash:', new_hash) if old_hash == new_hash: do_update = False if do_update: debug_print('doing update') if path_type == 'tree': if (not os.path.exists(path) or not os.listdir(path)): raise ModuleError(self, "This directory is empty") self.copypath(path, os.path.join(local_db, ref.id)) # commit (and add to) repository # get commit id as version id # persist object-hash, commit-version to repository version = repo.get_current_repo().add_commit(ref.id) ref.version = version # write object-hash, commit-version to provenance if is_input: signature = new_hash else: signature = self.signature db_access.write_database({ 'id': ref.id, 'name': ref.name, 'tags': ref.tags, 'user': current_user(), 'date_created': current_time(), 'date_modified': current_time(), 'content_hash': new_hash, 'version': version, 'signature': signature, 'type': path_type })
def update_upstream(self, is_input=None, path_type=None): global db_access if is_input is None: if not self.has_input('value'): is_input = True else: # FIXME: check if the signature is the signature of # the value if so we know that it's an input... is_input = False self.persistent_ref = None self.persistent_path = None if is_input: return super(PersistentPath, self).update_upstream() # can check update_upstream if not hasattr(self, 'signature'): raise ModuleError(self, 'Module has no signature') ref_exists = False if not self.has_input('ref'): # create new reference with no name or tags ref = PersistentRef() ref.signature = self.signature else: # update single port self.update_upstream_port('ref') ref = self.get_input('ref') if db_access.ref_exists(ref.id, ref.version): ref_exists = True if ref.version is None: ref.version = \ repo.get_current_repo().get_latest_version(ref.id) signature = db_access.get_signature(ref.id, ref.version) if signature == self.signature: # don't need to create a new version self.persistent_ref = ref # Note that ref_exists is True if the reference is a fixed # reference; if it was assigned a new uuid, we can still # reuse a reference with the same signature if not ref_exists: signature = self.signature debug_print('searching for signature', signature) sig_ref = db_access.search_by_signature(signature) debug_print('sig_ref:', sig_ref) if sig_ref: debug_print('setting persistent_ref') ref.id, ref.version, ref.name = sig_ref self.persistent_ref = ref # else: # ref.id = uuid.uuid1() # copy as normal # don't copy if equal # FIXME also need to check that the file actually exists here! if self.persistent_ref is not None: _, suffix = os.path.splitext(self.persistent_ref.name) self.persistent_path = repo.get_current_repo().get_path( self.persistent_ref.id, self.persistent_ref.version, out_suffix=suffix) debug_print("FOUND persistent path") debug_print(self.persistent_path) debug_print(self.persistent_ref.local_path) if self.persistent_ref is None or self.persistent_path is None: debug_print("NOT FOUND persistent path") super(PersistentPath, self).update_upstream()