def _save(self, storPath): origTaskDir = os.path.join(storPath, self.id) if not getProcPool().os.path.exists(origTaskDir): raise se.TaskDirError("_save: no such task dir '%s'" % origTaskDir) taskDir = os.path.join(storPath, self.id + TEMP_EXT) self.log.debug("_save: orig %s temp %s", origTaskDir, taskDir) if getProcPool().os.path.exists(taskDir): getProcPool().fileUtils.cleanupdir(taskDir) getProcPool().os.mkdir(taskDir) try: self.njobs = len(self.jobs) self.nrecoveries = len(self.recoveries) self._saveTaskMetaFile(taskDir) if self.state == State.finished: self._saveTaskResultMetaFile(taskDir) for jn in range(self.njobs): self._saveJobMetaFile(taskDir, jn) for rn in range(self.nrecoveries): self._saveRecoveryMetaFile(taskDir, rn) except Exception as e: self.log.error("Unexpected error", exc_info=True) try: getProcPool().fileUtils.cleanupdir(taskDir) except: self.log.warning("can't remove temp taskdir %s" % taskDir) raise se.TaskPersistError("%s persist failed: %s" % (self, e)) # Make sure backup dir doesn't exist getProcPool().fileUtils.cleanupdir(origTaskDir + BACKUP_EXT) getProcPool().os.rename(origTaskDir, origTaskDir + BACKUP_EXT) getProcPool().os.rename(taskDir, origTaskDir) getProcPool().fileUtils.cleanupdir(origTaskDir + BACKUP_EXT) getProcPool().fileUtils.fsyncPath(origTaskDir)
def persist(self): if self.persistPolicy == TaskPersistType.none: return if not self.store: raise se.TaskPersistError("no store defined") if self.state == State.init: raise se.TaskStateError("can't persist in state %s" % self.state) self._save(self.store)
def setPersistence(self, store, persistPolicy=TaskPersistType.auto, cleanPolicy=TaskCleanType.auto): self.persistPolicy = TaskPersistType(persistPolicy) self.store = store self.setCleanPolicy(cleanPolicy) if self.persistPolicy != TaskPersistType.none and not self.store: raise se.TaskPersistError("no store defined") taskDir = os.path.join(self.store, self.id) try: getProcPool().fileUtils.createdir(taskDir) except Exception as e: self.log.error("Unexpected error", exc_info=True) raise se.TaskPersistError("%s: cannot access/create taskdir" " %s: %s" % (self, taskDir, e)) if (self.persistPolicy == TaskPersistType.auto and self.state != State.init): self.persist()