def _load(self, storPath, ext=""): self.log.debug("%s: load from %s, ext '%s'", self, storPath, ext) if self.state != State.init: raise se.TaskMetaDataLoadError("task %s - can't load self: " "not in init state" % self) taskDir = os.path.join(storPath, str(self.id) + str(ext)) if not getProcPool().os.path.exists(taskDir): raise se.TaskDirError("load: no such task dir '%s'" % taskDir) oldid = self.id self._loadTaskMetaFile(taskDir) if self.id != oldid: raise se.TaskMetaDataLoadError("task %s: loaded file do not match" " id (%s != %s)" % (self, self.id, oldid)) if self.state == State.finished: self._loadTaskResultMetaFile(taskDir) for jn in range(self.njobs): self.jobs.append(Job("load", None)) self._loadJobMetaFile(taskDir, jn) self.jobs[jn].setOwnerTask(self) for rn in range(self.nrecoveries): self.recoveries.append(Recovery("load", "load", "load", "load", "")) self._loadRecoveryMetaFile(taskDir, rn) self.recoveries[rn].setOwnerTask(self)
def _save(self, storPath): origTaskDir = os.path.join(storPath, self.id) if not getProcPool().os.path.exists(origTaskDir): raise se.TaskDirError("_save: no such task dir '%s'" % origTaskDir) taskDir = os.path.join(storPath, self.id + TEMP_EXT) self.log.debug("_save: orig %s temp %s", origTaskDir, taskDir) if getProcPool().os.path.exists(taskDir): getProcPool().fileUtils.cleanupdir(taskDir) getProcPool().os.mkdir(taskDir) try: self.njobs = len(self.jobs) self.nrecoveries = len(self.recoveries) self._saveTaskMetaFile(taskDir) if self.state == State.finished: self._saveTaskResultMetaFile(taskDir) for jn in range(self.njobs): self._saveJobMetaFile(taskDir, jn) for rn in range(self.nrecoveries): self._saveRecoveryMetaFile(taskDir, rn) except Exception as e: self.log.error("Unexpected error", exc_info=True) try: getProcPool().fileUtils.cleanupdir(taskDir) except: self.log.warning("can't remove temp taskdir %s" % taskDir) raise se.TaskPersistError("%s persist failed: %s" % (self, e)) # Make sure backup dir doesn't exist getProcPool().fileUtils.cleanupdir(origTaskDir + BACKUP_EXT) getProcPool().os.rename(origTaskDir, origTaskDir + BACKUP_EXT) getProcPool().os.rename(taskDir, origTaskDir) getProcPool().fileUtils.cleanupdir(origTaskDir + BACKUP_EXT) getProcPool().fileUtils.fsyncPath(origTaskDir)
def loadTask(cls, store, taskid): t = Task(taskid) if getProcPool().os.path.exists(os.path.join(store, taskid)): ext = "" # TBD: is this the correct order (temp < backup) + should temp # be considered at all? elif getProcPool().os.path.exists( os.path.join(store, taskid + TEMP_EXT)): ext = TEMP_EXT elif getProcPool().os.path.exists( os.path.join(store, taskid + BACKUP_EXT)): ext = BACKUP_EXT else: raise se.TaskDirError("loadTask: no such task dir '%s/%s'" % (store, taskid)) t._load(store, ext) return t