def __cbTargetGenerateLinkFile(self, result, targetCtx): # If stopping don't do anything if self._isStopping(): return targetCtx.debug("Generating target's link file '%s'", targetCtx.getLinkWorkPath()) self._setTargetState(targetCtx, TargetStateEnum.link_file_generation) mimeType = targetCtx.reporter.report.analysis.mimeType if mimeType != 'application/ogg': self.warning("Target output not an ogg file, " "not writing link") return result cortadoArgs = varsets.getCortadoArgs(targetCtx) cortadoArgString = "&".join("%s=%s" % (urllib.quote(str(k)), urllib.quote(str(v))) for (k, v) in cortadoArgs.items()) link = targetCtx.getLinkURL(cortadoArgString) templateVars = varsets.getLinkTemplateVars(targetCtx) templateVars.update(cortadoArgs) for k, v in templateVars.items(): templateVars[k] = urllib.quote(str(v)) templateVars["outputURL"] = link #FIXME: Don't reference the global context template = self._context.config.profile.linkTemplate % templateVars workPath = targetCtx.getLinkWorkPath() fileutils.ensureDirExists(os.path.dirname(workPath), "temporary target link") handle = open(workPath, 'w') handle.write(template) handle.close() linkPath = targetCtx.getLinkFromWork(workPath) targetCtx.reporter.addFile(workPath, linkPath) return result
def __init__(self, targetContext): BaseTarget.__init__(self, targetContext) self._bins = {} self._pipelineInfo = {} self._config = targetContext.getTranscodingConfig() self._outputPath = targetContext.getOutputWorkPath() fileutils.ensureDirExists(os.path.dirname(self._outputPath), "transcoding output")
def possess(daemonizeTo=None): fileutils.ensureDirExists(configure.logdir, "log file") fileutils.ensureDirExists(configure.rundir, "run file") if not daemonizeTo: daemonizeTo = '/' pid = process.getPid('transcoder-admin') if pid: if process.checkPidRunning(pid): raise iherrors.SystemError( 'A flumotion-transcoder-admin is already running ' + 'as pid %d' % pid) else: log.warning("flumotion-transcoder-admin should have been " "running with pid %s. Restarting", str(pid)) process.deletePidFile('transcoder-admin') logPath = os.path.join(configure.logdir, 'transcoder-admin.log') # here we daemonize; so we also change our pid if not daemonizeTo: daemonizeTo = '/' process.daemonize(stdout=logPath, stderr=logPath, directory=daemonizeTo) log.info('Started daemon') # from now on I should keep running, whatever happens log.debug('writing pid file') process.writePidFile('transcoder-admin')
def __loadAdminData(self): self.debug("Loading admin data from '%s'", self._adminPath) loader = inifile.IniFile() adminData = dataprops.AdminData() loader.loadFromFile(adminData, self._adminPath) self._adminData = adminData basePath = os.path.dirname(self._adminPath) relDir = self._adminData.customersDir absPath = fileutils.makeAbsolute(relDir, basePath) absDir = fileutils.ensureAbsDirPath(absPath) self._customersDir = absDir fileutils.ensureDirExists(self._customersDir, "customers configuration") self.debug("Loading customers data from directory '%s'", absDir) self._customersData.clear() files = os.listdir(absDir) for f in files: if f.startswith(".") or not f.endswith('.ini'): self.log("Ignoring customer data file '%s'", f) continue self.log("Loading customer data file '%s'", f) data = dataprops.CustomerData() try: loader.loadFromFile(data, absDir + f) except Exception, e: log.notifyException(self, e, "Fail to load customer data " "from file '%s'", absDir + f) continue identifier = f.rsplit(os.path.extsep, 1)[0] self._customersData[identifier] = data
def __asyncMove(self, logger, src, dest, pathAttr=None): logger.log("Moving '%s' to '%s'", src, dest) fileutils.ensureDirExists(os.path.dirname(dest), "", pathAttr) if os.path.exists(dest): logger.debug("Output file '%s' already exists; deleting it", dest) d = threads.deferToThread(shutil.move, src, dest) if pathAttr: d.addCallback(defer.dropResult, pathAttr.apply, dest) d.addCallback(defer.overrideResult, logger) return d
def _doProcessing(self): targCtx = self.getContext() context = targCtx.context srcCtx = context.getSourceContext() sourcePath = srcCtx.getInputPath() destPath = targCtx.getOutputWorkPath() fileutils.ensureDirExists(os.path.dirname(destPath), "identity output") self._outputs.append(destPath) d = threads.deferToThread(shutil.copy, sourcePath, destPath) d.addCallback(defer.overrideResult, self) return d
def __writeReport(self, report): if self._reportForcedPath: reportPath = self._reportForcedPath else: reportPath = self._reportDefaultPath #if running in diagnose mode, don't overrite it if self._diagnoseMode: reportPath = reportPath + ".diag" self.debug("Writing report file '%s'", reportPath) report.status = self._status fileutils.ensureDirExists(os.path.dirname(reportPath), "report", self._pathAttr) saver = IniFile() saver.saveToFile(report, reportPath) if self._pathAttr: self._pathAttr.apply(reportPath)
def prepare(self, workerCtx): adminCtx = workerCtx.getAdminContext() adminLocal = adminCtx.getLocal() localPath = self._configPath.localize(adminLocal) # The .ini file is created here... saver = inifile.IniFile() # Set the datetime of file creation self._config.touch() try: fileutils.ensureDirExists(os.path.dirname(localPath), "transcoding config", self._pathAttr) saver.saveToFile(self._config, localPath) except Exception, e: message = ("Failed to save transcoder config file '%s': %s" % (localPath, log.getExceptionMessage(e))) log.warning("%s", message) raise admerrs.PropertiesError(message)
def __startThumbnailer(self): self._startLock.acquire() try: if self._working: return while True: if not self._pending: if self._finalizing: self._waiters.fireCallbacks(self) self.__shutdownPipeline() return buffer, vars = self._pending.pop(0) template = self._getOutputPath() thumbPath = vars.substitute(template) if thumbPath in self._thumbnails: self.warning("Thumbnail file '%s' already created, " "keeping the first one", thumbPath) continue fileutils.ensureDirExists(os.path.dirname(thumbPath), "thumbnails") self.__startupPipeline(buffer, thumbPath) return finally: self._startLock.release()
def __loadActivityData(self): basePath = os.path.dirname(self._adminPath) relDir = self._adminData.activitiesDir absPath = fileutils.makeAbsolute(relDir, basePath) absDir = fileutils.ensureAbsDirPath(absPath) self._activeActivitiesDir = absDir self._failedActivitiesDir = fileutils.ensureAbsDirPath(absDir + "failed") self._doneActivitiesDir = fileutils.ensureAbsDirPath(absDir + "done") self._invalidActivitiesDir = fileutils.ensureAbsDirPath(absDir + "invalid") fileutils.ensureDirExists(self._activeActivitiesDir, "activities data base") fileutils.ensureDirExists(self._failedActivitiesDir, "failed activities") fileutils.ensureDirExists(self._doneActivitiesDir, "done activities") fileutils.ensureDirExists(self._invalidActivitiesDir, "invalid activities") self.debug("Loading activities data from directory '%s'", absDir) loader = inifile.IniFile() self._activitiesData.clear() files = os.listdir(absDir) for f in files: if not f.endswith('.ini'): self.log("Ignoring activity data file '%s'", f) continue if f.startswith("transcoding-"): data = dataprops.TranscodingActivityData() elif f.startswith("notification-"): data = dataprops.NotificationActivityData() else: self.log("Ignoring activity data file '%s'", f) continue self.log("Loading activity data file '%s'", f) try: loader.loadFromFile(data, absDir + f) except Exception, e: log.notifyException(self, e, "Fail to load activity data " "from file '%s'", absDir + f) self.__safeMove(absDir, self._invalidActivitiesDir, f) continue if data.state == ActivityStateEnum.done: self.__safeMove(absDir, self._doneActivitiesDir, f) elif data.state == ActivityStateEnum.failed: self.__safeMove(absDir, self._failedActivitiesDir, f) else: self._activitiesData[f] = (absDir + f, data)
def _store(self): # WARNING: This function is not generic, it's specific for activities # PyChecker doesn't like dynamic attributes __pychecker__ = "no-classattr" identifier = self._identifier or self.__newIdentifier() data = self._data or self._template['class']() newPath = self.__getPath(identifier, self.state) newTmpPath = newPath + ".tmp" for attr, value in self._fields.items(): if isinstance(value, list): l = getattr(data, attr) del l[:] for v in value: l.append(utils.deepCopy(v)) elif isinstance(value, dict): d = getattr(data, attr) d.clear() for k, v in value.items(): d[k] = utils.deepCopy(v) else: setattr(data, attr, utils.deepCopy(value)) try: fileutils.ensureDirExists(os.path.dirname(newTmpPath), "activities") saver = inifile.IniFile() # The activities can contains sensitive information (passwords) # so we don't want it to be readable by others oldmask = os.umask(0137) try: saver.saveToFile(data, newTmpPath) finally: os.umask(oldmask) except Exception, e: log.notifyException(self._source, e, "File to save activity data file '%s'", newPath) raise e