def __cbTargetGenerateLinkFile(self, result, targetCtx):
# If stopping don't do anything
if self._isStopping(): return
targetCtx.debug("Generating target's link file '%s'",
targetCtx.getLinkWorkPath())
self._setTargetState(targetCtx, TargetStateEnum.link_file_generation)
mimeType = targetCtx.reporter.report.analysis.mimeType
if mimeType != 'application/ogg':
self.warning("Target output not an ogg file, "
"not writing link")
return result
cortadoArgs = varsets.getCortadoArgs(targetCtx)
cortadoArgString = "&".join("%s=%s" % (urllib.quote(str(k)),
urllib.quote(str(v)))
for (k, v) in cortadoArgs.items())
link = targetCtx.getLinkURL(cortadoArgString)
templateVars = varsets.getLinkTemplateVars(targetCtx)
templateVars.update(cortadoArgs)
for k, v in templateVars.items():
templateVars[k] = urllib.quote(str(v))
templateVars["outputURL"] = link
#FIXME: Don't reference the global context
template = self._context.config.profile.linkTemplate % templateVars
workPath = targetCtx.getLinkWorkPath()
fileutils.ensureDirExists(os.path.dirname(workPath),
"temporary target link")
handle = open(workPath, 'w')
handle.write(template)
handle.close()
linkPath = targetCtx.getLinkFromWork(workPath)
targetCtx.reporter.addFile(workPath, linkPath)
return result
def __init__(self, targetContext):
BaseTarget.__init__(self, targetContext)
self._bins = {}
self._pipelineInfo = {}
self._config = targetContext.getTranscodingConfig()
self._outputPath = targetContext.getOutputWorkPath()
fileutils.ensureDirExists(os.path.dirname(self._outputPath), "transcoding output")
def possess(daemonizeTo=None):
fileutils.ensureDirExists(configure.logdir, "log file")
fileutils.ensureDirExists(configure.rundir, "run file")
if not daemonizeTo:
daemonizeTo = '/'
pid = process.getPid('transcoder-admin')
if pid:
if process.checkPidRunning(pid):
raise iherrors.SystemError(
'A flumotion-transcoder-admin is already running '
+ 'as pid %d' % pid)
else:
log.warning("flumotion-transcoder-admin should have been "
"running with pid %s. Restarting", str(pid))
process.deletePidFile('transcoder-admin')
logPath = os.path.join(configure.logdir, 'transcoder-admin.log')
# here we daemonize; so we also change our pid
if not daemonizeTo:
daemonizeTo = '/'
process.daemonize(stdout=logPath, stderr=logPath,
directory=daemonizeTo)
log.info('Started daemon')
# from now on I should keep running, whatever happens
log.debug('writing pid file')
process.writePidFile('transcoder-admin')
def __loadAdminData(self):
self.debug("Loading admin data from '%s'", self._adminPath)
loader = inifile.IniFile()
adminData = dataprops.AdminData()
loader.loadFromFile(adminData, self._adminPath)
self._adminData = adminData
basePath = os.path.dirname(self._adminPath)
relDir = self._adminData.customersDir
absPath = fileutils.makeAbsolute(relDir, basePath)
absDir = fileutils.ensureAbsDirPath(absPath)
self._customersDir = absDir
fileutils.ensureDirExists(self._customersDir, "customers configuration")
self.debug("Loading customers data from directory '%s'", absDir)
self._customersData.clear()
files = os.listdir(absDir)
for f in files:
if f.startswith(".") or not f.endswith('.ini'):
self.log("Ignoring customer data file '%s'", f)
continue
self.log("Loading customer data file '%s'", f)
data = dataprops.CustomerData()
try:
loader.loadFromFile(data, absDir + f)
except Exception, e:
log.notifyException(self, e,
"Fail to load customer data "
"from file '%s'", absDir + f)
continue
identifier = f.rsplit(os.path.extsep, 1)[0]
self._customersData[identifier] = data
def __asyncMove(self, logger, src, dest, pathAttr=None):
logger.log("Moving '%s' to '%s'", src, dest)
fileutils.ensureDirExists(os.path.dirname(dest), "", pathAttr)
if os.path.exists(dest):
logger.debug("Output file '%s' already exists; deleting it", dest)
d = threads.deferToThread(shutil.move, src, dest)
if pathAttr:
d.addCallback(defer.dropResult, pathAttr.apply, dest)
d.addCallback(defer.overrideResult, logger)
return d
def _doProcessing(self):
targCtx = self.getContext()
context = targCtx.context
srcCtx = context.getSourceContext()
sourcePath = srcCtx.getInputPath()
destPath = targCtx.getOutputWorkPath()
fileutils.ensureDirExists(os.path.dirname(destPath), "identity output")
self._outputs.append(destPath)
d = threads.deferToThread(shutil.copy, sourcePath, destPath)
d.addCallback(defer.overrideResult, self)
return d
def __writeReport(self, report):
if self._reportForcedPath:
reportPath = self._reportForcedPath
else:
reportPath = self._reportDefaultPath
#if running in diagnose mode, don't overrite it
if self._diagnoseMode:
reportPath = reportPath + ".diag"
self.debug("Writing report file '%s'", reportPath)
report.status = self._status
fileutils.ensureDirExists(os.path.dirname(reportPath),
"report", self._pathAttr)
saver = IniFile()
saver.saveToFile(report, reportPath)
if self._pathAttr:
self._pathAttr.apply(reportPath)
def prepare(self, workerCtx):
adminCtx = workerCtx.getAdminContext()
adminLocal = adminCtx.getLocal()
localPath = self._configPath.localize(adminLocal)
# The .ini file is created here...
saver = inifile.IniFile()
# Set the datetime of file creation
self._config.touch()
try:
fileutils.ensureDirExists(os.path.dirname(localPath),
"transcoding config", self._pathAttr)
saver.saveToFile(self._config, localPath)
except Exception, e:
message = ("Failed to save transcoder config file '%s': %s"
% (localPath, log.getExceptionMessage(e)))
log.warning("%s", message)
raise admerrs.PropertiesError(message)
def __startThumbnailer(self):
self._startLock.acquire()
try:
if self._working:
return
while True:
if not self._pending:
if self._finalizing:
self._waiters.fireCallbacks(self)
self.__shutdownPipeline()
return
buffer, vars = self._pending.pop(0)
template = self._getOutputPath()
thumbPath = vars.substitute(template)
if thumbPath in self._thumbnails:
self.warning("Thumbnail file '%s' already created, " "keeping the first one", thumbPath)
continue
fileutils.ensureDirExists(os.path.dirname(thumbPath), "thumbnails")
self.__startupPipeline(buffer, thumbPath)
return
finally:
self._startLock.release()
def __loadActivityData(self):
basePath = os.path.dirname(self._adminPath)
relDir = self._adminData.activitiesDir
absPath = fileutils.makeAbsolute(relDir, basePath)
absDir = fileutils.ensureAbsDirPath(absPath)
self._activeActivitiesDir = absDir
self._failedActivitiesDir = fileutils.ensureAbsDirPath(absDir + "failed")
self._doneActivitiesDir = fileutils.ensureAbsDirPath(absDir + "done")
self._invalidActivitiesDir = fileutils.ensureAbsDirPath(absDir + "invalid")
fileutils.ensureDirExists(self._activeActivitiesDir,
"activities data base")
fileutils.ensureDirExists(self._failedActivitiesDir,
"failed activities")
fileutils.ensureDirExists(self._doneActivitiesDir,
"done activities")
fileutils.ensureDirExists(self._invalidActivitiesDir,
"invalid activities")
self.debug("Loading activities data from directory '%s'", absDir)
loader = inifile.IniFile()
self._activitiesData.clear()
files = os.listdir(absDir)
for f in files:
if not f.endswith('.ini'):
self.log("Ignoring activity data file '%s'", f)
continue
if f.startswith("transcoding-"):
data = dataprops.TranscodingActivityData()
elif f.startswith("notification-"):
data = dataprops.NotificationActivityData()
else:
self.log("Ignoring activity data file '%s'", f)
continue
self.log("Loading activity data file '%s'", f)
try:
loader.loadFromFile(data, absDir + f)
except Exception, e:
log.notifyException(self, e,
"Fail to load activity data "
"from file '%s'", absDir + f)
self.__safeMove(absDir, self._invalidActivitiesDir, f)
continue
if data.state == ActivityStateEnum.done:
self.__safeMove(absDir, self._doneActivitiesDir, f)
elif data.state == ActivityStateEnum.failed:
self.__safeMove(absDir, self._failedActivitiesDir, f)
else:
self._activitiesData[f] = (absDir + f, data)
def _store(self):
# WARNING: This function is not generic, it's specific for activities
# PyChecker doesn't like dynamic attributes
__pychecker__ = "no-classattr"
identifier = self._identifier or self.__newIdentifier()
data = self._data or self._template['class']()
newPath = self.__getPath(identifier, self.state)
newTmpPath = newPath + ".tmp"
for attr, value in self._fields.items():
if isinstance(value, list):
l = getattr(data, attr)
del l[:]
for v in value:
l.append(utils.deepCopy(v))
elif isinstance(value, dict):
d = getattr(data, attr)
d.clear()
for k, v in value.items():
d[k] = utils.deepCopy(v)
else:
setattr(data, attr, utils.deepCopy(value))
try:
fileutils.ensureDirExists(os.path.dirname(newTmpPath),
"activities")
saver = inifile.IniFile()
# The activities can contains sensitive information (passwords)
# so we don't want it to be readable by others
oldmask = os.umask(0137)
try:
saver.saveToFile(data, newTmpPath)
finally:
os.umask(oldmask)
except Exception, e:
log.notifyException(self._source, e,
"File to save activity data file '%s'",
newPath)
raise e
