def remediate_audio(self, path):
# Create our Input() object to specify file we will be transcoding
input_file = Input(path)
# Create variables we will be using in the following code
codec = None
output_path = re.match('^(.*)\.audio\.\d\.mkv$', path).group(1) + '.final.audio.' + re.match('^.*\.audio\.(\d)\.mkv$', path).group(1) + '.mkv'
output_file = Output(output_path)
# Check if self.finalfiles is not none and if so create a list object. This is so we don't lose files already in the list
if self.finalfiles is None:
self.finalfiles = []
# Create our AudioCodec() object to define the output codec and bitrate
codec = AudioCodec('ac3')
codec = codec.bitrate('448k')
# Instance our FFmpeg() object to begin processing
remediate_audio_job = FFmpeg('/usr/bin/ffmpeg', input_file, codec, output_file)
# Begin the transcoding operation iterating over the piped stdout from the command
with remediate_audio_job as self._process:
while not self.abort and self._process.running:
for line in self._process.readlines():
if mediamanager.VERBOSE:
logger.log(line, logger.DEBUG)
if self._process.failed:
return False
elif self._process.successful:
self.finalfiles.append(output_path)
return True
else:
return False
def ScanForMedia(cls):
if mediamanager.SCANNER_VERBOSE:
logger.log(u'ScanForMedia :: Starting', logger.DEBUG)
_files = []
_media = {}
for _source in mediamanager.SOURCES:
if mediamanager.SCANNER_VERBOSE:
logger.log(u'ScanForMedia :: Getting files from source ' + _source, logger.DEBUG)
if re.match('^(~).*$', _source) is not None:
_files.extend(Files.getFilesFromDir(os.path.join(os.path.expanduser("~"), re.match('^~\/(.*)$', _source).group(1))))
else:
_files.extend(Files.getFilesFromDir(_source))
if mediamanager.SCANNER_VERBOSE:
logger.log(u'ScanForMedia :: Found %d files' % len(_files), logger.DEBUG)
for item in _files:
if isinstance(item, Files.VideoInfo):
match = re.match('^(.*)\..*$',item.basename)
if match is not None:
_media[match.group(1)] = item
if mediamanager.SCANNER_VERBOSE:
logger.log(u'ScanForMedia :: Found %d video files' % len(_media.keys()), logger.DEBUG)
logger.log(u'ScanForMedia :: Completed', logger.DEBUG)
return _media
def shutdown(): halt() logger.log(u'Exiting MediaManager') os._exit(0)
def execute(self):
self.amActive = True
try:
self.job.run()
except Exception, e:
logger.log(u"QueueTranscode :: Exception caught running job; %s" % e, logger.DEBUG)
def run(self):
# only start a new task if one isn't already going
if self.thread == None or self.thread.isAlive() == False:
# if the thread is dead then the current item should be finished
if self.currentItem != None:
self.currentItem.finish()
self.currentItem = None
# if there's something in the queue then run it in a thread and take it out of the queue
if len(self.queues[QueuePriorities.HIGH]) > 0 or len(self.queues[QueuePriorities.NORMAL]) > 0 or len(self.queues[QueuePriorities.LOW]) > 0:
# sort by priority
queueItem = self.pop_queueitem()
if queueItem.priority == QueuePriorities.NORMAL and media.is_pending(queueItem.job._file):
logger.log("Queue_Transcode :: Item is already pending verification. Will remove from queue and continue; %s" % queueItem.name, logger.DEBUG)
queueItem = None
elif queueItem.priority == QueuePriorities.HIGH and not media.is_pending(queueItem.job._file):
logger.log("Queue_Transcode :: Item is no longer pending verification. Will remove from queue and continue; %s" % queueItem.name, logger.DEBUG)
queueItem = None
else:
# launch the queue item in a thread
# TODO: improve thread name
threadName = self.queue_name + '-' + queueItem.get_thread_name()
self.thread = threading.Thread(None, queueItem.execute, threadName)
self.thread.start()
self.currentItem = queueItem
def add_item(self, item):
if not self.is_in_queue(item):
with self.queue_lock:
item.added = datetime.datetime.now()
self.queues[item.priority].append(item)
return True
else:
if mediamanager.VERBOSE:
logger.log(u"Not adding item, it's already in the queue", logger.DEBUG)
return False
def finalize(self):
# Create some variables to contain all the paths and basenames
self._pending_file_tempbasename = self._outfile_base + '.' + 'copytemp'
self._pending_file_temppath = os.path.join(os.path.dirname(self._file.path), self._pending_file_tempbasename)
self._pending_file_basename = self._outfile_base + '.' + mediamanager.TRANSCODER_PENDING_EXTENSION
self._pending_file_path = os.path.join(os.path.dirname(self._file.path), self._pending_file_basename)
# First copy file to final destination directory with a temporary name
try:
shutil.copyfile(self._muxed_file_path, self._pending_file_temppath)
except IOError, e:
logger.log("finalize() :: Exception raised during copy to temp location, not writable; %s" % self._file.path, logger.DEBUG)
return False
def is_compliant_audio(fileInfo, stream_id=None):
if stream_id == None:
for stream in fileInfo.audioStreams:
if (re.match(re.compile('^.*(' + mediamanager.COMPLIANCE['Media']['Audio']['Codec'] + ').*$', re.I), stream['codec']) is None):
return False
return True
else:
try:
if (re.match(re.compile('^.*(' + mediamanager.COMPLIANCE['Media']['Audio']['Codec'] + ').*$', re.I), fileInfo.audioStreams[stream_id]['codec']) is None):
return False
else:
return True
except Exception, e:
logger.log(u"is_compliant_audio :: Exception caught; %s" % e, logger.DEBUG)
return False
def stop(self):
logger.log("%s :: Stop has been called" % self.queue_name, logger.DEBUG)
try:
if self.thread.isAlive():
self.currentItem.finish()
logger.log("%s :: Waiting for queue thread to stop" % self.queue_name, logger.DEBUG)
self.thread.join(10)
self.thread = None
else:
logger.log("%s :: Thread is not running" % self.queue_name, logger.DEBUG)
except Exception, e:
logger.log("%s :: Exception was caught; %s" % (self.queue_name, e), logger.DEBUG)
def runAction(self):
while True:
current_time = datetime.datetime.now()
should_run = False
# check if interval has passed
if (len(self.action.queues[queue_generic.QueuePriorities.HIGH])) > 0:
should_run = True
elif (len(self.action.queues[queue_generic.QueuePriorities.NORMAL]) > 0 and len(media.PENDING.keys()) <= mediamanager.TRANSCODER_PENDING_LIMIT):
should_run = True
elif (len(self.action.queues[queue_generic.QueuePriorities.LOW]) > 0 and len(media.PENDING.keys()) <= mediamanager.TRANSCODER_PENDING_LIMIT):
should_run = True
if should_run:
self.lastRun = current_time
try:
if logger.LOG_EXTRA_DEBUG:
logger.log(u"Starting new thread: " + self.threadName, logger.DEBUG)
self.action.run()
except Exception, e:
logger.log(u"Exception generated in thread " + self.threadName, logger.ERROR)
logger.log(repr(traceback.format_exc()), logger.DEBUG)
self.abort = True
if self.abort:
self.abort = False
try:
self.action.stop()
except:
pass
finally:
self.thread = None
return
time.sleep(1)
def remux_files(self):
# Create our output file object for ffmpegwrapper
if mediamanager.COMPLIANCE['Media']['Container']['Codec'] == 'Matroska':
self._muxed_file_basename = self._outfile_base + '.' + 'mkv'
elif mediamanager.COMPLIANCE['Media']['Container']['Codec'] == 'MPEG-4':
self._muxed_file_basename = self._outfile_base + '.' + 'mp4'
self._muxed_file_path = os.path.join(mediamanager.TRANSCODER_TEMPDIR, self._muxed_file_basename)
output_file = Output(self._muxed_file_path)
# Create our FFmpeg job object with path to ffmpeg binary
remux_job = FFmpeg('/usr/bin/ffmpeg')
# Iterate for self.finalfiles and create Input() objects for input files and insert at the end of the list
for _file in self.finalfiles:
input_file = Input(_file)
remux_job.insert(len(remux_job), input_file)
# Add video codec object with 'copy' as the codec specified so it just muxes the streams
remux_job.insert(len(remux_job), VideoCodec('copy'))
# Add audio codec object with 'copy' as the codec specified so it just muxes the streams
remux_job.insert(len(remux_job), AudioCodec('copy'))
# Add Output() object to specify the output path and container
remux_job.insert(len(remux_job), output_file)
# Begin remux job and iterate over the piped stdout from the command logging it to debug
with remux_job as self._process:
while not self.abort and self._process.running:
for line in self._process.readlines():
if mediamanager.VERBOSE:
logger.log(line, logger.DEBUG)
if self._process.failed:
return False
elif self._process.successful:
return True
else:
return False
def finish(self):
# Lock the finish operation so only one thread can cleanup at a time
with self.__FINISH_LOCK__:
# If another thread cleaned up while waiting for the lock than just return
if self.__FINISHED__:
return
# Save the current self.abort value so we can late determine if finish() was called because of an abort
wasaborted = self.abort
logger.log("TranscodeJob :: Finish has been called", logger.DEBUG)
try:
self.abort = True
# Check if self.process exists and running
if self._process is not None and self._process.poll() is None:
# Kill the popen process
self._process.terminate()
# Release the reference to the process
self._process = None
except Exception, e:
logger.log("TranscodeJob :: Exception caught in finish(); %s" % e, logger.DEBUG)
# Delete all contents from tempdir since we are either done or aborted
filelist = os.listdir(mediamanager.TRANSCODER_TEMPDIR)
for f in filelist:
os.remove(os.path.join(mediamanager.TRANSCODER_TEMPDIR ,f))
# If finish() was called and being aborted during operation than check if a pending file was created and delete it
outfile_basename = self._outfile_base + '.' + mediamanager.TRANSCODER_PENDING_EXTENSION
if wasaborted and os.path.isfile(os.path.join(os.path.dirname(self._file.path), outfile_basename)):
os.remove(os.path.join(os.path.dirname(self._file.path), outfile_basename))
self.__FINISHED__ = True
def demux_file(self):
# Return if we couldn't get the base file name without extension from the self._file.basename
if not self._outfile_base:
logger.log("Unable to get base file name; %s" % self._file.path, logger.DEBUG)
return False
# Create our Input() object to specify the file we will be demuxing
input_file = Input(self._file.path)
# Create a list of the codecs with maps or input stream to output file
codecs = []
# Create a list to contain the paths of all the output files
self.demuxfiles = []
# Iterate over self._file.videoStreams and create codecs for each video stream with 'copy' specified so no transcoding is done
for i, stream in enumerate(self._file.videoStreams):
stream_id = int(stream['track_id']) - 1
outfile_path = os.path.join(mediamanager.TRANSCODER_TEMPDIR, self._outfile_base + '.video.' + str(stream_id) + '.mkv')
video_codec = VideoCodec('copy')
video_codec = video_codec.mapstream("0:%d" % stream_id, outfile_path)
self.demuxfiles.append(outfile_path)
codecs.append(video_codec)
# Iterate over self._file.audioStreams and create codecs for each audio stream with 'copy' specified so no transcoding is done
for i, stream in enumerate(self._file.audioStreams):
stream_id = int(stream['track_id']) - 1
outfile_path = os.path.join(mediamanager.TRANSCODER_TEMPDIR, self._outfile_base + '.audio.' + str(stream_id) + '.mkv')
audio_codec = AudioCodec('copy')
audio_codec = audio_codec.mapstream("0:%d" % stream_id, outfile_path)
self.demuxfiles.append(outfile_path)
codecs.append(audio_codec)
# Create FFmpeg() object to begin demux operation
demux_job = FFmpeg('/usr/bin/ffmpeg', input_file)
# Iterate over items in codecs list and insert them at the end of the FFmpeg() object's list
for codec in codecs:
demux_job.insert(len(demux_job), codec)
# Begin demux operation and iterate over piped stdout from the popen process
logger.log("demux_job: %s" % demux_job, logger.DEBUG)
with demux_job as self._process:
while not self.abort and self._process.running:
for line in self._process.readlines():
if mediamanager.VERBOSE:
logger.log(line, logger.DEBUG)
if self._process.failed:
return False
elif self._process.successful:
return True
else:
return False
def cleanPending():
with PENDING_LOCK:
_files = []
pattern = re.compile('^.*\.(' + mediamanager.TRANSCODER_PENDING_EXTENSION + ')$', re.I)
for _source in mediamanager.SOURCES:
if re.match('^(~).*$', _source) is not None:
_files.extend(Files.getFilesFromDir(os.path.join(os.path.expanduser("~"), re.match('^~\/(.*)$', _source).group(1)), pattern))
else:
_files.extend(Files.getFilesFromDir(_source, pattern))
for _file in _files:
try:
logger.log("CleanPending :: Removing pending file; %s" % _file.path, logger.DEBUG)
os.remove(_file.path)
except Exception, e:
logger.log("CleanPending :: Exception raised cleaing up pending file; %s" % _file.path, logger.DEBUG)
logger.log("CleanPending :: Exception :: %s" % e, logger.DEBUG)
def halt():
global __INITIALIZED__, schedulerScanner
with INIT_LOCK:
if __INITIALIZED__:
logger.log(u'Aborting all threads')
schedulerScanner.abort = True
logger.log(u'Waiting for the SCANNER thread to exit')
try:
schedulerScanner.thread.join(10)
except:
pass
schedulerTranscode.abort = True
logger.log(u'Waiting for the TRANSCODER thread to exit')
try:
schedulerTranscode.thread.join(10)
except:
pass
__INITIALIZED__ = False
def runAction(self):
while True:
current_time = datetime.datetime.now()
should_run = False
# check if interval has passed
if current_time - self.lastRun >= self.cycleTime:
# check if wanting to start around certain time taking interval into account
if self.start_time:
hour_diff = current_time.time().hour - self.start_time.hour
if hour_diff >= 0 and hour_diff < self.cycleTime.seconds / 3600:
should_run = True
else:
# set lastRun to only check start_time after another cycleTime
self.lastRun = current_time
else:
should_run = True
if should_run:
self.lastRun = current_time
try:
if logger.LOG_EXTRA_DEBUG:
logger.log(u"Starting new thread: " + self.threadName, logger.DEBUG)
self.action.run()
except Exception, e:
logger.log(u"Exception generated in thread " + self.threadName, logger.ERROR)
logger.log(repr(traceback.format_exc()), logger.DEBUG)
if self.abort:
self.abort = False
self.thread = None
return
time.sleep(1)
def pause(self): logger.log(u"Pausing queue") self.min_priority = 999999999999
def finish(self): logger.log(u"QueueTranscode :: Finish has been called; %s" % self.thread_name, logger.DEBUG) self.job.finish() self.amActive = False
def run(self):
global MEDIA, NONCOMPLIANT
if self.amActive == True:
if mediamanager.SCANNER_VERBOSE:
logger.log(u'Scanner is still running, not starting it again', logger.MESSAGE)
return
if mediamanager.SCANNER_VERBOSE:
logger.log(u'Scanner has started', logger.MESSAGE)
self.amActive = True
if mediamanager.SCANNER_VERBOSE:
logger.log(u'Scanner is starting ScanForMedia()', logger.DEBUG)
MEDIA = Scan.ScanForMedia()
if mediamanager.SCANNER_VERBOSE:
logger.log(u'Scanner has completed ScanForMedia()', logger.DEBUG)
logger.log(u'Scanner is starting updatePending()', logger.DEBUG)
if updatePending():
if mediamanager.SCANNER_VERBOSE:
logger.log(u'Scanner has completed updatePending()', logger.DEBUG)
else:
if mediamanager.SCANNER_VERBOSE:
logger.log(u'Scanner is skipping updatePending() due to insufficient time lapse since last update', logger.DEBUG)
if mediamanager.SCANNER_VERBOSE:
logger.log(u'Scanner is queing items for transcode', logger.DEBUG)
NONCOMPLIANT = Scan.ScanForCompliance(MEDIA.values())
transcode_files = []
for _file in NONCOMPLIANT:
if not re.match('^(.*)\..*$', _file.basename).group(1) in PENDING.keys():
transcode_files.append(_file)
for _file in transcode_files:
queueItem = None
job = transcode.Factory.TranscodeJobFactory(_file)
if job is not None:
queueItem = mediamanager.queue_transcode.QueueItemTranscode(job)
else:
if mediamanager.SCANNER_VERBOSE:
logger.log(u"Scanner TranscodeJob is None; %s" % _file.path, logger.DEBUG)
if queueItem is not None and mediamanager.schedulerTranscode.action.add_item(queueItem):
if mediamanager.SCANNER_VERBOSE:
logger.log(u'Scanner has queued item for transcode; %s' % _file.path, logger.DEBUG)
else:
if queueItem is None:
if mediamanager.SCANNER_VERBOSE:
logger.log(u'Scanner unable to queue item, queueItem is None; %s' % _file.path, logger.DEBUG)
if mediamanager.SCANNER_VERBOSE:
logger.log(u'Scanner has completed', logger.MESSAGE)
self.amActive = False
def unpause(self): logger.log(u"Unpausing queue") self.min_priority = 0
return False
except Exception, e:
logger.log("finalize() :: Exception raised during copy to temp location, unknown; %s" % self._file.path, logger.DEBUG)
logger.log("finalize() :: Exception :: %s" % e, logger.DEBUG)
try:
if os.path.isfile(self._pending_file_temppath):
os.remove(self._pending_file_temppath)
except Exception, e:
logger.log("finalize() :: Exception raised cleaing up partially copied file; %s" % self._file.path, logger.DEBUG)
return False
# Second do a move operation from temp basename to final basename
try:
shutil.move(self._pending_file_temppath, self._pending_file_path)
except Exception, e:
logger.log("finalize() :: Exception raised during move to final location, unknown; %s" % self._file.path, logger.DEBUG)
logger.log("finalize() :: Exception :: %s" % e, logger.DEBUG)
try:
if os.path.isfile(self._pending_file_temppath):
os.remove(self._pending_file_temppath)
except Exception, e:
logger.log("finalize() :: Exception raised cleaing up partially copied file; %s" % self._file.path, logger.DEBUG)
try:
if os.path.isfile(self._pending_file_path):
os.remove(self._pending_file_path)
except Exception, e:
logger.log("finalize() :: Exception raised cleaing up partially moved file; %s" % self._file.path, logger.DEBUG)
return False
return True
def ScanForCompliance(cls, files):
retval = []
if mediamanager.SCANNER_VERBOSE:
logger.log(u"ScanForCompliance :: Starting", logger.DEBUG)
for fileInfo in files:
if isinstance(fileInfo, Files.VideoInfo):
compliant = True
if is_compliant_container(fileInfo):
if mediamanager.SCANNER_VERBOSE:
logger.log(u"ScanForCompliance :: File has compliant container; %s" % fileInfo.path, logger.DEBUG)
else:
if mediamanager.SCANNER_VERBOSE:
logger.log(u"ScanForCompliance :: File has non-compliant container %s; %s" % (fileInfo.container, fileInfo.path))
compliant = False
for i, stream in enumerate(fileInfo.videoStreams):
if is_compliant_video(fileInfo, stream_id=i):
if mediamanager.SCANNER_VERBOSE:
logger.log(u"ScanForCompliance :: File has compliant video stream at ID %s; %s" % (i, fileInfo.path), logger.DEBUG)
else:
if mediamanager.SCANNER_VERBOSE:
logger.log(u"ScanForCompliance :: File has non-compliant video stream at ID %s of codec %s; %s" % (i, stream['codec'], fileInfo.path), logger.DEBUG)
compliant = False
for i, stream in enumerate(fileInfo.audioStreams):
if is_compliant_audio(fileInfo, stream_id=i):
if mediamanager.SCANNER_VERBOSE:
logger.log(u"ScanForCompliance :: File has compliant audio stream at ID %s; %s" % (i, fileInfo.path), logger.DEBUG)
else:
if mediamanager.SCANNER_VERBOSE:
logger.log(u"ScanForCompliance :: File has non-compliant audio stream at ID %s of codec %s; %s" % (i, stream['codec'], fileInfo.path), logger.DEBUG)
compliant = False
if not compliant:
retval.append(fileInfo)
if mediamanager.SCANNER_VERBOSE:
logger.log(u"ScanForCompliance :: Completed", logger.DEBUG)
return retval
def run(self):
# List of all finalized files that need to be remuxed into final product
self.finalfiles = []
logger.log("Entering transcode job; %s" % self._file.path, logger.MESSAGE)
#Beginning demux operations
logger.log("Beginning deumx; %s" % self._file.path, logger.DEBUG)
demux_successful = self.demux_file()
logger.log("Completed deumx; %s" % self._file.path, logger.DEBUG)
if not demux_successful:
logger.log("Demux failed; %s" % self._file.path, logger.MESSAGE)
logger.log("Exiting transcode job; %s" % self._file.path, logger.MESSAGE)
return
else:
logger.log("Demux successful; %s" % self._file.path, logger.DEBUG)
# Since we aren't transcoding video just identify the streams and add them to the self.finalfiles list
for _file in self.demuxfiles:
if re.match('^.*\.(video.\d)\..*$', _file) is not None:
self.finalfiles.append(_file)
# Check if we need to abort before proceeding
if self.abort:
self.finish()
return
# Begin audio transcoding operation looping over demuxed files where contents is audio
convert_results = []
for _file in self.demuxfiles:
if re.match('^.*\.(audio.\d)\..*$', _file) is not None:
logger.log("Beginning remediate audio; %s" % _file, logger.DEBUG)
result = self.remediate_audio(_file)
if result:
logger.log("Remediate audio successful; %s" % _file, logger.DEBUG)
else:
logger.log("Remediate audio successful; %s" % _file, logger.DEBUG)
convert_results.append(result)
if False in convert_results:
logger.log("Remediation of audio Failed; %s" % _file, logger.DEBUG)
logger.log("Exiting transcode job; %s" % self._file.path, logger.MESSAGE)
return
# Check if we need to abort before proceeding
if self.abort:
self.finish()
return
# Begin remuxing operation
logger.log("Beginning remux of sreams; %s" % self._file.path, logger.DEBUG)
if self.remux_files():
logger.log("Remuxing of sreams successful; %s" % self._file.path, logger.DEBUG)
else:
logger.log("Remuxing of streams unsuccessful; %s" % self._file.path, logger.DEBUG)
logger.log("Exiting transcode job; %s" % self._file.path, logger.MESSAGE)
return
# Perform final operation of copying file from temp to src file's parent directory and renaming according to user set pending extension
logger.log("Beginning finalization of job; %s" % self._file.path, logger.DEBUG)
if not self.finalize():
logger.log("Finalization of job failed; %s" % self._file.path, logger.DEBUG)
logger.log("Exiting transcode job; %s" % self._file.path, logger.MESSAGE)
return
else:
logger.log("Finalization of job successful; %s" % self._file.path, logger.DEBUG)
# All done. State some logging.
logger.log("Completed job execution; %s" % self._file.path, logger.DEBUG)
logger.log("Exiting transcode job; %s" % self._file.path, logger.MESSAGE)
def run(self): logger.log(u'VerificationDeclineJob :: Job run for file %s' % self.name)
def run(self): logger.log(u'VerificationAcceptJob :: Job run for file %s' % self.name)