def main(self):
while True:
try:
cursor = models.conn.cursor()
utils.lock_on_string(cursor, 'video_queue', 1000000);
try:
current_video = models.Video.objects.filter(status='pending_upload_to_s3')[0]
except:
utils.unlock_on_string(cursor, 'video_queue')
time.sleep(10)
continue
remote_check = S3_CONN._make_request('HEAD', 'camera', 'originals/%s' % current_video.md5, {}, {})
if remote_check.status == 200:
logging.info("File %s already exists on S3." % current_video.md5)
else:
tmp_video_file = self.get_tmp_video(current_video.md5)
logging.info("Got pending video %s and started uploading to S3." % current_video.md5)
options = {'Content-Type': current_video.mimetype or 'application/octet-stream', 'X-Amz-Acl': 'private'}
S3_CONN.put(S3UploaderDaemon.S3_BUCKET_NAME, "originals/%s" % current_video.md5, S3.S3Object(tmp_video_file), options)
logging.info("Finished uploading %s to S3." % current_video.md5)
current_video.status = 'pending_transcoding'
current_video.save()
utils.unlock_on_string(cursor, 'video_queue');
except:
file_name = os.path.join(base, '..', 'log', 's3_uploader-%s.error' % str(time.time()).replace('.', ''))
logging.info("OMGWTFLOLBBQ: %s." % file_name)
error_details = open(file_name, 'w')
traceback.print_exc(file=error_details)
error_details.close()
time.sleep(15)
def main(self):
if not os.path.exists(TranscoderDaemon.TMP_VIDEO_ROOT):
os.makedirs(TranscoderDaemon.TMP_VIDEO_ROOT)
self.load_jobs()
while True:
try:
cursor = models.conn.cursor()
utils.lock_on_string(cursor, 'video_queue', 1000000);
try:
current_video = models.Video.objects.filter(status='pending_transcoding')[0]
except IndexError:
utils.unlock_on_string(cursor, 'video_queue')
time.sleep(10)
continue
logging.info("Downloading original video %s so we can transcode the shit out of it." % current_video.md5)
self.save_tmp_video_and_create_references(current_video)
current_video.status = 'transcoding'
current_video.save()
utils.unlock_and_lock_again_real_quick(cursor, 'video_queue')
logging.info("Preparing to run transcoding jobs on video %s." % current_video.md5)
for job in self.jobs:
job_passes = job.transcodingjobpass_set.select_related().order_by('step_number')
for job_pass in job_passes:
if job_pass.use_result_from is None:
logging.info('steps: %s' % range(1, job_pass.step_number)[::-1])
for step in range(1, job_pass.step_number+1)[::-1]:
source_pass_filename = '%s.%s.%s' % (current_video.md5, str(step-1), job_pass.transcoding_pass.from_extension)
logging.info('---> %s: ' % source_pass_filename)
source_pass_path = os.path.join(TranscoderDaemon.TMP_VIDEO_ROOT, job.job_slug, source_pass_filename)
if os.path.exists(source_pass_path):
break
else:
tp = job_pass.use_result_from.transcoding_pass
source_pass_filename = '%s.%s.%s' % (current_video.md5, job_pass.use_result_from.step_number, tp.from_extension)
logging.info('---> %s: ' % source_pass_filename)
source_pass_path = os.path.join(TranscoderDaemon.TMP_VIDEO_ROOT, job.job_slug, source_pass_filename)
target_extension = job_pass.transcoding_pass.to_extension
target_pass_filename = '%s.%s.%s' % (current_video.md5, str(job_pass.step_number), target_extension)
target_pass_path = os.path.join(TranscoderDaemon.TMP_VIDEO_ROOT, job.job_slug, target_pass_filename)
command = job_pass.transcoding_pass.command.replace('$SOURCE', source_pass_path).replace('$TARGET', target_pass_path)
logging.info("Running: %s" % command)
command_status, command_output = commands.getstatusoutput(command)
logging.info("Transcode completed, uploading result back to S3.")
source_url = self.put_the_result_back_in_s3(current_video, job.job_slug, target_pass_path, target_extension)
models.VideoVersion(video=current_video, trancoded_with=job, url=source_url)
logging.info("Transcoded and uploaded video %s with the %s encoding." % (current_video.md5, job.job_slug))
utils.unlock_on_string(cursor, 'video_queue');
except:
current_video.status = 'pending_transcoding'
current_video.save()
file_name = os.path.join(base, '..', 'log', 'transcoder-%s.error' % str(time.time()).replace('.', ''))
logging.info("OMGWTFLOLBBQ: %s." % file_name)
error_details = open(file_name, 'w')
traceback.print_exc(file=error_details)
error_details.close()
time.sleep(15)
def main(self):
self.debug_log("\n testing")
if not os.path.exists(TranscoderDaemon.TMP_VIDEO_ROOT):
os.makedirs(TranscoderDaemon.TMP_VIDEO_ROOT)
# self.load_jobs()
self.jobs = models.TranscodingJob.objects.all()
while True:
try:
cursor = models.conn.cursor()
utils.lock_on_string(cursor, 'video_queue', 1000000);
try:
current_video = models.Video.objects.filter(status='pending_transcoding')[0]
except IndexError:
utils.unlock_on_string(cursor, 'video_queue')
time.sleep(10)
continue
if not os.path.exists(os.path.join(TranscoderDaemon.TMP_VIDEO_ROOT, 'originals', current_video.set_slug)):
os.makedirs(os.path.join(TranscoderDaemon.TMP_VIDEO_ROOT, 'originals', current_video.set_slug))
if not os.path.exists(os.path.join(TranscoderDaemon.TMP_VIDEO_ROOT, 'transcoding', current_video.set_slug)):
os.makedirs(os.path.join(TranscoderDaemon.TMP_VIDEO_ROOT, 'transcoding', current_video.set_slug))
if not os.path.exists(os.path.join(TranscoderDaemon.TMP_VIDEO_ROOT, 'versions', current_video.set_slug)):
os.makedirs(os.path.join(TranscoderDaemon.TMP_VIDEO_ROOT, 'versions', current_video.set_slug))
self.debug_log("\nDownloading original video %s so we can transcode the shit out of it." % current_video.md5)
logging.info("Downloading original video %s so we can transcode the shit out of it." % current_video.md5)
self.save_tmp_video_and_create_references(current_video)
current_video.status = 'transcoding'
current_video.save()
utils.unlock_and_lock_again_real_quick(cursor, 'video_queue')
self.debug_log("\nPreparing to run transcoding jobs on video %s." % current_video.md5);
logging.info("Preparing to run transcoding jobs on video %s." % current_video.md5)
for job in self.jobs:
job_slug_parts = job.job_slug.split('-')
framerate = job_slug_parts[1][-2:]
self.debug_log("\nTranscoding job framerate = %s." % framerate);
self.debug_log("\nVideo framerate choice = %s." % current_video.fps_choice);
if (framerate == '24' or framerate == '30'):
if (int(current_video.fps_choice) != int(framerate)):
self.debug_log("\nIgnoring this transcoding job");
continue
self.debug_log("\n\n Job: %s." % job.job_slug);
job_passes = job.transcodingjobpass_set.select_related().order_by('step_number')
current_pass_stack = ''
original_filename = "%s.%s" % (current_video.md5, current_video.extension)
original_path = os.path.join(TranscoderDaemon.TMP_VIDEO_ROOT, 'originals', current_video.set_slug, original_filename)
source_pass_path = original_path
for job_pass in job_passes:
self.debug_log("\n\n Job Pass: %s." % job_pass.transcoding_pass);
current_pass_stack = "%s__%s" % (current_pass_stack, job_pass.transcoding_pass.slug)
self.debug_log("\ncurrent_pass_stack: %s" % current_pass_stack);
target_extension = job_pass.transcoding_pass.to_extension
self.debug_log("\n target_extension: %s" % target_extension);
target_pass_filename = "%s%s.%s" % (current_video.md5, current_pass_stack, target_extension)
self.debug_log("\n target_pass_filename: %s" % target_pass_filename);
target_pass_path = os.path.join(TranscoderDaemon.TMP_VIDEO_ROOT, 'transcoding', current_video.set_slug, target_pass_filename)
self.debug_log("\n target_pass_path: %s" % target_pass_path);
self.debug_log("\n source_pass_path: %s target_pass_path: %s" % (source_pass_path, target_pass_path));
self.debug_log("\n os.path.exists(target_pass_path): %s" % os.path.exists(target_pass_path));
if os.path.exists(target_pass_path):
source_pass_path = target_pass_path
self.debug_log("\n some other transcoding job already generated the file for this pass, skip it: %s" % target_pass_path);
continue
else:
command = job_pass.transcoding_pass.command.replace('$SOURCE', source_pass_path).replace('$TARGET', target_pass_path)
self.debug_log("\n Running: %s" % command);
# command_status, command_output = commands.getstatusoutput(command)
pipe = os.popen('{ ' + command + '; } 2>&1', 'r')
command_output = pipe.read()
try:
command_status = pipe.close()
except:
pass
self.debug_log("\n command completed! %s" % command_output);
source_pass_path = target_pass_path
logging.info("Transcode completed, uploading result back to S3.")
# source_url = self.put_the_result_back_in_s3(current_video, job.job_slug, target_pass_path, target_extension)
if os.path.isdir(target_pass_path):
logging.info("%s is an image path, consider the 00000001.jpg file inside that path as the result" % (target_pass_path))
target_pass_path = os.path.join(target_pass_path, '00000001.jpg')
source_url = self.put_the_result_back_in_video_tmp(current_video, job.job_slug, target_pass_path, target_extension)
logging.info("Transcoded and uploaded video %s with the %s encoding." % (current_video.md5, job.job_slug))
utils.unlock_on_string(cursor, 'video_queue');
self.debug_log("\n Wait 1 second...");
time.sleep(1)
except:
self.debug_log("\n Error!");
traceback.print_exc(file=sys.stdout)
file_name = os.path.join(base, '..', 'log', 'transcoder-%s.error' % str(time.time()).replace('.', ''))
logging.info("OMGWTFLOLBBQ: %s." % file_name)
error_details = open(file_name, 'w')
traceback.print_exc(file=error_details)
error_details.close()
if (current_video):
current_video.status = 'pending_transcoding'
current_video.save()
time.sleep(15)