def main(self):
   while True:
     try:
       cursor = models.conn.cursor()
       utils.lock_on_string(cursor, 'video_queue', 1000000);
       try:
         current_video = models.Video.objects.filter(status='pending_upload_to_s3')[0]
       except:
         utils.unlock_on_string(cursor, 'video_queue')
         time.sleep(10)
         continue
       remote_check = S3_CONN._make_request('HEAD', 'camera', 'originals/%s' % current_video.md5, {}, {})
       if remote_check.status == 200:
         logging.info("File %s already exists on S3." % current_video.md5)
       else:
         tmp_video_file = self.get_tmp_video(current_video.md5)
         logging.info("Got pending video %s and started uploading to S3." % current_video.md5)
         options = {'Content-Type': current_video.mimetype or 'application/octet-stream', 'X-Amz-Acl': 'private'}
         S3_CONN.put(S3UploaderDaemon.S3_BUCKET_NAME, "originals/%s" % current_video.md5, S3.S3Object(tmp_video_file), options)
         logging.info("Finished uploading %s to S3." % current_video.md5)
       current_video.status = 'pending_transcoding'
       current_video.save()
       utils.unlock_on_string(cursor, 'video_queue');
     except:
       file_name = os.path.join(base, '..', 'log', 's3_uploader-%s.error' % str(time.time()).replace('.', ''))
       logging.info("OMGWTFLOLBBQ: %s." % file_name)
       error_details = open(file_name, 'w')
       traceback.print_exc(file=error_details)
       error_details.close()
       time.sleep(15)
示例#2
0
 def main(self):
   if not os.path.exists(TranscoderDaemon.TMP_VIDEO_ROOT):
     os.makedirs(TranscoderDaemon.TMP_VIDEO_ROOT)
   self.load_jobs()
   while True:
     try:
       cursor = models.conn.cursor()
       utils.lock_on_string(cursor, 'video_queue', 1000000);
       try:
         current_video = models.Video.objects.filter(status='pending_transcoding')[0]
       except IndexError:
         utils.unlock_on_string(cursor, 'video_queue')
         time.sleep(10)
         continue
       logging.info("Downloading original video %s so we can transcode the shit out of it." % current_video.md5)
       self.save_tmp_video_and_create_references(current_video)        
       current_video.status = 'transcoding'
       current_video.save()
       utils.unlock_and_lock_again_real_quick(cursor, 'video_queue')
       logging.info("Preparing to run transcoding jobs on video %s." % current_video.md5)
       for job in self.jobs:
         job_passes = job.transcodingjobpass_set.select_related().order_by('step_number')
         for job_pass in job_passes:
           if job_pass.use_result_from is None:
             logging.info('steps: %s' % range(1, job_pass.step_number)[::-1])
             for step in range(1, job_pass.step_number+1)[::-1]:
               source_pass_filename = '%s.%s.%s' % (current_video.md5, str(step-1), job_pass.transcoding_pass.from_extension)
               logging.info('---> %s: ' % source_pass_filename)
               source_pass_path = os.path.join(TranscoderDaemon.TMP_VIDEO_ROOT, job.job_slug, source_pass_filename)
               if os.path.exists(source_pass_path):
                 break
           else:
             tp = job_pass.use_result_from.transcoding_pass
             source_pass_filename = '%s.%s.%s' % (current_video.md5, job_pass.use_result_from.step_number, tp.from_extension)
             logging.info('---> %s: ' % source_pass_filename)
             source_pass_path = os.path.join(TranscoderDaemon.TMP_VIDEO_ROOT, job.job_slug, source_pass_filename)                
           target_extension = job_pass.transcoding_pass.to_extension
           target_pass_filename = '%s.%s.%s' % (current_video.md5, str(job_pass.step_number), target_extension)
           target_pass_path = os.path.join(TranscoderDaemon.TMP_VIDEO_ROOT, job.job_slug, target_pass_filename)
           command = job_pass.transcoding_pass.command.replace('$SOURCE', source_pass_path).replace('$TARGET', target_pass_path)
           logging.info("Running: %s" % command)
           command_status, command_output = commands.getstatusoutput(command)
         logging.info("Transcode completed, uploading result back to S3.")
         source_url = self.put_the_result_back_in_s3(current_video, job.job_slug, target_pass_path, target_extension)
         models.VideoVersion(video=current_video, trancoded_with=job, url=source_url)
         logging.info("Transcoded and uploaded video %s with the %s encoding." % (current_video.md5, job.job_slug))          
       utils.unlock_on_string(cursor, 'video_queue');
     except:
       current_video.status = 'pending_transcoding'
       current_video.save()
       file_name = os.path.join(base, '..', 'log', 'transcoder-%s.error' % str(time.time()).replace('.', ''))
       logging.info("OMGWTFLOLBBQ: %s." % file_name)
       error_details = open(file_name, 'w')
       traceback.print_exc(file=error_details)
       error_details.close()
       time.sleep(15)
 def main(self):
   self.debug_log("\n testing")
   if not os.path.exists(TranscoderDaemon.TMP_VIDEO_ROOT):
     os.makedirs(TranscoderDaemon.TMP_VIDEO_ROOT)
   # self.load_jobs()
   self.jobs = models.TranscodingJob.objects.all()
   while True:
     try:
       cursor = models.conn.cursor()
       utils.lock_on_string(cursor, 'video_queue', 1000000);
       try:
         current_video = models.Video.objects.filter(status='pending_transcoding')[0]
       except IndexError:
         utils.unlock_on_string(cursor, 'video_queue')
         time.sleep(10)
         continue
       if not os.path.exists(os.path.join(TranscoderDaemon.TMP_VIDEO_ROOT, 'originals', current_video.set_slug)):
         os.makedirs(os.path.join(TranscoderDaemon.TMP_VIDEO_ROOT, 'originals', current_video.set_slug))
       if not os.path.exists(os.path.join(TranscoderDaemon.TMP_VIDEO_ROOT, 'transcoding', current_video.set_slug)):
         os.makedirs(os.path.join(TranscoderDaemon.TMP_VIDEO_ROOT, 'transcoding', current_video.set_slug))
       if not os.path.exists(os.path.join(TranscoderDaemon.TMP_VIDEO_ROOT, 'versions', current_video.set_slug)):
         os.makedirs(os.path.join(TranscoderDaemon.TMP_VIDEO_ROOT, 'versions', current_video.set_slug))
       self.debug_log("\nDownloading original video %s so we can transcode the shit out of it." % current_video.md5)
       logging.info("Downloading original video %s so we can transcode the shit out of it." % current_video.md5)
       self.save_tmp_video_and_create_references(current_video)        
       current_video.status = 'transcoding'
       current_video.save()
       utils.unlock_and_lock_again_real_quick(cursor, 'video_queue')
       self.debug_log("\nPreparing to run transcoding jobs on video %s." % current_video.md5);
       logging.info("Preparing to run transcoding jobs on video %s." % current_video.md5)
       for job in self.jobs:
         job_slug_parts = job.job_slug.split('-')
         framerate = job_slug_parts[1][-2:]
         self.debug_log("\nTranscoding job framerate = %s." % framerate);
         self.debug_log("\nVideo framerate choice = %s." % current_video.fps_choice);
         if (framerate == '24' or framerate == '30'):
           if (int(current_video.fps_choice) != int(framerate)):
             self.debug_log("\nIgnoring this transcoding job");
             continue
         self.debug_log("\n\n Job: %s." % job.job_slug);
         job_passes = job.transcodingjobpass_set.select_related().order_by('step_number')
         current_pass_stack = ''
         original_filename = "%s.%s" % (current_video.md5, current_video.extension)
         original_path = os.path.join(TranscoderDaemon.TMP_VIDEO_ROOT, 'originals', current_video.set_slug, original_filename)
         source_pass_path = original_path
         for job_pass in job_passes:
           self.debug_log("\n\n Job Pass: %s." % job_pass.transcoding_pass);
           current_pass_stack = "%s__%s" % (current_pass_stack, job_pass.transcoding_pass.slug)
           self.debug_log("\ncurrent_pass_stack: %s" % current_pass_stack);
           target_extension = job_pass.transcoding_pass.to_extension
           self.debug_log("\n target_extension: %s" % target_extension);
           target_pass_filename = "%s%s.%s" % (current_video.md5, current_pass_stack, target_extension)
           self.debug_log("\n target_pass_filename: %s" % target_pass_filename);
           target_pass_path = os.path.join(TranscoderDaemon.TMP_VIDEO_ROOT, 'transcoding', current_video.set_slug, target_pass_filename)
           self.debug_log("\n target_pass_path: %s" % target_pass_path);
           self.debug_log("\n source_pass_path: %s target_pass_path: %s" % (source_pass_path, target_pass_path));
           self.debug_log("\n os.path.exists(target_pass_path): %s" % os.path.exists(target_pass_path));
           if os.path.exists(target_pass_path):
             source_pass_path = target_pass_path
             self.debug_log("\n some other transcoding job already generated the file for this pass, skip it: %s" % target_pass_path);
             continue
           else:
             command = job_pass.transcoding_pass.command.replace('$SOURCE', source_pass_path).replace('$TARGET', target_pass_path)
             self.debug_log("\n Running: %s" % command);
             # command_status, command_output = commands.getstatusoutput(command)
             pipe = os.popen('{ ' + command + '; } 2>&1', 'r')
             command_output = pipe.read()
             try:
               command_status = pipe.close()
             except:
               pass
             self.debug_log("\n command completed! %s" % command_output);
             source_pass_path = target_pass_path
         logging.info("Transcode completed, uploading result back to S3.")
         # source_url = self.put_the_result_back_in_s3(current_video, job.job_slug, target_pass_path, target_extension)
         if os.path.isdir(target_pass_path):
           logging.info("%s is an image path, consider the 00000001.jpg file inside that path as the result" % (target_pass_path))
           target_pass_path = os.path.join(target_pass_path, '00000001.jpg')
         source_url = self.put_the_result_back_in_video_tmp(current_video, job.job_slug, target_pass_path, target_extension)
         logging.info("Transcoded and uploaded video %s with the %s encoding." % (current_video.md5, job.job_slug))          
       utils.unlock_on_string(cursor, 'video_queue');
       self.debug_log("\n Wait 1 second...");
       time.sleep(1)
     except:
       self.debug_log("\n Error!");
       traceback.print_exc(file=sys.stdout)
       file_name = os.path.join(base, '..', 'log', 'transcoder-%s.error' % str(time.time()).replace('.', ''))
       logging.info("OMGWTFLOLBBQ: %s." % file_name)
       error_details = open(file_name, 'w')
       traceback.print_exc(file=error_details)
       error_details.close()
       if (current_video):
         current_video.status = 'pending_transcoding'
         current_video.save()
       time.sleep(15)