def setUp(self): """ Setup for command generate tests. """ self.video = Video(veda_id='XXXXXXXX2016-V00TEST') self.encode = Encode(video_object=self.video, profile_name=None) self.command_generate = CommandGenerate(VideoObject=self.video, EncodeObject=self.encode)
def setUpClass(cls): super(ApiCommunicateTest, cls).setUpClass() cls.Video = Video( veda_id='XXXXXXXX2016-V00TEST' ) cls.Video.valid = True cls.veda_video_status = 'veda status' cls.val_video_status = 'val status'
def setUp(self): super(VideoWorkerTest, self).setUp() self.VW = VideoWorker(**{'workdir': '/dummy-work-dir'}) # Provide dummy values for Video Worker. self.VW.VideoObject = Video(veda_id='XXXXXXXX2016-V00TEST') self.VW.VideoObject.valid = True self.VW.output_file = 'dummy-outfile' self.VW.source_file = 'dummy-sourcefile'
def run(self): self.settings = get_config() if self.encode_profile is None: logger.error('No Encode Profile Specified') return self.VideoObject = Video(veda_id=self.veda_id, ) if self.source_file is not None: self.VideoObject.mezz_filepath = os.path.join( self.workdir, self.source_file) self.VideoObject.activate() if not self.VideoObject.valid: logger.error('{id} : Invalid Video Data'.format( id=self.VideoObject.veda_id)) return if not os.path.exists(self.workdir): os.mkdir(self.workdir) logger.info('{id} | {encoding} : Ready for Encode'.format( id=self.VideoObject.veda_id, encoding=self.encode_profile)) # Pipeline Steps : # I. Intake # Ib. Validate Mezz # II. change status in APIs # III. Generate Encode Command # IV. Execute Encodes # IVa. Validate Products # (*)V. Deliver Encodes (sftp and others?), retrieve URLs # (*)VI. Change Status in APIs, add URLs # VII. Clean Directory self._engine_intake() if not self.VideoObject.valid: logger.error('Invalid Video / Local') return if self.VideoObject.val_id is not None: self._update_api() # generate video images command and update S3 and edxval # run against 'hls' encode only if self.encode_profile == 'hls': # Run HLS encode self._hls_pipeline() # Auto-video Images VideoImages(video_object=self.VideoObject, work_dir=self.workdir, source_file=self.source_file, jobid=self.jobid, settings=self.settings).create_and_update() else: self._static_pipeline() logger.info('{id} | {encoding} : Encode Complete'.format( id=self.VideoObject.veda_id, encoding=self.encode_profile)) if self.endpoint_url is not None and self.VideoObject.veda_id is not None: # Integrate with main veda_id = self.veda_id encode_profile = self.encode_profile deliverable_route.apply_async( (veda_id, encode_profile), queue=self.settings['celery_deliver_queue']) logger.info( '{id} | {encoding} : encoded file queued for delivery'.format( id=self.VideoObject.veda_id, encoding=self.encode_profile)) # Clean up workdir if self.jobid is not None: shutil.rmtree(self.workdir)
class VideoWorker(object): def __init__(self, **kwargs): self.settings = None self.veda_id = kwargs.get('veda_id', None) self.setup = kwargs.get('setup', False) self.jobid = kwargs.get('jobid', None) self.update_val_status = kwargs.get('update_val_status') self.encode_profile = kwargs.get('encode_profile', None) self.VideoObject = kwargs.get('VideoObject', None) self.instance_yaml = kwargs.get( 'instance_yaml', os.path.join( os.path.dirname(os.path.dirname(os.path.abspath(__file__))), 'instance_config.yaml')) self.workdir = kwargs.get('workdir', self.determine_workdir()) self.ffcommand = None self.source_file = kwargs.get('source_file', None) self.output_file = None self.endpoint_url = None # Pipeline Steps self.encoded = False self.delivered = False def determine_workdir(self): if not os.path.exists(ENCODE_WORK_DIR): os.mkdir(ENCODE_WORK_DIR) if self.jobid is None: return ENCODE_WORK_DIR else: return os.path.join(ENCODE_WORK_DIR, self.jobid) def run(self): self.settings = get_config() if self.encode_profile is None: logger.error('No Encode Profile Specified') return self.VideoObject = Video(veda_id=self.veda_id, ) if self.source_file is not None: self.VideoObject.mezz_filepath = os.path.join( self.workdir, self.source_file) self.VideoObject.activate() if not self.VideoObject.valid: logger.error('{id} : Invalid Video Data'.format( id=self.VideoObject.veda_id)) return if not os.path.exists(self.workdir): os.mkdir(self.workdir) logger.info('{id} | {encoding} : Ready for Encode'.format( id=self.VideoObject.veda_id, encoding=self.encode_profile)) # Pipeline Steps : # I. Intake # Ib. Validate Mezz # II. change status in APIs # III. Generate Encode Command # IV. Execute Encodes # IVa. Validate Products # (*)V. Deliver Encodes (sftp and others?), retrieve URLs # (*)VI. Change Status in APIs, add URLs # VII. Clean Directory self._engine_intake() if not self.VideoObject.valid: logger.error('Invalid Video / Local') return if self.VideoObject.val_id is not None: self._update_api() # generate video images command and update S3 and edxval # run against 'hls' encode only if self.encode_profile == 'hls': # Run HLS encode self._hls_pipeline() # Auto-video Images VideoImages(video_object=self.VideoObject, work_dir=self.workdir, source_file=self.source_file, jobid=self.jobid, settings=self.settings).create_and_update() else: self._static_pipeline() logger.info('{id} | {encoding} : Encode Complete'.format( id=self.VideoObject.veda_id, encoding=self.encode_profile)) if self.endpoint_url is not None and self.VideoObject.veda_id is not None: # Integrate with main veda_id = self.veda_id encode_profile = self.encode_profile deliverable_route.apply_async( (veda_id, encode_profile), queue=self.settings['celery_deliver_queue']) logger.info( '{id} | {encoding} : encoded file queued for delivery'.format( id=self.VideoObject.veda_id, encoding=self.encode_profile)) # Clean up workdir if self.jobid is not None: shutil.rmtree(self.workdir) def _static_pipeline(self): self._generate_encode() if self.ffcommand is None: return logger.info('ffcommand is written as %s', self.ffcommand) self._execute_encode() if self.encode_profile == 'audio_mp3': self.encoded = True else: self._validate_encode() if self.encoded and self.VideoObject.veda_id is not None: self._deliver_file() def _hls_pipeline(self): """ Activate HLS, use hls lib to upload """ if not os.path.exists(os.path.join(self.workdir, self.source_file)): logger.error( ': {id} | {encoding} Local raw video file not found'.format( id=self.VideoObject.veda_id, encoding=self.encode_profile)) return os.chdir(self.workdir) if self.settings['onsite_worker'] is True: hls_chunk_instance = Chunkey( mezz_file=os.path.join(self.workdir, self.source_file), DELIVER_BUCKET=self.settings['edx_s3_endpoint_bucket'], clean=False, ACCESS_KEY_ID=self.settings['edx_access_key_id'], SECRET_ACCESS_KEY=self.settings['edx_secret_access_key']) else: hls_chunk_instance = Chunkey( mezz_file=os.path.join(self.workdir, self.source_file), DELIVER_BUCKET=self.settings['edx_s3_endpoint_bucket'], clean=False, ) if hls_chunk_instance.complete: self.endpoint_url = hls_chunk_instance.manifest_url def _engine_intake(self): """ Copy file down from AWS S3 storage bucket """ if not self.VideoObject.valid: logger.error(': {id} Invalid Video'.format( id=self.VideoObject.veda_id, )) return if self.source_file is None: if self.settings['onsite_worker'] is True: conn = S3Connection(self.settings['veda_access_key_id'], self.settings['veda_secret_access_key']) else: conn = S3Connection() try: bucket = conn.get_bucket( self.settings['veda_s3_hotstore_bucket']) except S3ResponseError: logger.error('Invalid hotstore S3 bucket') return if self.VideoObject.mezz_extension is not None and len( self.VideoObject.mezz_extension) > 0: self.source_file = '.'.join((self.VideoObject.veda_id, self.VideoObject.mezz_extension)) else: self.source_file = self.VideoObject.veda_id source_key = bucket.get_key(self.source_file) if source_key is None: logger.error(': {id} S3 Intake object not found'.format( id=self.VideoObject.val_id)) return source_key.get_contents_to_filename( os.path.join(self.workdir, self.source_file)) if not os.path.exists(os.path.join(self.workdir, self.source_file)): logger.error(': {id} engine intake download error'.format( id=self.VideoObject.val_id)) return self.VideoObject.valid = ValidateVideo( filepath=os.path.join(self.workdir, self.source_file)).valid def _update_api(self): UpdateAPIStatus( val_video_status=VAL_TRANSCODE_STATUS, veda_video_status=NODE_TRANSCODE_STATUS, send_val=self.update_val_status, VideoObject=self.VideoObject, ).run() def _generate_encode(self): """ Generate the (shell) command / Encode Object """ encoding = Encode(video_object=self.VideoObject, profile_name=self.encode_profile) encoding.pull_data() if encoding.filetype is None: return self.ffcommand = CommandGenerate(VideoObject=self.VideoObject, EncodeObject=encoding, jobid=self.jobid, workdir=self.workdir, settings=self.settings).generate() def _execute_encode(self): """ if this is just a filepath, this should just work --no need to move the source-- """ if not os.path.exists(os.path.join(self.workdir, self.source_file)): logger.error(': {id} Encode input file not found'.format( id=self.VideoObject.veda_id)) return process = subprocess.Popen(self.ffcommand, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True, universal_newlines=True) Output.status_bar(process=process) self.output_file = self.ffcommand.split('/')[-1] if not os.path.exists(os.path.join(self.workdir, self.output_file)): logger.error(': {id} Encode output file not found'.format( id=self.VideoObject.veda_id)) def _validate_encode(self): """ Validate encode by matching (w/in 5 sec) encode duration, as well as standard validation tests """ if self.output_file is None: self.encoded = False return else: self.encoded = ValidateVideo(filepath=os.path.join( self.workdir, self.output_file), product_file=True, VideoObject=self.VideoObject).valid def _deliver_file(self): """ Deliver Here """ if not os.path.exists(os.path.join(self.workdir, self.output_file)): return D1 = Deliverable(VideoObject=self.VideoObject, encode_profile=self.encode_profile, output_file=self.output_file, jobid=self.jobid, workdir=self.workdir) D1.run() self.delivered = D1.delivered self.endpoint_url = D1.endpoint_url