def test_generate(self, mock_data, mock_call, mock_codec, mock_scalar,
mock_passes, mock_destination, mock_logger):
"""
Tests `generate` method works correctly.
"""
ffcommand = ['dummy-ffcommand-arg']
job_id = mock_data.get('job_id', None)
command_generate = CommandGenerate(
VideoObject=mock_data.get('video_object', self.video),
EncodeObject=mock_data.get('encode_object', self.encode),
jobid=job_id)
self.assertEqual(command_generate.ffcommand, [])
self.assertIsNone(command_generate.workdir)
command_generate.ffcommand = ffcommand
result_command = command_generate.generate()
if mock_data.get('error_message', ''):
mock_logger.error.assert_called_with(
mock_data.get('error_message'))
else:
self.assertIsNotNone(command_generate.workdir)
expected_workdir = os.path.join(
ENCODE_WORK_DIR, job_id) if job_id else ENCODE_WORK_DIR
self.assertEqual(command_generate.workdir, expected_workdir)
self.assertEqual(result_command, ' '.join(ffcommand))
if mock_data.get('ENFORCE_TARGET_ASPECT', False):
self.assertTrue(mock_scalar.called)
self.assertTrue(mock_call.called)
self.assertTrue(mock_codec.called)
self.assertTrue(mock_passes.called)
self.assertTrue(mock_destination.called)
def setUp(self):
"""
Setup for command generate tests.
"""
self.video = Video(veda_id='XXXXXXXX2016-V00TEST')
self.encode = Encode(video_object=self.video, profile_name=None)
self.command_generate = CommandGenerate(VideoObject=self.video,
EncodeObject=self.encode)
def _generate_encode(self):
"""
Generate the (shell) command / Encode Object
"""
encoding = Encode(video_object=self.VideoObject,
profile_name=self.encode_profile)
encoding.pull_data()
if encoding.filetype is None:
return
self.ffcommand = CommandGenerate(VideoObject=self.VideoObject,
EncodeObject=encoding,
jobid=self.jobid,
workdir=self.workdir,
settings=self.settings).generate()
class CommandGenerateTest(unittest.TestCase):
"""
Test class for Command Generate.
"""
def setUp(self):
"""
Setup for command generate tests.
"""
self.video = Video(veda_id='XXXXXXXX2016-V00TEST')
self.encode = Encode(video_object=self.video, profile_name=None)
self.command_generate = CommandGenerate(VideoObject=self.video,
EncodeObject=self.encode)
@data(
({
'video_object': None,
'error_message': 'Command generation: No Video object'
}),
({
'encode_object': None,
'error_message': 'Command generation: No Encode object'
}),
({
'job_id': 'dummy-job-id',
'ENFORCE_TARGET_ASPECT': True
}),
)
@patch('video_worker.generate_encode.logger')
@patch.object(CommandGenerate, '_destination')
@patch.object(CommandGenerate, '_passes')
@patch.object(CommandGenerate, '_scalar')
@patch.object(CommandGenerate, '_codec')
@patch.object(CommandGenerate, '_call')
def test_generate(self, mock_data, mock_call, mock_codec, mock_scalar,
mock_passes, mock_destination, mock_logger):
"""
Tests `generate` method works correctly.
"""
ffcommand = ['dummy-ffcommand-arg']
job_id = mock_data.get('job_id', None)
command_generate = CommandGenerate(
VideoObject=mock_data.get('video_object', self.video),
EncodeObject=mock_data.get('encode_object', self.encode),
jobid=job_id)
self.assertEqual(command_generate.ffcommand, [])
self.assertIsNone(command_generate.workdir)
command_generate.ffcommand = ffcommand
result_command = command_generate.generate()
if mock_data.get('error_message', ''):
mock_logger.error.assert_called_with(
mock_data.get('error_message'))
else:
self.assertIsNotNone(command_generate.workdir)
expected_workdir = os.path.join(
ENCODE_WORK_DIR, job_id) if job_id else ENCODE_WORK_DIR
self.assertEqual(command_generate.workdir, expected_workdir)
self.assertEqual(result_command, ' '.join(ffcommand))
if mock_data.get('ENFORCE_TARGET_ASPECT', False):
self.assertTrue(mock_scalar.called)
self.assertTrue(mock_call.called)
self.assertTrue(mock_codec.called)
self.assertTrue(mock_passes.called)
self.assertTrue(mock_destination.called)
@data(({
'veda_id': 'dummy-veda-id',
'mezz_extension': 'mp4'
}), ({
'mezz_extension': 'mp4'
}), ({
'filetype': 'mp3'
}))
def test_call(self, mock_data):
"""
Test that `_call` method works correctly.
"""
veda_id = mock_data.get('veda_id', None)
file_type = mock_data.get('filetype', None)
mezz_extension = mock_data.get('mezz_extension', '')
mezz_filepath = mock_data.get('mezz_filepath', 'dummy-mezz-path')
self.command_generate.workdir = ENCODE_WORK_DIR
self.command_generate.ffcommand = ['dummy-ffcommand-arg']
self.command_generate.settings.update(
{'ffmpeg_compiled': '-ffmpeg_compiled'})
self.command_generate.VideoObject.veda_id = veda_id
self.command_generate.VideoObject.mezz_extension = mezz_extension
self.command_generate.VideoObject.mezz_filepath = mezz_filepath
self.command_generate.EncodeObject.filetype = file_type
self.command_generate._call()
expected_ffcommand = [
'dummy-ffcommand-arg', '-ffmpeg_compiled', '-hide_banner', '-y',
'-i'
]
expected_ffcommand.append(
'{workdir}/{file_name}{file_extension}'.format(
workdir=ENCODE_WORK_DIR,
file_name=veda_id if veda_id else mezz_filepath,
file_extension='.' + mezz_extension if mezz_extension else ''))
if file_type != 'mp3':
expected_ffcommand.append('-c:v')
else:
expected_ffcommand.append('-c:a')
self.assertEqual(self.command_generate.ffcommand, expected_ffcommand)
@data(({
'filetype': 'mp3'
}), ({
'filetype': 'mp3',
'ffcommand': ['dummy-ffcommand-arg'],
'expected_ffcommand': ['dummy-ffcommand-arg', 'libmp3lame']
}), ({
'filetype': 'mp4',
'ffcommand': ['dummy-ffcommand-arg'],
'expected_ffcommand': ['dummy-ffcommand-arg', 'libx264']
}), ({
'filetype': 'webm',
'ffcommand': ['dummy-ffcommand-arg'],
'expected_ffcommand': ['dummy-ffcommand-arg', 'libvpx']
}))
def test_codec(self, mock_data):
"""
Tests that `_codec` mothod works correctly.
"""
self.command_generate.ffcommand = mock_data.get('ffcommand', None)
self.command_generate.EncodeObject.filetype = mock_data.get(
'filetype', None)
self.command_generate._codec()
self.assertEqual(self.command_generate.ffcommand,
mock_data.get('expected_ffcommand', None))
@data(
({
'ffcommand': None,
'expected_ffcommand': None
}),
({
'file_type': 'mp3',
'expected_ffcommand': []
}),
({
'resolution': 480,
'mezz_bitrate': 'test mezz rate',
'mezz_resolution': '720x480',
'expected_ffcommand': []
}),
({
'resolution': 480,
'mezz_resolution': '720x420',
'expected_ffcommand': ['-vf', 'scale=853:480']
}),
# Add more coverage for _scalar method - See EDUCATOR-1071
)
def test_scalar(self, mock_data):
"""
Tests `_scalar` method works correctly.
"""
self.command_generate.ffcommand = mock_data.get('ffcommand', [])
self.command_generate.EncodeObject.filetype = mock_data.get(
'file_type', 'mp4')
self.command_generate.EncodeObject.resolution = mock_data.get(
'resolution', '480')
self.command_generate.VideoObject.mezz_resolution = mock_data.get(
'mezz_resolution', 'Unparsed')
self.command_generate.VideoObject.mezz_bitrate = mock_data.get(
'mezz_bitrate', 'Unparsed')
self.command_generate._scalar()
self.assertEqual(self.command_generate.ffcommand,
mock_data.get('expected_ffcommand', []))
@data(('mp3', 100, 50, ['-b:a', '100k']),
('mp4', 100, 50, ['-crf', '100']), ('webm', 100, 50, [
'-b:v', '50k', '-minrate', '10k', '-maxrate', '62k', '-bufsize',
'26k'
]), ('webm', 100, 200, [
'-b:v', '100k', '-minrate', '10k', '-maxrate', '125k',
'-bufsize', '76k'
]))
@unpack
def test_passes(self, file_type, rate_factor, mezz_bitrate,
expected_ffcommand):
"""
Tests that `_passes` works correctly.
"""
self.command_generate.EncodeObject.filetype = file_type
self.command_generate.EncodeObject.rate_factor = rate_factor
self.command_generate.VideoObject.mezz_bitrate = mezz_bitrate
self.command_generate._passes()
self.assertEqual(self.command_generate.ffcommand, expected_ffcommand)
@data(
{
'file_type': 'mp4',
'veda_id': 'dummy-veda-id',
'expected_ffcommand': ['-movflags', 'faststart']
},
{
'file_type': 'webm',
'veda_id': 'dummy-veda-id',
'expected_ffcommand': ['-c:a', 'libvorbis']
},
{
'file_type': 'mp4',
'veda_id': None,
'expected_ffcommand': ['-movflags', 'faststart']
},
)
@unpack
def test_destination(self, file_type, veda_id, expected_ffcommand):
"""
Tests that `_destination` works correctly.
"""
encode_suffix = 'dummy-encode-suffix'
mezz_file_path = 'dummy-mezz-path'
self.command_generate.workdir = ENCODE_WORK_DIR
self.command_generate.EncodeObject.encode_suffix = encode_suffix
self.command_generate.EncodeObject.filetype = file_type
self.command_generate.VideoObject.veda_id = veda_id
self.command_generate.VideoObject.mezz_filepath = mezz_file_path
# Add a correct expected ffcommand path
expected_ffcommand.append(
'{workdir}/{veda_id}_{encode_suffix}.{file_type}'.format(
workdir=ENCODE_WORK_DIR,
veda_id=veda_id if veda_id else mezz_file_path,
encode_suffix=encode_suffix,
file_type=file_type))
self.command_generate._destination()
self.assertEqual(self.command_generate.ffcommand, expected_ffcommand)
class VideoWorker(object):
def __init__(self, **kwargs):
self.settings = None
self.veda_id = kwargs.get('veda_id', None)
self.setup = kwargs.get('setup', False)
self.jobid = kwargs.get('jobid', None)
self.update_val_status = kwargs.get('update_val_status')
self.encode_profile = kwargs.get('encode_profile', None)
self.VideoObject = kwargs.get('VideoObject', None)
self.instance_yaml = kwargs.get(
'instance_yaml',
os.path.join(
os.path.dirname(os.path.dirname(os.path.abspath(__file__))),
'instance_config.yaml'))
self.workdir = kwargs.get('workdir', self.determine_workdir())
self.ffcommand = None
self.source_file = kwargs.get('source_file', None)
self.output_file = None
self.endpoint_url = None
# Pipeline Steps
self.encoded = False
self.delivered = False
def determine_workdir(self):
if not os.path.exists(ENCODE_WORK_DIR):
os.mkdir(ENCODE_WORK_DIR)
if self.jobid is None:
return ENCODE_WORK_DIR
else:
return os.path.join(ENCODE_WORK_DIR, self.jobid)
def run(self):
self.settings = get_config()
if self.encode_profile is None:
logger.error('No Encode Profile Specified')
return
self.VideoObject = Video(veda_id=self.veda_id, )
if self.source_file is not None:
self.VideoObject.mezz_filepath = os.path.join(
self.workdir, self.source_file)
self.VideoObject.activate()
if not self.VideoObject.valid:
logger.error('{id} : Invalid Video Data'.format(
id=self.VideoObject.veda_id))
return
if not os.path.exists(self.workdir):
os.mkdir(self.workdir)
logger.info('{id} | {encoding} : Ready for Encode'.format(
id=self.VideoObject.veda_id, encoding=self.encode_profile))
# Pipeline Steps :
# I. Intake
# Ib. Validate Mezz
# II. change status in APIs
# III. Generate Encode Command
# IV. Execute Encodes
# IVa. Validate Products
# (*)V. Deliver Encodes (sftp and others?), retrieve URLs
# (*)VI. Change Status in APIs, add URLs
# VII. Clean Directory
self._engine_intake()
if not self.VideoObject.valid:
logger.error('Invalid Video / Local')
return
if self.VideoObject.val_id is not None:
self._update_api()
# generate video images command and update S3 and edxval
# run against 'hls' encode only
if self.encode_profile == 'hls':
# Run HLS encode
self._hls_pipeline()
# Auto-video Images
VideoImages(video_object=self.VideoObject,
work_dir=self.workdir,
source_file=self.source_file,
jobid=self.jobid,
settings=self.settings).create_and_update()
else:
self._static_pipeline()
logger.info('{id} | {encoding} : Encode Complete'.format(
id=self.VideoObject.veda_id, encoding=self.encode_profile))
if self.endpoint_url is not None and self.VideoObject.veda_id is not None:
# Integrate with main
veda_id = self.veda_id
encode_profile = self.encode_profile
deliverable_route.apply_async(
(veda_id, encode_profile),
queue=self.settings['celery_deliver_queue'])
logger.info(
'{id} | {encoding} : encoded file queued for delivery'.format(
id=self.VideoObject.veda_id, encoding=self.encode_profile))
# Clean up workdir
if self.jobid is not None:
shutil.rmtree(self.workdir)
def _static_pipeline(self):
self._generate_encode()
if self.ffcommand is None:
return
logger.info('ffcommand is written as %s', self.ffcommand)
self._execute_encode()
if self.encode_profile == 'audio_mp3':
self.encoded = True
else:
self._validate_encode()
if self.encoded and self.VideoObject.veda_id is not None:
self._deliver_file()
def _hls_pipeline(self):
"""
Activate HLS, use hls lib to upload
"""
if not os.path.exists(os.path.join(self.workdir, self.source_file)):
logger.error(
': {id} | {encoding} Local raw video file not found'.format(
id=self.VideoObject.veda_id, encoding=self.encode_profile))
return
os.chdir(self.workdir)
if self.settings['onsite_worker'] is True:
hls_chunk_instance = Chunkey(
mezz_file=os.path.join(self.workdir, self.source_file),
DELIVER_BUCKET=self.settings['edx_s3_endpoint_bucket'],
clean=False,
ACCESS_KEY_ID=self.settings['edx_access_key_id'],
SECRET_ACCESS_KEY=self.settings['edx_secret_access_key'])
else:
hls_chunk_instance = Chunkey(
mezz_file=os.path.join(self.workdir, self.source_file),
DELIVER_BUCKET=self.settings['edx_s3_endpoint_bucket'],
clean=False,
)
if hls_chunk_instance.complete:
self.endpoint_url = hls_chunk_instance.manifest_url
def _engine_intake(self):
"""
Copy file down from AWS S3 storage bucket
"""
if not self.VideoObject.valid:
logger.error(': {id} Invalid Video'.format(
id=self.VideoObject.veda_id, ))
return
if self.source_file is None:
if self.settings['onsite_worker'] is True:
conn = S3Connection(self.settings['veda_access_key_id'],
self.settings['veda_secret_access_key'])
else:
conn = S3Connection()
try:
bucket = conn.get_bucket(
self.settings['veda_s3_hotstore_bucket'])
except S3ResponseError:
logger.error('Invalid hotstore S3 bucket')
return
if self.VideoObject.mezz_extension is not None and len(
self.VideoObject.mezz_extension) > 0:
self.source_file = '.'.join((self.VideoObject.veda_id,
self.VideoObject.mezz_extension))
else:
self.source_file = self.VideoObject.veda_id
source_key = bucket.get_key(self.source_file)
if source_key is None:
logger.error(': {id} S3 Intake object not found'.format(
id=self.VideoObject.val_id))
return
source_key.get_contents_to_filename(
os.path.join(self.workdir, self.source_file))
if not os.path.exists(os.path.join(self.workdir,
self.source_file)):
logger.error(': {id} engine intake download error'.format(
id=self.VideoObject.val_id))
return
self.VideoObject.valid = ValidateVideo(
filepath=os.path.join(self.workdir, self.source_file)).valid
def _update_api(self):
UpdateAPIStatus(
val_video_status=VAL_TRANSCODE_STATUS,
veda_video_status=NODE_TRANSCODE_STATUS,
send_val=self.update_val_status,
VideoObject=self.VideoObject,
).run()
def _generate_encode(self):
"""
Generate the (shell) command / Encode Object
"""
encoding = Encode(video_object=self.VideoObject,
profile_name=self.encode_profile)
encoding.pull_data()
if encoding.filetype is None:
return
self.ffcommand = CommandGenerate(VideoObject=self.VideoObject,
EncodeObject=encoding,
jobid=self.jobid,
workdir=self.workdir,
settings=self.settings).generate()
def _execute_encode(self):
"""
if this is just a filepath, this should just work
--no need to move the source--
"""
if not os.path.exists(os.path.join(self.workdir, self.source_file)):
logger.error(': {id} Encode input file not found'.format(
id=self.VideoObject.veda_id))
return
process = subprocess.Popen(self.ffcommand,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
shell=True,
universal_newlines=True)
Output.status_bar(process=process)
self.output_file = self.ffcommand.split('/')[-1]
if not os.path.exists(os.path.join(self.workdir, self.output_file)):
logger.error(': {id} Encode output file not found'.format(
id=self.VideoObject.veda_id))
def _validate_encode(self):
"""
Validate encode by matching (w/in 5 sec) encode duration,
as well as standard validation tests
"""
if self.output_file is None:
self.encoded = False
return
else:
self.encoded = ValidateVideo(filepath=os.path.join(
self.workdir, self.output_file),
product_file=True,
VideoObject=self.VideoObject).valid
def _deliver_file(self):
"""
Deliver Here
"""
if not os.path.exists(os.path.join(self.workdir, self.output_file)):
return
D1 = Deliverable(VideoObject=self.VideoObject,
encode_profile=self.encode_profile,
output_file=self.output_file,
jobid=self.jobid,
workdir=self.workdir)
D1.run()
self.delivered = D1.delivered
self.endpoint_url = D1.endpoint_url