class TestEndtoEnd(unittest.TestCase):
def setUp(self):
self.WS = WorkerSetup()
if os.path.exists(self.WS.instance_yaml):
self.WS.run()
self.settings = self.WS.settings_dict
self.encode_profile = 'desktop_mp4'
self.veda_id = 'XXXXXXXX2016-V00TEST'
self.jobid = 'xx4xx'
self.VW = VedaWorker(veda_id=self.veda_id,
encode_profile=self.encode_profile,
jobid=self.jobid)
def test_intake(self):
if not os.path.exists(self.WS.instance_yaml):
self.assertTrue(True)
return None
"""
copied from __init__
"""
self.VW.VideoObject = Video(veda_id=self.VW.veda_id)
self.VW.VideoObject.activate()
self.assertTrue(self.VW.VideoObject.valid)
self.VW.settings = self.settings
self.VW._ENG_INTAKE()
self.assertTrue(self.VW.VideoObject.valid)
self.assertTrue(
os.path.exists(os.path.join(self.VW.workdir, self.VW.source_file)))
self.assertTrue(self.VW.VideoObject.valid)
# self._UPDATE_API()
# self._GENERATE_ENCODE()
# self._EXECUTE_ENCODE()
# self._VALIDATE_ENCODE()
# if self.encoded is True:
# self._DELIVER_FILE()
# if self.endpoint_url is not None:
# """
# Integrate with main
# """
# final_name = self.output_file
# celery_task_fire.deliverable_route.apply_async(
# (final_name, ),
# queue='transcode_stat'
# )
# """
# Clean up workdir
# """
def tearDown(self):
pass
def setUp(self):
self.WS = WorkerSetup()
if os.path.exists(self.WS.instance_yaml):
self.WS.run()
self.settings = self.WS.settings_dict
self.encode_profile = 'desktop_mp4'
self.veda_id = 'XXXXXXXX2016-V00TEST'
self.jobid = 'xx4xx'
self.VW = VedaWorker(veda_id=self.veda_id,
encode_profile=self.encode_profile,
jobid=self.jobid)
class TestAssetConnection(unittest.TestCase):
def setUp(self):
self.WS = WorkerSetup()
if os.path.exists(self.WS.instance_yaml):
self.WS.run()
self.settings = self.WS.settings_dict
def test_storage_setup(self):
if not os.path.exists(self.WS.instance_yaml):
return None
salient_variables = [
'aws_deliver_access_key', 'aws_deliver_secret_key',
'aws_deliver_bucket'
]
for s in salient_variables:
self.assertTrue(len(self.settings[s]) > 0)
def test_delivery_setup(self):
if not os.path.exists(self.WS.instance_yaml):
return None
salient_variables = [
'aws_access_key', 'aws_secret_key', 'aws_storage_bucket'
]
for s in salient_variables:
self.assertTrue(len(self.settings[s]) > 0)
def test_storage_connection(self):
if not os.path.exists(self.WS.instance_yaml):
return None
conn = S3Connection(self.settings['aws_access_key'],
self.settings['aws_secret_key'])
try:
bucket = conn.get_bucket(self.settings['aws_storage_bucket'])
self.assertTrue(True)
except:
self.assertFalse(True)
def test_delivery_connection(self):
if not os.path.exists(self.WS.instance_yaml):
return None
conn = S3Connection(self.settings['aws_deliver_access_key'],
self.settings['aws_deliver_secret_key'])
try:
bucket = conn.get_bucket(self.settings['aws_deliver_bucket'])
self.assertTrue(True)
except:
self.assertFalse(True)
class TestIntake(unittest.TestCase):
def setUp(self):
self.WS = WorkerSetup()
if os.path.exists(self.WS.instance_yaml):
self.WS.run()
self.settings = self.WS.settings_dict
self.encode_profile = 'desktop_mp4'
self.veda_id = 'XXXXXXXX2016-V00TEST'
self.jobid = 'xx4xx'
self.VW = VedaWorker(
veda_id=self.veda_id,
encode_profile=self.encode_profile,
jobid=self.jobid
)
def test_intake(self):
if not os.path.exists(self.WS.instance_yaml):
self.assertTrue(True)
return None
"""
copied from __init__
"""
self.VW.VideoObject = Video(
veda_id=self.VW.veda_id
)
self.VW.VideoObject.activate()
self.assertTrue(self.VW.VideoObject.valid)
self.VW.settings = self.settings
self.VW._ENG_INTAKE()
print self.VW.VideoObject
self.assertTrue(self.VW.VideoObject.valid)
self.assertTrue(
os.path.exists(
os.path.join(
self.VW.workdir,
self.VW.source_file
)
)
)
self.assertTrue(self.VW.VideoObject.valid)
def tearDown(self):
pass
def setUp(self):
self.WS = WorkerSetup()
if os.path.exists(self.WS.instance_yaml):
self.WS.run()
self.settings = self.WS.settings_dict
self.encode_profile = 'desktop_mp4'
"""
Gen abstractions
"""
# Video
self.VideoObject = Video(veda_id='XXXXXXXX2016-V00TEST', )
self.VideoObject.activate()
# Encode
self.E = Encode(VideoObject=self.VideoObject,
profile_name=self.encode_profile)
self.E.pull_data()
self.ffcommand = None
class TestCeleryConnect(unittest.TestCase):
def setUp(self):
self.WS = WorkerSetup()
if os.path.exists(self.WS.instance_yaml):
self.WS.run()
self.settings = self.WS.settings_dict
def test_celery_setup(self):
if not os.path.exists(self.WS.instance_yaml):
self.assertTrue(True)
return None
salient_variables = [
'celery_app_name', 'celery_receiver_queue', 'rabbitmq_user',
'rabbitmq_pass', 'rabbitmq_broker'
]
for s in salient_variables:
self.assertFalse(len(self.settings[s]) == 0)
class Test_Encode_Command(unittest.TestCase):
def setUp(self):
self.WS = WorkerSetup()
if os.path.exists(self.WS.instance_yaml):
self.WS.run()
self.settings = self.WS.settings_dict
self.encode_profile = 'desktop_mp4'
"""
Gen abstractions
"""
# Video
self.VideoObject = Video(veda_id='XXXXXXXX2016-V00TEST', )
self.VideoObject.activate()
# Encode
self.E = Encode(VideoObject=self.VideoObject,
profile_name=self.encode_profile)
self.E.pull_data()
self.ffcommand = None
def test_generate_command(self):
if not os.path.exists(self.WS.instance_yaml):
self.assertTrue(True)
return None
"""
Generate the (shell) command / Encode Object
"""
self.assertTrue(self.VideoObject.valid is True)
# self.assertTrue(self.VideoObject.valid is True)
"""
Generate the Encode Object
"""
self.E.pull_data()
self.assertFalse(self.E.filetype is None)
"""
Generate the (shell) command
"""
self.ffcommand = CommandGenerate(VideoObject=self.VideoObject,
EncodeObject=self.E).generate()
self.assertFalse(self.ffcommand is None)
"""
def setUp(self):
self.WS = WorkerSetup()
if os.path.exists(self.WS.instance_yaml):
self.WS.run()
self.settings = self.WS.settings_dict
self.encode_profile = 'desktop_mp4'
self.veda_id = 'XXXXXXXX2016-V00TEST'
self.jobid = 'xx4xx'
self.VW = VedaWorker(
veda_id=self.veda_id,
encode_profile=self.encode_profile,
jobid=self.jobid
)
class TestIntake(unittest.TestCase):
def setUp(self):
self.WS = WorkerSetup()
if os.path.exists(self.WS.instance_yaml):
self.WS.run()
self.settings = self.WS.settings_dict
self.encode_profile = 'desktop_mp4'
self.veda_id = 'XXXXXXXX2016-V00TEST'
self.jobid = 'xx4xx'
self.VW = VedaWorker(veda_id=self.veda_id,
encode_profile=self.encode_profile,
jobid=self.jobid)
def test_intake(self):
if not os.path.exists(self.WS.instance_yaml):
self.assertTrue(True)
return None
"""
copied from __init__
"""
self.VW.VideoObject = Video(veda_id=self.VW.veda_id)
self.VW.VideoObject.activate()
self.assertTrue(self.VW.VideoObject.valid)
self.VW.settings = self.settings
self.VW._ENG_INTAKE()
print self.VW.VideoObject
self.assertTrue(self.VW.VideoObject.valid)
self.assertTrue(
os.path.exists(os.path.join(self.VW.workdir, self.VW.source_file)))
self.assertTrue(self.VW.VideoObject.valid)
def tearDown(self):
pass
class TestCeleryConnect(unittest.TestCase):
def setUp(self):
self.WS = WorkerSetup()
if os.path.exists(self.WS.instance_yaml):
self.WS.run()
self.settings = self.WS.settings_dict
def test_celery_setup(self):
if not os.path.exists(self.WS.instance_yaml):
self.assertTrue(True)
return None
salient_variables = [
'celery_app_name',
'celery_receiver_queue',
'rabbitmq_user',
'rabbitmq_pass',
'rabbitmq_broker'
]
for s in salient_variables:
self.assertFalse(len(self.settings[s]) == 0)
class TestAssetConnection(unittest.TestCase):
def setUp(self):
self.WS = WorkerSetup()
if os.path.exists(self.WS.instance_yaml):
self.WS.run()
self.settings = self.WS.settings_dict
def test_storage_setup(self):
if not os.path.exists(self.WS.instance_yaml):
return None
salient_variables = [
'aws_deliver_access_key',
'aws_deliver_secret_key',
'aws_deliver_bucket'
]
for s in salient_variables:
self.assertTrue(len(self.settings[s]) > 0)
def test_delivery_setup(self):
if not os.path.exists(self.WS.instance_yaml):
return None
salient_variables = [
'aws_access_key',
'aws_secret_key',
'aws_storage_bucket'
]
for s in salient_variables:
self.assertTrue(len(self.settings[s]) > 0)
def test_storage_connection(self):
if not os.path.exists(self.WS.instance_yaml):
return None
conn = S3Connection(
self.settings['aws_access_key'],
self.settings['aws_secret_key']
)
try:
bucket = conn.get_bucket(self.settings['aws_storage_bucket'])
self.assertTrue(True)
except:
self.assertFalse(True)
def test_delivery_connection(self):
if not os.path.exists(self.WS.instance_yaml):
return None
conn = S3Connection(
self.settings['aws_deliver_access_key'],
self.settings['aws_deliver_secret_key']
)
try:
bucket = conn.get_bucket(self.settings['aws_deliver_bucket'])
self.assertTrue(True)
except:
self.assertFalse(True)
from __future__ import absolute_import
import os
import sys
from os.path import expanduser
from celery import Celery
import shutil
"""
Start Celery Worker (if VEDA-attached node)
"""
sys.path.append(os.path.dirname(os.path.abspath(__file__)))
from config import WorkerSetup
WS = WorkerSetup()
if os.path.exists(WS.instance_yaml):
WS.run()
settings = WS.settings_dict
homedir = expanduser("~")
def cel_Start():
app = Celery(
settings['celery_app_name'],
broker='amqp://' + settings['rabbitmq_user'] + \
':' + settings['rabbitmq_pass'] + '@' + settings['rabbitmq_broker'] + ':5672//',
backend='amqp://' + settings['rabbitmq_user'] + \
':' + settings['rabbitmq_pass'] + '@' + settings['rabbitmq_broker'] + ':5672//',
include=['celeryapp']
class TestAPIConnection(unittest.TestCase):
def setUp(self):
self.WS = WorkerSetup()
if os.path.exists(self.WS.instance_yaml):
self.WS.run()
self.settings = self.WS.settings_dict
def test_val_setup(self):
if not os.path.exists(self.WS.instance_yaml):
self.assertTrue(True)
return None
salient_variables = [
'val_api_url',
'val_client_id',
'val_password',
'val_secret_key',
'val_username',
'val_token_url',
]
for s in salient_variables:
self.assertTrue(len(self.WS.settings_dict[s]) > 0)
def test_veda_setup(self):
if not os.path.exists(self.WS.instance_yaml):
self.assertTrue(True)
return None
salient_variables = [
'veda_api_url',
'veda_auth_url',
'veda_client_id',
'veda_secret_key',
'veda_token_url',
]
for s in salient_variables:
self.assertTrue(len(self.WS.settings_dict[s]) > 0)
def test_val_connection(self):
if not os.path.exists(self.WS.instance_yaml):
self.assertTrue(True)
return None
val_token = generate_apitoken.val_tokengen()
self.assertFalse(val_token == None)
headers = {
'Authorization': 'Bearer ' + val_token,
'content-type': 'application/json'
}
s = requests.get(self.WS.settings_dict['val_api_url'],
headers=headers,
timeout=20)
self.assertFalse(s.status_code == 404)
self.assertFalse(s.status_code > 299)
def test_veda_connection(self):
if not os.path.exists(self.WS.instance_yaml):
self.assertTrue(True)
return None
veda_token = generate_apitoken.veda_tokengen()
self.assertFalse(veda_token == None)
headers = {
'Authorization': 'Token ' + veda_token, # + veda_token,
'content-type': 'application/json'
}
s = requests.get(self.WS.settings_dict['veda_api_url'] + '/',
headers=headers,
timeout=20)
self.assertFalse(s.status_code == 404)
self.assertFalse(s.status_code > 299)
-resolution (frame size)
-CRF (increase for lower bitrate videos)
This is built to generate commands for a very small number of encodes and is not a substitute
for knowledgable use of ffmpeg if one's intention is to broaden use beyond the very limited set of
endpoints the edX platform provides
input, via two classes, encode and video, which can be generated either via the node independently
or via celery connection to VEDA (VEDA will send video_id and encode_profile via Celery queue)
"""
from global_vars import *
from reporting import ErrorObject, Output
from config import WorkerSetup
WS = WorkerSetup()
if os.path.exists(WS.instance_yaml):
WS.run()
settings = WS.settings_dict
class CommandGenerate():
def __init__(self, VideoObject, EncodeObject, **kwargs):
self.VideoObject = VideoObject
self.EncodeObject = EncodeObject
self.jobid = kwargs.get('jobid', None)
self.ffcommand = []
def run(self):
WS = WorkerSetup()
if self.setup is True:
WS.setup = True
WS.run()
self.settings = WS.settings_dict
if self.encode_profile is None:
ErrorObject().print_error(
message = 'No Encode Profile Specified'
)
return None
self.VideoObject = Video(
veda_id=self.veda_id
)
self.VideoObject.activate()
if self.VideoObject.valid is False:
ErrorObject().print_error(
message = 'Invalid Video / VEDA Data'
)
return None
if not os.path.exists(self.workdir):
os.mkdir(self.workdir)
"""
Pipeline Steps :
I. Intake
Ib. Validate Mezz
II. change status in APIs
III. Generate Encode Command
IV. Execute Encodes
IVa. Validate Products
(*)V. Deliver Encodes (sftp and others?), retrieve URLs
(*)VI. Change Status in APIs, add URLs
VII. Clean Directory
"""
self._ENG_INTAKE()
if self.VideoObject.valid is False:
ErrorObject().print_error(
message = 'Invalid Video / Local'
)
return None
self._UPDATE_API()
if self.encode_profile == 'hls':
self._HLSPipeline()
else:
self._StaticPipeline()
print self.endpoint_url
if self.endpoint_url is not None:
"""
Integrate with main
"""
veda_id = self.veda_id
encode_profile = self.encode_profile
celeryapp.deliverable_route.apply_async(
(veda_id, encode_profile),
queue='transcode_stat'
)
"""
Clean up workdir
"""
if self.jobid is not None:
shutil.rmtree(
self.workdir
)
def setUp(self):
self.WS = WorkerSetup()
if os.path.exists(self.WS.instance_yaml):
self.WS.run()
self.settings = self.WS.settings_dict
class TestAPIConnection(unittest.TestCase):
def setUp(self):
self.WS = WorkerSetup()
if os.path.exists(self.WS.instance_yaml):
self.WS.run()
self.settings = self.WS.settings_dict
def test_val_setup(self):
if not os.path.exists(self.WS.instance_yaml):
self.assertTrue(True)
return None
salient_variables = [
'val_api_url',
'val_client_id',
'val_password',
'val_secret_key',
'val_username',
'val_token_url',
]
for s in salient_variables:
self.assertTrue(len(self.WS.settings_dict[s]) > 0)
def test_veda_setup(self):
if not os.path.exists(self.WS.instance_yaml):
self.assertTrue(True)
return None
salient_variables = [
'veda_api_url',
'veda_auth_url',
'veda_client_id',
'veda_secret_key',
'veda_token_url',
]
for s in salient_variables:
self.assertTrue(len(self.WS.settings_dict[s]) > 0)
def test_val_connection(self):
if not os.path.exists(self.WS.instance_yaml):
self.assertTrue(True)
return None
val_token = generate_apitoken.val_tokengen()
self.assertFalse(val_token == None)
headers = {
'Authorization': 'Bearer ' + val_token,
'content-type': 'application/json'
}
s = requests.get(self.WS.settings_dict['val_api_url'], headers=headers, timeout=20)
self.assertFalse(s.status_code == 404)
self.assertFalse(s.status_code > 299)
def test_veda_connection(self):
if not os.path.exists(self.WS.instance_yaml):
self.assertTrue(True)
return None
veda_token = generate_apitoken.veda_tokengen()
self.assertFalse(veda_token == None)
headers = {
'Authorization': 'Token ' + veda_token, # + veda_token,
'content-type': 'application/json'
}
s = requests.get(self.WS.settings_dict['veda_api_url'] + '/', headers=headers, timeout=20)
self.assertFalse(s.status_code == 404)
self.assertFalse(s.status_code > 299)
def setUp(self):
self.WS = WorkerSetup()
if os.path.exists(self.WS.instance_yaml):
self.WS.run()
self.settings = self.WS.settings_dict