Exemplo n.º 1
0
class TestEndtoEnd(unittest.TestCase):
    def setUp(self):
        self.WS = WorkerSetup()
        if os.path.exists(self.WS.instance_yaml):
            self.WS.run()
        self.settings = self.WS.settings_dict
        self.encode_profile = 'desktop_mp4'
        self.veda_id = 'XXXXXXXX2016-V00TEST'
        self.jobid = 'xx4xx'

        self.VW = VedaWorker(veda_id=self.veda_id,
                             encode_profile=self.encode_profile,
                             jobid=self.jobid)

    def test_intake(self):
        if not os.path.exists(self.WS.instance_yaml):
            self.assertTrue(True)
            return None
        """
        copied from __init__
        """
        self.VW.VideoObject = Video(veda_id=self.VW.veda_id)
        self.VW.VideoObject.activate()
        self.assertTrue(self.VW.VideoObject.valid)
        self.VW.settings = self.settings

        self.VW._ENG_INTAKE()
        self.assertTrue(self.VW.VideoObject.valid)
        self.assertTrue(
            os.path.exists(os.path.join(self.VW.workdir, self.VW.source_file)))

        self.assertTrue(self.VW.VideoObject.valid)

        # self._UPDATE_API()
        # self._GENERATE_ENCODE()
        # self._EXECUTE_ENCODE()
        # self._VALIDATE_ENCODE()
        # if self.encoded is True:
        #     self._DELIVER_FILE()

        # if self.endpoint_url is not None:
        #     """
        #     Integrate with main
        #     """
        #     final_name = self.output_file
        #     celery_task_fire.deliverable_route.apply_async(
        #         (final_name, ),
        #         queue='transcode_stat'
        #         )

        # """
        # Clean up workdir
        # """

    def tearDown(self):
        pass
Exemplo n.º 2
0
    def setUp(self):
        self.WS = WorkerSetup()
        if os.path.exists(self.WS.instance_yaml):
            self.WS.run()
        self.settings = self.WS.settings_dict
        self.encode_profile = 'desktop_mp4'
        self.veda_id = 'XXXXXXXX2016-V00TEST'
        self.jobid = 'xx4xx'

        self.VW = VedaWorker(veda_id=self.veda_id,
                             encode_profile=self.encode_profile,
                             jobid=self.jobid)
class TestAssetConnection(unittest.TestCase):
    def setUp(self):
        self.WS = WorkerSetup()
        if os.path.exists(self.WS.instance_yaml):
            self.WS.run()
        self.settings = self.WS.settings_dict

    def test_storage_setup(self):
        if not os.path.exists(self.WS.instance_yaml):
            return None

        salient_variables = [
            'aws_deliver_access_key', 'aws_deliver_secret_key',
            'aws_deliver_bucket'
        ]
        for s in salient_variables:
            self.assertTrue(len(self.settings[s]) > 0)

    def test_delivery_setup(self):
        if not os.path.exists(self.WS.instance_yaml):
            return None

        salient_variables = [
            'aws_access_key', 'aws_secret_key', 'aws_storage_bucket'
        ]

        for s in salient_variables:
            self.assertTrue(len(self.settings[s]) > 0)

    def test_storage_connection(self):
        if not os.path.exists(self.WS.instance_yaml):
            return None

        conn = S3Connection(self.settings['aws_access_key'],
                            self.settings['aws_secret_key'])
        try:
            bucket = conn.get_bucket(self.settings['aws_storage_bucket'])
            self.assertTrue(True)
        except:
            self.assertFalse(True)

    def test_delivery_connection(self):
        if not os.path.exists(self.WS.instance_yaml):
            return None

        conn = S3Connection(self.settings['aws_deliver_access_key'],
                            self.settings['aws_deliver_secret_key'])
        try:
            bucket = conn.get_bucket(self.settings['aws_deliver_bucket'])
            self.assertTrue(True)
        except:
            self.assertFalse(True)
Exemplo n.º 4
0
class TestIntake(unittest.TestCase):

    def setUp(self):
        self.WS = WorkerSetup()
        if os.path.exists(self.WS.instance_yaml):
            self.WS.run()
        self.settings = self.WS.settings_dict
        self.encode_profile = 'desktop_mp4'
        self.veda_id = 'XXXXXXXX2016-V00TEST'
        self.jobid = 'xx4xx'

        self.VW = VedaWorker(
            veda_id=self.veda_id, 
            encode_profile=self.encode_profile,
            jobid=self.jobid
            )


    def test_intake(self):
        if not os.path.exists(self.WS.instance_yaml):
            self.assertTrue(True)
            return None

        """
        copied from __init__
        """
        self.VW.VideoObject = Video(
            veda_id=self.VW.veda_id
            )
        self.VW.VideoObject.activate()
        self.assertTrue(self.VW.VideoObject.valid)
        self.VW.settings = self.settings

        self.VW._ENG_INTAKE()
        print self.VW.VideoObject
        self.assertTrue(self.VW.VideoObject.valid)
        self.assertTrue(
            os.path.exists(
                os.path.join(
                    self.VW.workdir, 
                    self.VW.source_file
                    )
                )
            )

        self.assertTrue(self.VW.VideoObject.valid)


    def tearDown(self):
        pass
Exemplo n.º 5
0
    def setUp(self):
        self.WS = WorkerSetup()
        if os.path.exists(self.WS.instance_yaml):
            self.WS.run()
        self.settings = self.WS.settings_dict
        self.encode_profile = 'desktop_mp4'
        """
        Gen abstractions
        """
        # Video
        self.VideoObject = Video(veda_id='XXXXXXXX2016-V00TEST', )
        self.VideoObject.activate()

        # Encode
        self.E = Encode(VideoObject=self.VideoObject,
                        profile_name=self.encode_profile)
        self.E.pull_data()
        self.ffcommand = None
Exemplo n.º 6
0
class TestCeleryConnect(unittest.TestCase):
    def setUp(self):
        self.WS = WorkerSetup()
        if os.path.exists(self.WS.instance_yaml):
            self.WS.run()
        self.settings = self.WS.settings_dict

    def test_celery_setup(self):
        if not os.path.exists(self.WS.instance_yaml):
            self.assertTrue(True)
            return None

        salient_variables = [
            'celery_app_name', 'celery_receiver_queue', 'rabbitmq_user',
            'rabbitmq_pass', 'rabbitmq_broker'
        ]

        for s in salient_variables:
            self.assertFalse(len(self.settings[s]) == 0)
Exemplo n.º 7
0
class Test_Encode_Command(unittest.TestCase):
    def setUp(self):
        self.WS = WorkerSetup()
        if os.path.exists(self.WS.instance_yaml):
            self.WS.run()
        self.settings = self.WS.settings_dict
        self.encode_profile = 'desktop_mp4'
        """
        Gen abstractions
        """
        # Video
        self.VideoObject = Video(veda_id='XXXXXXXX2016-V00TEST', )
        self.VideoObject.activate()

        # Encode
        self.E = Encode(VideoObject=self.VideoObject,
                        profile_name=self.encode_profile)
        self.E.pull_data()
        self.ffcommand = None

    def test_generate_command(self):
        if not os.path.exists(self.WS.instance_yaml):
            self.assertTrue(True)
            return None
        """
        Generate the (shell) command / Encode Object
        """
        self.assertTrue(self.VideoObject.valid is True)
        # self.assertTrue(self.VideoObject.valid is True)
        """
        Generate the Encode Object
        """
        self.E.pull_data()
        self.assertFalse(self.E.filetype is None)
        """
        Generate the (shell) command
        """
        self.ffcommand = CommandGenerate(VideoObject=self.VideoObject,
                                         EncodeObject=self.E).generate()

        self.assertFalse(self.ffcommand is None)
        """
Exemplo n.º 8
0
    def setUp(self):
        self.WS = WorkerSetup()
        if os.path.exists(self.WS.instance_yaml):
            self.WS.run()
        self.settings = self.WS.settings_dict
        self.encode_profile = 'desktop_mp4'
        self.veda_id = 'XXXXXXXX2016-V00TEST'
        self.jobid = 'xx4xx'

        self.VW = VedaWorker(
            veda_id=self.veda_id, 
            encode_profile=self.encode_profile,
            jobid=self.jobid
            )
Exemplo n.º 9
0
class TestIntake(unittest.TestCase):
    def setUp(self):
        self.WS = WorkerSetup()
        if os.path.exists(self.WS.instance_yaml):
            self.WS.run()
        self.settings = self.WS.settings_dict
        self.encode_profile = 'desktop_mp4'
        self.veda_id = 'XXXXXXXX2016-V00TEST'
        self.jobid = 'xx4xx'

        self.VW = VedaWorker(veda_id=self.veda_id,
                             encode_profile=self.encode_profile,
                             jobid=self.jobid)

    def test_intake(self):
        if not os.path.exists(self.WS.instance_yaml):
            self.assertTrue(True)
            return None
        """
        copied from __init__
        """
        self.VW.VideoObject = Video(veda_id=self.VW.veda_id)
        self.VW.VideoObject.activate()
        self.assertTrue(self.VW.VideoObject.valid)
        self.VW.settings = self.settings

        self.VW._ENG_INTAKE()
        print self.VW.VideoObject
        self.assertTrue(self.VW.VideoObject.valid)
        self.assertTrue(
            os.path.exists(os.path.join(self.VW.workdir, self.VW.source_file)))

        self.assertTrue(self.VW.VideoObject.valid)

    def tearDown(self):
        pass
Exemplo n.º 10
0
class TestCeleryConnect(unittest.TestCase):

    def setUp(self):
        self.WS = WorkerSetup()
        if os.path.exists(self.WS.instance_yaml):
            self.WS.run()
        self.settings = self.WS.settings_dict


    def test_celery_setup(self):
        if not os.path.exists(self.WS.instance_yaml):
            self.assertTrue(True)
            return None

        salient_variables = [
            'celery_app_name', 
            'celery_receiver_queue', 
            'rabbitmq_user',
            'rabbitmq_pass',
            'rabbitmq_broker'
            ]

        for s in salient_variables:
            self.assertFalse(len(self.settings[s]) == 0)
Exemplo n.º 11
0
class TestAssetConnection(unittest.TestCase):
    
    def setUp(self):
        self.WS = WorkerSetup()
        if os.path.exists(self.WS.instance_yaml):
            self.WS.run()
        self.settings = self.WS.settings_dict


    def test_storage_setup(self):
        if not os.path.exists(self.WS.instance_yaml):
            return None
        
        salient_variables = [
            'aws_deliver_access_key',
            'aws_deliver_secret_key',
            'aws_deliver_bucket'
            ]
        for s in salient_variables:
            self.assertTrue(len(self.settings[s]) > 0)


    def test_delivery_setup(self):
        if not os.path.exists(self.WS.instance_yaml):
            return None

        salient_variables = [
            'aws_access_key',
            'aws_secret_key',
            'aws_storage_bucket'
            ]

        for s in salient_variables:
            self.assertTrue(len(self.settings[s]) > 0)


    def test_storage_connection(self):
        if not os.path.exists(self.WS.instance_yaml):
            return None

        conn = S3Connection(
            self.settings['aws_access_key'], 
            self.settings['aws_secret_key']
            )
        try:
            bucket = conn.get_bucket(self.settings['aws_storage_bucket'])
            self.assertTrue(True)
        except:
            self.assertFalse(True)


    def test_delivery_connection(self):
        if not os.path.exists(self.WS.instance_yaml):
            return None

        conn = S3Connection(
            self.settings['aws_deliver_access_key'], 
            self.settings['aws_deliver_secret_key']
            )
        try:
            bucket = conn.get_bucket(self.settings['aws_deliver_bucket'])
            self.assertTrue(True)
        except:
            self.assertFalse(True)
Exemplo n.º 12
0
from __future__ import absolute_import
import os
import sys
from os.path import expanduser
from celery import Celery
import shutil
"""

Start Celery Worker (if VEDA-attached node)

"""
sys.path.append(os.path.dirname(os.path.abspath(__file__)))

from config import WorkerSetup

WS = WorkerSetup()
if os.path.exists(WS.instance_yaml):
    WS.run()
settings = WS.settings_dict

homedir = expanduser("~")


def cel_Start():
    app = Celery(
        settings['celery_app_name'],
        broker='amqp://' + settings['rabbitmq_user'] + \
            ':' + settings['rabbitmq_pass'] + '@' + settings['rabbitmq_broker'] + ':5672//',
        backend='amqp://' + settings['rabbitmq_user'] + \
            ':' + settings['rabbitmq_pass'] + '@' + settings['rabbitmq_broker'] + ':5672//',
        include=['celeryapp']
Exemplo n.º 13
0
class TestAPIConnection(unittest.TestCase):
    def setUp(self):
        self.WS = WorkerSetup()
        if os.path.exists(self.WS.instance_yaml):
            self.WS.run()
        self.settings = self.WS.settings_dict

    def test_val_setup(self):
        if not os.path.exists(self.WS.instance_yaml):
            self.assertTrue(True)
            return None

        salient_variables = [
            'val_api_url',
            'val_client_id',
            'val_password',
            'val_secret_key',
            'val_username',
            'val_token_url',
        ]
        for s in salient_variables:
            self.assertTrue(len(self.WS.settings_dict[s]) > 0)

    def test_veda_setup(self):
        if not os.path.exists(self.WS.instance_yaml):
            self.assertTrue(True)
            return None

        salient_variables = [
            'veda_api_url',
            'veda_auth_url',
            'veda_client_id',
            'veda_secret_key',
            'veda_token_url',
        ]
        for s in salient_variables:
            self.assertTrue(len(self.WS.settings_dict[s]) > 0)

    def test_val_connection(self):
        if not os.path.exists(self.WS.instance_yaml):
            self.assertTrue(True)
            return None

        val_token = generate_apitoken.val_tokengen()
        self.assertFalse(val_token == None)

        headers = {
            'Authorization': 'Bearer ' + val_token,
            'content-type': 'application/json'
        }
        s = requests.get(self.WS.settings_dict['val_api_url'],
                         headers=headers,
                         timeout=20)

        self.assertFalse(s.status_code == 404)
        self.assertFalse(s.status_code > 299)

    def test_veda_connection(self):
        if not os.path.exists(self.WS.instance_yaml):
            self.assertTrue(True)
            return None

        veda_token = generate_apitoken.veda_tokengen()
        self.assertFalse(veda_token == None)
        headers = {
            'Authorization': 'Token ' + veda_token,  # + veda_token,
            'content-type': 'application/json'
        }
        s = requests.get(self.WS.settings_dict['veda_api_url'] + '/',
                         headers=headers,
                         timeout=20)

        self.assertFalse(s.status_code == 404)
        self.assertFalse(s.status_code > 299)
Exemplo n.º 14
0
-resolution (frame size)
-CRF (increase for lower bitrate videos)

This is built to generate commands for a very small number of encodes and is not a substitute 
for knowledgable use of ffmpeg if one's intention is to broaden use beyond the very limited set of 
endpoints the edX platform provides

input, via two classes, encode and video, which can be generated either via the node independently
or via celery connection to VEDA (VEDA will send video_id and encode_profile via Celery queue)

"""
from global_vars import *
from reporting import ErrorObject, Output
from config import WorkerSetup
WS = WorkerSetup()
if os.path.exists(WS.instance_yaml):
    WS.run()
settings = WS.settings_dict




class CommandGenerate():

    def __init__(self, VideoObject, EncodeObject, **kwargs):
        self.VideoObject = VideoObject
        self.EncodeObject = EncodeObject
        self.jobid = kwargs.get('jobid', None)
        self.ffcommand = []
Exemplo n.º 15
0
    def run(self):

        WS = WorkerSetup()
        if self.setup is True:
            WS.setup = True

        WS.run()
        self.settings = WS.settings_dict

        if self.encode_profile is None:
            ErrorObject().print_error(
                message = 'No Encode Profile Specified'
                )
            return None

        self.VideoObject = Video(
            veda_id=self.veda_id
            )
        self.VideoObject.activate()
        if self.VideoObject.valid is False:
            ErrorObject().print_error(
                message = 'Invalid Video / VEDA Data'
                )
            return None

        if not os.path.exists(self.workdir):
            os.mkdir(self.workdir)


        """
        Pipeline Steps :
          I. Intake
            Ib. Validate Mezz
          II. change status in APIs
          III. Generate Encode Command
          IV. Execute Encodes
            IVa. Validate Products
          (*)V. Deliver Encodes (sftp and others?), retrieve URLs
          (*)VI. Change Status in APIs, add URLs
          VII. Clean Directory

        """
        self._ENG_INTAKE()

        if self.VideoObject.valid is False:
            ErrorObject().print_error(
                message = 'Invalid Video / Local'
                )
            return None

        self._UPDATE_API()

        if self.encode_profile == 'hls':
            self._HLSPipeline()
        else:
            self._StaticPipeline()

        print self.endpoint_url

        if self.endpoint_url is not None:
            """
            Integrate with main
            """
            veda_id = self.veda_id
            encode_profile = self.encode_profile
            celeryapp.deliverable_route.apply_async(
                (veda_id, encode_profile),
                queue='transcode_stat'
                )
        """
        Clean up workdir
        """
        if self.jobid is not None:
            shutil.rmtree(
                self.workdir
                )
Exemplo n.º 16
0
 def setUp(self):
     self.WS = WorkerSetup()
     if os.path.exists(self.WS.instance_yaml):
         self.WS.run()
     self.settings = self.WS.settings_dict
Exemplo n.º 17
0
class TestAPIConnection(unittest.TestCase):

    def setUp(self):
        self.WS = WorkerSetup()
        if os.path.exists(self.WS.instance_yaml):
            self.WS.run()
        self.settings = self.WS.settings_dict


    def test_val_setup(self):
        if not os.path.exists(self.WS.instance_yaml):
            self.assertTrue(True)
            return None

        salient_variables = [
            'val_api_url', 
            'val_client_id',
            'val_password',
            'val_secret_key', 
            'val_username',
            'val_token_url',
            ]
        for s in salient_variables:
            self.assertTrue(len(self.WS.settings_dict[s]) > 0)


    def test_veda_setup(self):
        if not os.path.exists(self.WS.instance_yaml):
            self.assertTrue(True)
            return None

        salient_variables = [
            'veda_api_url', 
            'veda_auth_url',
            'veda_client_id',
            'veda_secret_key', 
            'veda_token_url',
            ]
        for s in salient_variables:
            self.assertTrue(len(self.WS.settings_dict[s]) > 0)



    def test_val_connection(self):
        if not os.path.exists(self.WS.instance_yaml):
            self.assertTrue(True)
            return None

        val_token = generate_apitoken.val_tokengen()
        self.assertFalse(val_token == None)

        headers = {
            'Authorization': 'Bearer ' + val_token, 
            'content-type': 'application/json'
            }
        s = requests.get(self.WS.settings_dict['val_api_url'], headers=headers, timeout=20)

        self.assertFalse(s.status_code == 404)
        self.assertFalse(s.status_code > 299)


    def test_veda_connection(self):
        if not os.path.exists(self.WS.instance_yaml):
            self.assertTrue(True)
            return None

        veda_token = generate_apitoken.veda_tokengen()
        self.assertFalse(veda_token == None)
        headers = {
            'Authorization': 'Token ' + veda_token, # + veda_token,
            'content-type': 'application/json'
            }
        s = requests.get(self.WS.settings_dict['veda_api_url'] + '/', headers=headers, timeout=20)

        self.assertFalse(s.status_code == 404)
        self.assertFalse(s.status_code > 299)
Exemplo n.º 18
0
 def setUp(self):
     self.WS = WorkerSetup()
     if os.path.exists(self.WS.instance_yaml):
         self.WS.run()
     self.settings = self.WS.settings_dict