コード例 #1
0
class TestEndtoEnd(unittest.TestCase):
    def setUp(self):
        self.WS = WorkerSetup()
        if os.path.exists(self.WS.instance_yaml):
            self.WS.run()
        self.settings = self.WS.settings_dict
        self.encode_profile = 'desktop_mp4'
        self.veda_id = 'XXXXXXXX2016-V00TEST'
        self.jobid = 'xx4xx'

        self.VW = VedaWorker(veda_id=self.veda_id,
                             encode_profile=self.encode_profile,
                             jobid=self.jobid)

    def test_intake(self):
        if not os.path.exists(self.WS.instance_yaml):
            self.assertTrue(True)
            return None
        """
        copied from __init__
        """
        self.VW.VideoObject = Video(veda_id=self.VW.veda_id)
        self.VW.VideoObject.activate()
        self.assertTrue(self.VW.VideoObject.valid)
        self.VW.settings = self.settings

        self.VW._ENG_INTAKE()
        self.assertTrue(self.VW.VideoObject.valid)
        self.assertTrue(
            os.path.exists(os.path.join(self.VW.workdir, self.VW.source_file)))

        self.assertTrue(self.VW.VideoObject.valid)

        # self._UPDATE_API()
        # self._GENERATE_ENCODE()
        # self._EXECUTE_ENCODE()
        # self._VALIDATE_ENCODE()
        # if self.encoded is True:
        #     self._DELIVER_FILE()

        # if self.endpoint_url is not None:
        #     """
        #     Integrate with main
        #     """
        #     final_name = self.output_file
        #     celery_task_fire.deliverable_route.apply_async(
        #         (final_name, ),
        #         queue='transcode_stat'
        #         )

        # """
        # Clean up workdir
        # """

    def tearDown(self):
        pass
コード例 #2
0
    def setUp(self):
        self.WS = WorkerSetup()
        if os.path.exists(self.WS.instance_yaml):
            self.WS.run()
        self.settings = self.WS.settings_dict
        self.encode_profile = 'desktop_mp4'
        self.veda_id = 'XXXXXXXX2016-V00TEST'
        self.jobid = 'xx4xx'

        self.VW = VedaWorker(veda_id=self.veda_id,
                             encode_profile=self.encode_profile,
                             jobid=self.jobid)
コード例 #3
0
class TestAssetConnection(unittest.TestCase):
    def setUp(self):
        self.WS = WorkerSetup()
        if os.path.exists(self.WS.instance_yaml):
            self.WS.run()
        self.settings = self.WS.settings_dict

    def test_storage_setup(self):
        if not os.path.exists(self.WS.instance_yaml):
            return None

        salient_variables = [
            'aws_deliver_access_key', 'aws_deliver_secret_key',
            'aws_deliver_bucket'
        ]
        for s in salient_variables:
            self.assertTrue(len(self.settings[s]) > 0)

    def test_delivery_setup(self):
        if not os.path.exists(self.WS.instance_yaml):
            return None

        salient_variables = [
            'aws_access_key', 'aws_secret_key', 'aws_storage_bucket'
        ]

        for s in salient_variables:
            self.assertTrue(len(self.settings[s]) > 0)

    def test_storage_connection(self):
        if not os.path.exists(self.WS.instance_yaml):
            return None

        conn = S3Connection(self.settings['aws_access_key'],
                            self.settings['aws_secret_key'])
        try:
            bucket = conn.get_bucket(self.settings['aws_storage_bucket'])
            self.assertTrue(True)
        except:
            self.assertFalse(True)

    def test_delivery_connection(self):
        if not os.path.exists(self.WS.instance_yaml):
            return None

        conn = S3Connection(self.settings['aws_deliver_access_key'],
                            self.settings['aws_deliver_secret_key'])
        try:
            bucket = conn.get_bucket(self.settings['aws_deliver_bucket'])
            self.assertTrue(True)
        except:
            self.assertFalse(True)
コード例 #4
0
ファイル: test_intake.py プロジェクト: yro/veda_worker
class TestIntake(unittest.TestCase):

    def setUp(self):
        self.WS = WorkerSetup()
        if os.path.exists(self.WS.instance_yaml):
            self.WS.run()
        self.settings = self.WS.settings_dict
        self.encode_profile = 'desktop_mp4'
        self.veda_id = 'XXXXXXXX2016-V00TEST'
        self.jobid = 'xx4xx'

        self.VW = VedaWorker(
            veda_id=self.veda_id, 
            encode_profile=self.encode_profile,
            jobid=self.jobid
            )


    def test_intake(self):
        if not os.path.exists(self.WS.instance_yaml):
            self.assertTrue(True)
            return None

        """
        copied from __init__
        """
        self.VW.VideoObject = Video(
            veda_id=self.VW.veda_id
            )
        self.VW.VideoObject.activate()
        self.assertTrue(self.VW.VideoObject.valid)
        self.VW.settings = self.settings

        self.VW._ENG_INTAKE()
        print self.VW.VideoObject
        self.assertTrue(self.VW.VideoObject.valid)
        self.assertTrue(
            os.path.exists(
                os.path.join(
                    self.VW.workdir, 
                    self.VW.source_file
                    )
                )
            )

        self.assertTrue(self.VW.VideoObject.valid)


    def tearDown(self):
        pass
コード例 #5
0
    def setUp(self):
        self.WS = WorkerSetup()
        if os.path.exists(self.WS.instance_yaml):
            self.WS.run()
        self.settings = self.WS.settings_dict
        self.encode_profile = 'desktop_mp4'
        """
        Gen abstractions
        """
        # Video
        self.VideoObject = Video(veda_id='XXXXXXXX2016-V00TEST', )
        self.VideoObject.activate()

        # Encode
        self.E = Encode(VideoObject=self.VideoObject,
                        profile_name=self.encode_profile)
        self.E.pull_data()
        self.ffcommand = None
コード例 #6
0
class TestCeleryConnect(unittest.TestCase):
    def setUp(self):
        self.WS = WorkerSetup()
        if os.path.exists(self.WS.instance_yaml):
            self.WS.run()
        self.settings = self.WS.settings_dict

    def test_celery_setup(self):
        if not os.path.exists(self.WS.instance_yaml):
            self.assertTrue(True)
            return None

        salient_variables = [
            'celery_app_name', 'celery_receiver_queue', 'rabbitmq_user',
            'rabbitmq_pass', 'rabbitmq_broker'
        ]

        for s in salient_variables:
            self.assertFalse(len(self.settings[s]) == 0)
コード例 #7
0
class Test_Encode_Command(unittest.TestCase):
    def setUp(self):
        self.WS = WorkerSetup()
        if os.path.exists(self.WS.instance_yaml):
            self.WS.run()
        self.settings = self.WS.settings_dict
        self.encode_profile = 'desktop_mp4'
        """
        Gen abstractions
        """
        # Video
        self.VideoObject = Video(veda_id='XXXXXXXX2016-V00TEST', )
        self.VideoObject.activate()

        # Encode
        self.E = Encode(VideoObject=self.VideoObject,
                        profile_name=self.encode_profile)
        self.E.pull_data()
        self.ffcommand = None

    def test_generate_command(self):
        if not os.path.exists(self.WS.instance_yaml):
            self.assertTrue(True)
            return None
        """
        Generate the (shell) command / Encode Object
        """
        self.assertTrue(self.VideoObject.valid is True)
        # self.assertTrue(self.VideoObject.valid is True)
        """
        Generate the Encode Object
        """
        self.E.pull_data()
        self.assertFalse(self.E.filetype is None)
        """
        Generate the (shell) command
        """
        self.ffcommand = CommandGenerate(VideoObject=self.VideoObject,
                                         EncodeObject=self.E).generate()

        self.assertFalse(self.ffcommand is None)
        """
コード例 #8
0
ファイル: test_intake.py プロジェクト: yro/veda_worker
    def setUp(self):
        self.WS = WorkerSetup()
        if os.path.exists(self.WS.instance_yaml):
            self.WS.run()
        self.settings = self.WS.settings_dict
        self.encode_profile = 'desktop_mp4'
        self.veda_id = 'XXXXXXXX2016-V00TEST'
        self.jobid = 'xx4xx'

        self.VW = VedaWorker(
            veda_id=self.veda_id, 
            encode_profile=self.encode_profile,
            jobid=self.jobid
            )
コード例 #9
0
class TestIntake(unittest.TestCase):
    def setUp(self):
        self.WS = WorkerSetup()
        if os.path.exists(self.WS.instance_yaml):
            self.WS.run()
        self.settings = self.WS.settings_dict
        self.encode_profile = 'desktop_mp4'
        self.veda_id = 'XXXXXXXX2016-V00TEST'
        self.jobid = 'xx4xx'

        self.VW = VedaWorker(veda_id=self.veda_id,
                             encode_profile=self.encode_profile,
                             jobid=self.jobid)

    def test_intake(self):
        if not os.path.exists(self.WS.instance_yaml):
            self.assertTrue(True)
            return None
        """
        copied from __init__
        """
        self.VW.VideoObject = Video(veda_id=self.VW.veda_id)
        self.VW.VideoObject.activate()
        self.assertTrue(self.VW.VideoObject.valid)
        self.VW.settings = self.settings

        self.VW._ENG_INTAKE()
        print self.VW.VideoObject
        self.assertTrue(self.VW.VideoObject.valid)
        self.assertTrue(
            os.path.exists(os.path.join(self.VW.workdir, self.VW.source_file)))

        self.assertTrue(self.VW.VideoObject.valid)

    def tearDown(self):
        pass
コード例 #10
0
ファイル: test_node_connect.py プロジェクト: yro/veda_worker
class TestCeleryConnect(unittest.TestCase):

    def setUp(self):
        self.WS = WorkerSetup()
        if os.path.exists(self.WS.instance_yaml):
            self.WS.run()
        self.settings = self.WS.settings_dict


    def test_celery_setup(self):
        if not os.path.exists(self.WS.instance_yaml):
            self.assertTrue(True)
            return None

        salient_variables = [
            'celery_app_name', 
            'celery_receiver_queue', 
            'rabbitmq_user',
            'rabbitmq_pass',
            'rabbitmq_broker'
            ]

        for s in salient_variables:
            self.assertFalse(len(self.settings[s]) == 0)
コード例 #11
0
ファイル: test_asset_connect.py プロジェクト: yro/veda_worker
class TestAssetConnection(unittest.TestCase):
    
    def setUp(self):
        self.WS = WorkerSetup()
        if os.path.exists(self.WS.instance_yaml):
            self.WS.run()
        self.settings = self.WS.settings_dict


    def test_storage_setup(self):
        if not os.path.exists(self.WS.instance_yaml):
            return None
        
        salient_variables = [
            'aws_deliver_access_key',
            'aws_deliver_secret_key',
            'aws_deliver_bucket'
            ]
        for s in salient_variables:
            self.assertTrue(len(self.settings[s]) > 0)


    def test_delivery_setup(self):
        if not os.path.exists(self.WS.instance_yaml):
            return None

        salient_variables = [
            'aws_access_key',
            'aws_secret_key',
            'aws_storage_bucket'
            ]

        for s in salient_variables:
            self.assertTrue(len(self.settings[s]) > 0)


    def test_storage_connection(self):
        if not os.path.exists(self.WS.instance_yaml):
            return None

        conn = S3Connection(
            self.settings['aws_access_key'], 
            self.settings['aws_secret_key']
            )
        try:
            bucket = conn.get_bucket(self.settings['aws_storage_bucket'])
            self.assertTrue(True)
        except:
            self.assertFalse(True)


    def test_delivery_connection(self):
        if not os.path.exists(self.WS.instance_yaml):
            return None

        conn = S3Connection(
            self.settings['aws_deliver_access_key'], 
            self.settings['aws_deliver_secret_key']
            )
        try:
            bucket = conn.get_bucket(self.settings['aws_deliver_bucket'])
            self.assertTrue(True)
        except:
            self.assertFalse(True)
コード例 #12
0
from __future__ import absolute_import
import os
import sys
from os.path import expanduser
from celery import Celery
import shutil
"""

Start Celery Worker (if VEDA-attached node)

"""
sys.path.append(os.path.dirname(os.path.abspath(__file__)))

from config import WorkerSetup

WS = WorkerSetup()
if os.path.exists(WS.instance_yaml):
    WS.run()
settings = WS.settings_dict

homedir = expanduser("~")


def cel_Start():
    app = Celery(
        settings['celery_app_name'],
        broker='amqp://' + settings['rabbitmq_user'] + \
            ':' + settings['rabbitmq_pass'] + '@' + settings['rabbitmq_broker'] + ':5672//',
        backend='amqp://' + settings['rabbitmq_user'] + \
            ':' + settings['rabbitmq_pass'] + '@' + settings['rabbitmq_broker'] + ':5672//',
        include=['celeryapp']
コード例 #13
0
class TestAPIConnection(unittest.TestCase):
    def setUp(self):
        self.WS = WorkerSetup()
        if os.path.exists(self.WS.instance_yaml):
            self.WS.run()
        self.settings = self.WS.settings_dict

    def test_val_setup(self):
        if not os.path.exists(self.WS.instance_yaml):
            self.assertTrue(True)
            return None

        salient_variables = [
            'val_api_url',
            'val_client_id',
            'val_password',
            'val_secret_key',
            'val_username',
            'val_token_url',
        ]
        for s in salient_variables:
            self.assertTrue(len(self.WS.settings_dict[s]) > 0)

    def test_veda_setup(self):
        if not os.path.exists(self.WS.instance_yaml):
            self.assertTrue(True)
            return None

        salient_variables = [
            'veda_api_url',
            'veda_auth_url',
            'veda_client_id',
            'veda_secret_key',
            'veda_token_url',
        ]
        for s in salient_variables:
            self.assertTrue(len(self.WS.settings_dict[s]) > 0)

    def test_val_connection(self):
        if not os.path.exists(self.WS.instance_yaml):
            self.assertTrue(True)
            return None

        val_token = generate_apitoken.val_tokengen()
        self.assertFalse(val_token == None)

        headers = {
            'Authorization': 'Bearer ' + val_token,
            'content-type': 'application/json'
        }
        s = requests.get(self.WS.settings_dict['val_api_url'],
                         headers=headers,
                         timeout=20)

        self.assertFalse(s.status_code == 404)
        self.assertFalse(s.status_code > 299)

    def test_veda_connection(self):
        if not os.path.exists(self.WS.instance_yaml):
            self.assertTrue(True)
            return None

        veda_token = generate_apitoken.veda_tokengen()
        self.assertFalse(veda_token == None)
        headers = {
            'Authorization': 'Token ' + veda_token,  # + veda_token,
            'content-type': 'application/json'
        }
        s = requests.get(self.WS.settings_dict['veda_api_url'] + '/',
                         headers=headers,
                         timeout=20)

        self.assertFalse(s.status_code == 404)
        self.assertFalse(s.status_code > 299)
コード例 #14
0
ファイル: generate_encode.py プロジェクト: yro/veda_worker
-resolution (frame size)
-CRF (increase for lower bitrate videos)

This is built to generate commands for a very small number of encodes and is not a substitute 
for knowledgable use of ffmpeg if one's intention is to broaden use beyond the very limited set of 
endpoints the edX platform provides

input, via two classes, encode and video, which can be generated either via the node independently
or via celery connection to VEDA (VEDA will send video_id and encode_profile via Celery queue)

"""
from global_vars import *
from reporting import ErrorObject, Output
from config import WorkerSetup
WS = WorkerSetup()
if os.path.exists(WS.instance_yaml):
    WS.run()
settings = WS.settings_dict




class CommandGenerate():

    def __init__(self, VideoObject, EncodeObject, **kwargs):
        self.VideoObject = VideoObject
        self.EncodeObject = EncodeObject
        self.jobid = kwargs.get('jobid', None)
        self.ffcommand = []
コード例 #15
0
    def run(self):

        WS = WorkerSetup()
        if self.setup is True:
            WS.setup = True

        WS.run()
        self.settings = WS.settings_dict

        if self.encode_profile is None:
            ErrorObject().print_error(
                message = 'No Encode Profile Specified'
                )
            return None

        self.VideoObject = Video(
            veda_id=self.veda_id
            )
        self.VideoObject.activate()
        if self.VideoObject.valid is False:
            ErrorObject().print_error(
                message = 'Invalid Video / VEDA Data'
                )
            return None

        if not os.path.exists(self.workdir):
            os.mkdir(self.workdir)


        """
        Pipeline Steps :
          I. Intake
            Ib. Validate Mezz
          II. change status in APIs
          III. Generate Encode Command
          IV. Execute Encodes
            IVa. Validate Products
          (*)V. Deliver Encodes (sftp and others?), retrieve URLs
          (*)VI. Change Status in APIs, add URLs
          VII. Clean Directory

        """
        self._ENG_INTAKE()

        if self.VideoObject.valid is False:
            ErrorObject().print_error(
                message = 'Invalid Video / Local'
                )
            return None

        self._UPDATE_API()

        if self.encode_profile == 'hls':
            self._HLSPipeline()
        else:
            self._StaticPipeline()

        print self.endpoint_url

        if self.endpoint_url is not None:
            """
            Integrate with main
            """
            veda_id = self.veda_id
            encode_profile = self.encode_profile
            celeryapp.deliverable_route.apply_async(
                (veda_id, encode_profile),
                queue='transcode_stat'
                )
        """
        Clean up workdir
        """
        if self.jobid is not None:
            shutil.rmtree(
                self.workdir
                )
コード例 #16
0
ファイル: test_asset_connect.py プロジェクト: yro/veda_worker
 def setUp(self):
     self.WS = WorkerSetup()
     if os.path.exists(self.WS.instance_yaml):
         self.WS.run()
     self.settings = self.WS.settings_dict
コード例 #17
0
ファイル: test_api_connect.py プロジェクト: yro/veda_worker
class TestAPIConnection(unittest.TestCase):

    def setUp(self):
        self.WS = WorkerSetup()
        if os.path.exists(self.WS.instance_yaml):
            self.WS.run()
        self.settings = self.WS.settings_dict


    def test_val_setup(self):
        if not os.path.exists(self.WS.instance_yaml):
            self.assertTrue(True)
            return None

        salient_variables = [
            'val_api_url', 
            'val_client_id',
            'val_password',
            'val_secret_key', 
            'val_username',
            'val_token_url',
            ]
        for s in salient_variables:
            self.assertTrue(len(self.WS.settings_dict[s]) > 0)


    def test_veda_setup(self):
        if not os.path.exists(self.WS.instance_yaml):
            self.assertTrue(True)
            return None

        salient_variables = [
            'veda_api_url', 
            'veda_auth_url',
            'veda_client_id',
            'veda_secret_key', 
            'veda_token_url',
            ]
        for s in salient_variables:
            self.assertTrue(len(self.WS.settings_dict[s]) > 0)



    def test_val_connection(self):
        if not os.path.exists(self.WS.instance_yaml):
            self.assertTrue(True)
            return None

        val_token = generate_apitoken.val_tokengen()
        self.assertFalse(val_token == None)

        headers = {
            'Authorization': 'Bearer ' + val_token, 
            'content-type': 'application/json'
            }
        s = requests.get(self.WS.settings_dict['val_api_url'], headers=headers, timeout=20)

        self.assertFalse(s.status_code == 404)
        self.assertFalse(s.status_code > 299)


    def test_veda_connection(self):
        if not os.path.exists(self.WS.instance_yaml):
            self.assertTrue(True)
            return None

        veda_token = generate_apitoken.veda_tokengen()
        self.assertFalse(veda_token == None)
        headers = {
            'Authorization': 'Token ' + veda_token, # + veda_token,
            'content-type': 'application/json'
            }
        s = requests.get(self.WS.settings_dict['veda_api_url'] + '/', headers=headers, timeout=20)

        self.assertFalse(s.status_code == 404)
        self.assertFalse(s.status_code > 299)
コード例 #18
0
 def setUp(self):
     self.WS = WorkerSetup()
     if os.path.exists(self.WS.instance_yaml):
         self.WS.run()
     self.settings = self.WS.settings_dict