def upload_to_s3(key):
    temp_vid_slug = key + '_tmp'
    if os.path.isdir(settings.LIVE_VIDEOS_PATH):
        send_to_s3(temp_vid_slug, os.path.join(settings.LIVE_VIDEOS_PATH, choice(video_list)))
    else:
        send_to_s3(temp_vid_slug, settings.FILE_LOCATION)
    enqueue(temp_vid_slug)
Exemplo n.º 2
0
    def handle_noargs(self, **options):
        #check if rabbitmq is working
        try:
            connection = pika.BlockingConnection(
                pika.ConnectionParameters(settings.RABBITMQ_SERVER))
        except:
            logger.critical(
                "We are experiencing a connection problem and cannot perform uploads for now. Please try again later."
            )

        logger.info("Uploading file to s3...")
        upload_to_s3(FILE_KEY, FILE_LOCATION)
        logger.info("Upload process successful.")

        logger.info("Enqueuing slug to rabbitmq...")
        enqueue(FILE_KEY)
        logger.info("Enqueue process successful.")

        #should indicate that video was uploaded and is stored in s3
        logger.info("Checking if video is already in s3 (should be there)...")
        check_from_s3(FILE_KEY)
        #check if an expiring url can be generated, similar to how Screenbird gets url for video playing
        logger.info("Checking generation of expiring url...")
        try:
            conn = Connection(is_secure=True)
            url = conn.generate_object_expiring_url(FILE_KEY)
            if not url:
                logger.critical("Failed to generate expiring url.")
                raise Exception
        except:
            logger.critical("Failed to check for an expiring url.")
        logger.info("Expiring url successfully generated.")

        logger.info("Deleting file from s3...")
        delete_from_s3(FILE_KEY)
        logger.info("Delete process successful.")

        #should indicate that video is not anymore stored in s3 because of the delete
        logger.info(
            "Checking if video is still in s3 (should NOT be there anymore)..."
        )
        check_from_s3(FILE_KEY)
Exemplo n.º 3
0
def handle_uploaded_file(video, slug):
    """
    Handler for uploaded video via the java recorder.
    Sends the video to s3 and queues it for encoding on the ec2 nodes.
    
    """
    try:
        connection = pika.BlockingConnection(
            pika.ConnectionParameters(settings.RABBITMQ_SERVER))
    except:
        logger.exception('connection problem')
        raise forms.ValidationError(
            "We are experiencing a connection problem "
            "and cannot perform uploads for now. Please try again later.")
    filename = os.path.join(settings.MEDIA_ROOT, 'tmp/%s.mp4' % slug)
    logger.debug('Opening file %s' % filename)
    tmp_file = open(filename, 'wb+')
    logger.debug('file opened')
    tmp_slug = "%s_tmp" % slug
    for chunk in video.chunks():
        tmp_file.write(chunk)
    tmp_file.close()
    logger.debug('sending to s3')

    if settings.PUSH_TO_S3:
        send_to_s3(tmp_slug, filename)
    logger.debug('done sending')

    logger.debug('enqueuing file')
    enqueue(tmp_slug)
    logger.debug('done enqueue')

    logger.debug('removing file')
    # Remove the file from the server after being uploaded to s3 and
    # queued for encoding
    if settings.PUSH_TO_S3:
        os.remove(filename)
    logger.debug('done removing')
Exemplo n.º 4
0
    def handle_noargs(self, **options):
        #check if rabbitmq is working
        try:
            connection = pika.BlockingConnection(pika.ConnectionParameters(settings.RABBITMQ_SERVER))
        except:
            logger.critical("We are experiencing a connection problem and cannot perform uploads for now. Please try again later.")
        
        logger.info("Uploading file to s3...")
        upload_to_s3(FILE_KEY, FILE_LOCATION)
        logger.info("Upload process successful.")
        
        logger.info("Enqueuing slug to rabbitmq...")
        enqueue(FILE_KEY)
        logger.info("Enqueue process successful.")

        #should indicate that video was uploaded and is stored in s3
        logger.info("Checking if video is already in s3 (should be there)...")
        check_from_s3(FILE_KEY)
        #check if an expiring url can be generated, similar to how Screenbird gets url for video playing
        logger.info("Checking generation of expiring url...")
        try:
            conn = Connection(is_secure=True)
            url = conn.generate_object_expiring_url(FILE_KEY)
            if not url:
                logger.critical("Failed to generate expiring url.")
                raise Exception
        except:
            logger.critical("Failed to check for an expiring url.")
        logger.info("Expiring url successfully generated.")
           
        logger.info("Deleting file from s3...")         
        delete_from_s3(FILE_KEY)
        logger.info("Delete process successful.")
        
        #should indicate that video is not anymore stored in s3 because of the delete
        logger.info("Checking if video is still in s3 (should NOT be there anymore)...")
        check_from_s3(FILE_KEY)
Exemplo n.º 5
0
        video.save()
        logger.info("Video slug: %s" % slug)
        slug_tmp = slug + "_tmp"

        logger.info("Uploading file to s3...")
        upload_to_s3(slug_tmp, LONG_VIDEO)
        logger.info("Upload process successful.")

        # should indicate that video was uploaded and is stored in s3
        logger.info("Checking if video is already in s3 (should be there)...")
        check = check_from_s3(slug_tmp)
        if check != "Video is in s3":
            raise AssertionError("Video was not uploaded to s3 properly.")

        logger.info("Enqueuing slug to rabbitmq...")
        enqueue(slug_tmp)
        logger.info("Enqueue process successful.")

        # Mark reserved slug as used
        try:
            reserved_slug = ReservedSlug.objects.get(slug=slug)
        except:
            pass
        else:
            reserved_slug.used = True
            reserved_slug.save()

        # Simulating waiting for encoding to finish
        logger.info("Waiting for encoding to finish.")
        wait_time = math.ceil((LONG_VIDEO_ENCODING * 2) * 60)
        logger.info("ETA: %s" % wait_time)
Exemplo n.º 6
0
    def handle_noargs(self, **options):
        short_video_size_passed = False
        short_video_duration_passed = False
        short_video_encode_time_passed = False

        long_video_size_passed = False
        long_video_duration_passed = False
        long_video_encode_time_passed = False

        #check if rabbitmq is working
        try:
            connection = pika.BlockingConnection(
                pika.ConnectionParameters(settings.RABBITMQ_SERVER))
        except:
            logger.critical(
                "We are experiencing a connection problem and cannot perform uploads for now. Please try again later."
            )

        # Get uploader
        uploader, created = User.objects.get_or_create(username='******')

        # Get video
        slug = reserve_slug(SLUG_LENGTH)
        title = "Encode Test A"
        description = "5-minute video with audio"
        video = Video(title=title,
                      slug=slug,
                      uploader=uploader,
                      description=description)
        video.save()
        print("Video slug: %s" % slug)
        slug_tmp = slug + '_tmp'

        print("Uploading file to s3...")
        upload_to_s3(slug_tmp, LONG_VIDEO)
        print("Upload process successful.")

        print("Enqueuing slug to rabbitmq...")
        enqueue(slug_tmp)
        print("Enqueue process successful.")

        # Mark reserved slug as used
        try:
            reserved_slug = ReservedSlug.objects.get(slug=slug)
        except:
            pass
        else:
            reserved_slug.used = True
            reserved_slug.save()

        while not is_encoded(slug):
            time.sleep(3)

        print("Encoded!")

        # Checks
        video = Video.objects.get(slug=slug)
        video_duration = video.video_duration
        if Decimal(str(video_duration)) != Decimal(str(LONG_VIDEO_LENGTH)):
            print("Long video duration check failed.")
            print("%s vs %s" % (video_duration, LONG_VIDEO_LENGTH))
        else:
            print("Long video duration check passed.")
            long_video_duration_passed = True

        video_status = VideoStatus.objects.get(video_slug=slug)
        duration_difference = math.fabs(
            Decimal(str(video_status.encode_duration)) -
            Decimal(str(LONG_VIDEO_ENCODING)))
        percent_error = (duration_difference / LONG_VIDEO_ENCODING) * 100
        if percent_error > 30.0:
            print("Long video encoding time check failed.")
        else:
            print("Long video encoding time check passed.")
            long_video_encode_time_passed = True
        print("Percent error: %s" % percent_error)

        video_file_len = len(get_from_s3(slug))
        size_difference = math.fabs(video_file_len - LONG_VIDEO_SIZE)
        percent_error = (size_difference / LONG_VIDEO_SIZE) * 100
        if percent_error > 5.0:
            print("Long video encoded size check failed.")
        else:
            print("Long video encoded size check passed.")
            long_video_size_passed = True
        print("Percent error: %s" % percent_error)

        # Get video
        slug = reserve_slug(SLUG_LENGTH)
        title = "Encode Test B"
        description = "2-minute video with no audio"
        video = Video(title=title,
                      slug=slug,
                      uploader=uploader,
                      description=description)
        video.save()
        print("Video slug: %s" % slug)
        slug_tmp = slug + '_tmp'

        print("Uploading file to s3...")
        upload_to_s3(slug_tmp, SHORT_VIDEO)
        print("Upload process successful.")

        print("Enqueuing slug to rabbitmq...")
        enqueue(slug_tmp)
        print("Enqueue process successful.")

        # Mark reserved slug as used
        try:
            reserved_slug = ReservedSlug.objects.get(slug=slug)
        except:
            pass
        else:
            reserved_slug.used = True
            reserved_slug.save()

        while not is_encoded(slug):
            time.sleep(3)

        print("Encoded!")

        # Checks
        video = Video.objects.get(slug=slug)
        video_duration = video.video_duration
        if Decimal(str(video_duration)) != Decimal(str(SHORT_VIDEO_LENGTH)):
            print("Short video duration check failed.")
            print("%s vs %s" % (video_duration, SHORT_VIDEO_LENGTH))
        else:
            print("Short video duration check passed.")
            short_video_duration_passed = True

        video_status = VideoStatus.objects.get(video_slug=slug)
        duration_difference = math.fabs(
            Decimal(str(video_status.encode_duration)) -
            Decimal(str(SHORT_VIDEO_ENCODING)))
        percent_error = (duration_difference / SHORT_VIDEO_ENCODING) * 100
        if percent_error > 30.0:
            print("Short video encoding time check failed.")
        else:
            print("Short video encoding time check passed.")
            short_video_encode_time_passed = True
        print("Percent error: %s" % percent_error)

        video_file_len = len(get_from_s3(slug))
        size_difference = math.fabs(video_file_len - SHORT_VIDEO_SIZE)
        percent_error = (size_difference / SHORT_VIDEO_SIZE) * 100
        if percent_error > 5.0:
            print("Short video encoded size check failed.")
        else:
            print("Short video encoded size check passed.")
            short_video_size_passed = True
        print("Percent error: %s" % percent_error)

        if not ((long_video_duration_passed and long_video_encode_time_passed
                 and long_video_size_passed) and
                (short_video_duration_passed and short_video_encode_time_passed
                 and short_video_size_passed)):
            print("Encoding tests failed!")
            message = \
"""
5-minute video with audio
    Video time check passed: %s
    Video encoding time check passed: %s
    Video size check passed: %s

2-minute video with audio
    Video time check passed: %s
    Video encoding time check passed: %s
    Video size check passed: %s
""" % (long_video_duration_passed, long_video_encode_time_passed, long_video_size_passed, short_video_duration_passed, short_video_encode_time_passed, short_video_size_passed)

            email = EmailMessage(
                subject="Encoding tests failed",
                body=message,
                from_email="*****@*****.**",
                # to = ["*****@*****.**" % settings.ASSEMBLA_SPACE,],
                to=[
                    "*****@*****.**",
                ])
        else:
            print("Encoding tests passed!")
Exemplo n.º 7
0
        video.save()
        logger.info("Video slug: %s" % slug)
        slug_tmp = slug + '_tmp'

        logger.info("Uploading file to s3...")
        upload_to_s3(slug_tmp, LONG_VIDEO)
        logger.info("Upload process successful.")

        #should indicate that video was uploaded and is stored in s3
        logger.info("Checking if video is already in s3 (should be there)...")
        check = check_from_s3(slug_tmp)
        if check != 'Video is in s3':
            raise AssertionError("Video was not uploaded to s3 properly.")

        logger.info("Enqueuing slug to rabbitmq...")
        enqueue(slug_tmp)
        logger.info("Enqueue process successful.")

        # Mark reserved slug as used
        try:
            reserved_slug = ReservedSlug.objects.get(slug=slug)
        except:
            pass
        else:
            reserved_slug.used = True
            reserved_slug.save()

        # Simulating waiting for encoding to finish
        logger.info("Waiting for encoding to finish.")
        wait_time = math.ceil((LONG_VIDEO_ENCODING * 2) * 60)
        logger.info("ETA: %s" % wait_time)
Exemplo n.º 8
0
    def handle_noargs(self, **options):
        short_video_size_passed = False
        short_video_duration_passed = False
        short_video_encode_time_passed = False

        long_video_size_passed = False
        long_video_duration_passed = False
        long_video_encode_time_passed = False

        #check if rabbitmq is working
        try:
            connection = pika.BlockingConnection(pika.ConnectionParameters(settings.RABBITMQ_SERVER))
        except:
            logger.critical("We are experiencing a connection problem and cannot perform uploads for now. Please try again later.")

        # Get uploader
        uploader, created = User.objects.get_or_create(username='******')

        # Get video
        slug = reserve_slug(SLUG_LENGTH)
        title = "Encode Test A"
        description = "5-minute video with audio"
        video = Video(title=title, slug=slug, uploader=uploader, description=description)
        video.save()
        print ("Video slug: %s" % slug)
        slug_tmp = slug + '_tmp'

        print ("Uploading file to s3...")
        upload_to_s3(slug_tmp, LONG_VIDEO)
        print ("Upload process successful.")
        
        print ("Enqueuing slug to rabbitmq...")
        enqueue(slug_tmp)
        print ("Enqueue process successful.")

        # Mark reserved slug as used
        try:
            reserved_slug = ReservedSlug.objects.get(slug=slug)
        except:
            pass
        else:
            reserved_slug.used = True
            reserved_slug.save()

        while not is_encoded(slug):
            time.sleep(3)

        print ("Encoded!")

        # Checks
        video = Video.objects.get(slug=slug)
        video_duration = video.video_duration
        if Decimal(str(video_duration)) != Decimal(str(LONG_VIDEO_LENGTH)):
            print ("Long video duration check failed.")
            print ("%s vs %s" % (video_duration, LONG_VIDEO_LENGTH))
        else:
            print ("Long video duration check passed.")
            long_video_duration_passed = True

        video_status = VideoStatus.objects.get(video_slug=slug)
        duration_difference = math.fabs(Decimal(str(video_status.encode_duration)) - Decimal(str(LONG_VIDEO_ENCODING)))
        percent_error = (duration_difference / LONG_VIDEO_ENCODING) * 100
        if percent_error > 30.0:
            print ("Long video encoding time check failed.")
        else:
            print ("Long video encoding time check passed.")
            long_video_encode_time_passed = True
        print ("Percent error: %s" % percent_error)

        video_file_len = len(get_from_s3(slug))
        size_difference = math.fabs(video_file_len - LONG_VIDEO_SIZE)
        percent_error = (size_difference / LONG_VIDEO_SIZE) * 100
        if percent_error > 5.0:
            print ("Long video encoded size check failed.")
        else:
            print ("Long video encoded size check passed.")
            long_video_size_passed = True
        print ("Percent error: %s" % percent_error)

        # Get video
        slug = reserve_slug(SLUG_LENGTH)
        title = "Encode Test B"
        description = "2-minute video with no audio"
        video = Video(title=title, slug=slug, uploader=uploader, description=description)
        video.save()
        print ("Video slug: %s" % slug)
        slug_tmp = slug + '_tmp'

        print ("Uploading file to s3...")
        upload_to_s3(slug_tmp, SHORT_VIDEO)
        print ("Upload process successful.")
        
        print ("Enqueuing slug to rabbitmq...")
        enqueue(slug_tmp)
        print ("Enqueue process successful.")

        # Mark reserved slug as used
        try:
            reserved_slug = ReservedSlug.objects.get(slug=slug)
        except:
            pass
        else:
            reserved_slug.used = True
            reserved_slug.save()

        while not is_encoded(slug):
            time.sleep(3)

        print ("Encoded!")

        # Checks
        video = Video.objects.get(slug=slug)
        video_duration = video.video_duration
        if Decimal(str(video_duration)) != Decimal(str(SHORT_VIDEO_LENGTH)):
            print ("Short video duration check failed.")
            print ("%s vs %s" % (video_duration, SHORT_VIDEO_LENGTH))
        else:
            print ("Short video duration check passed.")
            short_video_duration_passed = True

        video_status = VideoStatus.objects.get(video_slug=slug)
        duration_difference = math.fabs(Decimal(str(video_status.encode_duration)) - Decimal(str(SHORT_VIDEO_ENCODING)))
        percent_error = (duration_difference / SHORT_VIDEO_ENCODING) * 100
        if percent_error > 30.0:
            print ("Short video encoding time check failed.")
        else:
            print ("Short video encoding time check passed.")
            short_video_encode_time_passed = True
        print ("Percent error: %s" % percent_error)

        video_file_len = len(get_from_s3(slug))
        size_difference = math.fabs(video_file_len - SHORT_VIDEO_SIZE)
        percent_error = (size_difference / SHORT_VIDEO_SIZE) * 100
        if percent_error > 5.0:
            print ("Short video encoded size check failed.")
        else:
            print ("Short video encoded size check passed.")
            short_video_size_passed = True
        print ("Percent error: %s" % percent_error)

        if not ((long_video_duration_passed and long_video_encode_time_passed and long_video_size_passed) and (short_video_duration_passed and short_video_encode_time_passed and short_video_size_passed)):
            print ("Encoding tests failed!")
            message = \
"""
5-minute video with audio
    Video time check passed: %s
    Video encoding time check passed: %s
    Video size check passed: %s

2-minute video with audio
    Video time check passed: %s
    Video encoding time check passed: %s
    Video size check passed: %s
""" % (long_video_duration_passed, long_video_encode_time_passed, long_video_size_passed, short_video_duration_passed, short_video_encode_time_passed, short_video_size_passed)

            email = EmailMessage(
                subject = "Encoding tests failed",
                body = message,
                from_email = "*****@*****.**",
                # to = ["*****@*****.**" % settings.ASSEMBLA_SPACE,],
                to = ["*****@*****.**",]
            )
        else:
            print ("Encoding tests passed!")