def upload_to_s3(key):
temp_vid_slug = key + '_tmp'
if os.path.isdir(settings.LIVE_VIDEOS_PATH):
send_to_s3(temp_vid_slug, os.path.join(settings.LIVE_VIDEOS_PATH, choice(video_list)))
else:
send_to_s3(temp_vid_slug, settings.FILE_LOCATION)
enqueue(temp_vid_slug)
def handle_noargs(self, **options):
#check if rabbitmq is working
try:
connection = pika.BlockingConnection(
pika.ConnectionParameters(settings.RABBITMQ_SERVER))
except:
logger.critical(
"We are experiencing a connection problem and cannot perform uploads for now. Please try again later."
)
logger.info("Uploading file to s3...")
upload_to_s3(FILE_KEY, FILE_LOCATION)
logger.info("Upload process successful.")
logger.info("Enqueuing slug to rabbitmq...")
enqueue(FILE_KEY)
logger.info("Enqueue process successful.")
#should indicate that video was uploaded and is stored in s3
logger.info("Checking if video is already in s3 (should be there)...")
check_from_s3(FILE_KEY)
#check if an expiring url can be generated, similar to how Screenbird gets url for video playing
logger.info("Checking generation of expiring url...")
try:
conn = Connection(is_secure=True)
url = conn.generate_object_expiring_url(FILE_KEY)
if not url:
logger.critical("Failed to generate expiring url.")
raise Exception
except:
logger.critical("Failed to check for an expiring url.")
logger.info("Expiring url successfully generated.")
logger.info("Deleting file from s3...")
delete_from_s3(FILE_KEY)
logger.info("Delete process successful.")
#should indicate that video is not anymore stored in s3 because of the delete
logger.info(
"Checking if video is still in s3 (should NOT be there anymore)..."
)
check_from_s3(FILE_KEY)
def handle_uploaded_file(video, slug):
"""
Handler for uploaded video via the java recorder.
Sends the video to s3 and queues it for encoding on the ec2 nodes.
"""
try:
connection = pika.BlockingConnection(
pika.ConnectionParameters(settings.RABBITMQ_SERVER))
except:
logger.exception('connection problem')
raise forms.ValidationError(
"We are experiencing a connection problem "
"and cannot perform uploads for now. Please try again later.")
filename = os.path.join(settings.MEDIA_ROOT, 'tmp/%s.mp4' % slug)
logger.debug('Opening file %s' % filename)
tmp_file = open(filename, 'wb+')
logger.debug('file opened')
tmp_slug = "%s_tmp" % slug
for chunk in video.chunks():
tmp_file.write(chunk)
tmp_file.close()
logger.debug('sending to s3')
if settings.PUSH_TO_S3:
send_to_s3(tmp_slug, filename)
logger.debug('done sending')
logger.debug('enqueuing file')
enqueue(tmp_slug)
logger.debug('done enqueue')
logger.debug('removing file')
# Remove the file from the server after being uploaded to s3 and
# queued for encoding
if settings.PUSH_TO_S3:
os.remove(filename)
logger.debug('done removing')
def handle_noargs(self, **options):
#check if rabbitmq is working
try:
connection = pika.BlockingConnection(pika.ConnectionParameters(settings.RABBITMQ_SERVER))
except:
logger.critical("We are experiencing a connection problem and cannot perform uploads for now. Please try again later.")
logger.info("Uploading file to s3...")
upload_to_s3(FILE_KEY, FILE_LOCATION)
logger.info("Upload process successful.")
logger.info("Enqueuing slug to rabbitmq...")
enqueue(FILE_KEY)
logger.info("Enqueue process successful.")
#should indicate that video was uploaded and is stored in s3
logger.info("Checking if video is already in s3 (should be there)...")
check_from_s3(FILE_KEY)
#check if an expiring url can be generated, similar to how Screenbird gets url for video playing
logger.info("Checking generation of expiring url...")
try:
conn = Connection(is_secure=True)
url = conn.generate_object_expiring_url(FILE_KEY)
if not url:
logger.critical("Failed to generate expiring url.")
raise Exception
except:
logger.critical("Failed to check for an expiring url.")
logger.info("Expiring url successfully generated.")
logger.info("Deleting file from s3...")
delete_from_s3(FILE_KEY)
logger.info("Delete process successful.")
#should indicate that video is not anymore stored in s3 because of the delete
logger.info("Checking if video is still in s3 (should NOT be there anymore)...")
check_from_s3(FILE_KEY)
video.save()
logger.info("Video slug: %s" % slug)
slug_tmp = slug + "_tmp"
logger.info("Uploading file to s3...")
upload_to_s3(slug_tmp, LONG_VIDEO)
logger.info("Upload process successful.")
# should indicate that video was uploaded and is stored in s3
logger.info("Checking if video is already in s3 (should be there)...")
check = check_from_s3(slug_tmp)
if check != "Video is in s3":
raise AssertionError("Video was not uploaded to s3 properly.")
logger.info("Enqueuing slug to rabbitmq...")
enqueue(slug_tmp)
logger.info("Enqueue process successful.")
# Mark reserved slug as used
try:
reserved_slug = ReservedSlug.objects.get(slug=slug)
except:
pass
else:
reserved_slug.used = True
reserved_slug.save()
# Simulating waiting for encoding to finish
logger.info("Waiting for encoding to finish.")
wait_time = math.ceil((LONG_VIDEO_ENCODING * 2) * 60)
logger.info("ETA: %s" % wait_time)
def handle_noargs(self, **options):
short_video_size_passed = False
short_video_duration_passed = False
short_video_encode_time_passed = False
long_video_size_passed = False
long_video_duration_passed = False
long_video_encode_time_passed = False
#check if rabbitmq is working
try:
connection = pika.BlockingConnection(
pika.ConnectionParameters(settings.RABBITMQ_SERVER))
except:
logger.critical(
"We are experiencing a connection problem and cannot perform uploads for now. Please try again later."
)
# Get uploader
uploader, created = User.objects.get_or_create(username='******')
# Get video
slug = reserve_slug(SLUG_LENGTH)
title = "Encode Test A"
description = "5-minute video with audio"
video = Video(title=title,
slug=slug,
uploader=uploader,
description=description)
video.save()
print("Video slug: %s" % slug)
slug_tmp = slug + '_tmp'
print("Uploading file to s3...")
upload_to_s3(slug_tmp, LONG_VIDEO)
print("Upload process successful.")
print("Enqueuing slug to rabbitmq...")
enqueue(slug_tmp)
print("Enqueue process successful.")
# Mark reserved slug as used
try:
reserved_slug = ReservedSlug.objects.get(slug=slug)
except:
pass
else:
reserved_slug.used = True
reserved_slug.save()
while not is_encoded(slug):
time.sleep(3)
print("Encoded!")
# Checks
video = Video.objects.get(slug=slug)
video_duration = video.video_duration
if Decimal(str(video_duration)) != Decimal(str(LONG_VIDEO_LENGTH)):
print("Long video duration check failed.")
print("%s vs %s" % (video_duration, LONG_VIDEO_LENGTH))
else:
print("Long video duration check passed.")
long_video_duration_passed = True
video_status = VideoStatus.objects.get(video_slug=slug)
duration_difference = math.fabs(
Decimal(str(video_status.encode_duration)) -
Decimal(str(LONG_VIDEO_ENCODING)))
percent_error = (duration_difference / LONG_VIDEO_ENCODING) * 100
if percent_error > 30.0:
print("Long video encoding time check failed.")
else:
print("Long video encoding time check passed.")
long_video_encode_time_passed = True
print("Percent error: %s" % percent_error)
video_file_len = len(get_from_s3(slug))
size_difference = math.fabs(video_file_len - LONG_VIDEO_SIZE)
percent_error = (size_difference / LONG_VIDEO_SIZE) * 100
if percent_error > 5.0:
print("Long video encoded size check failed.")
else:
print("Long video encoded size check passed.")
long_video_size_passed = True
print("Percent error: %s" % percent_error)
# Get video
slug = reserve_slug(SLUG_LENGTH)
title = "Encode Test B"
description = "2-minute video with no audio"
video = Video(title=title,
slug=slug,
uploader=uploader,
description=description)
video.save()
print("Video slug: %s" % slug)
slug_tmp = slug + '_tmp'
print("Uploading file to s3...")
upload_to_s3(slug_tmp, SHORT_VIDEO)
print("Upload process successful.")
print("Enqueuing slug to rabbitmq...")
enqueue(slug_tmp)
print("Enqueue process successful.")
# Mark reserved slug as used
try:
reserved_slug = ReservedSlug.objects.get(slug=slug)
except:
pass
else:
reserved_slug.used = True
reserved_slug.save()
while not is_encoded(slug):
time.sleep(3)
print("Encoded!")
# Checks
video = Video.objects.get(slug=slug)
video_duration = video.video_duration
if Decimal(str(video_duration)) != Decimal(str(SHORT_VIDEO_LENGTH)):
print("Short video duration check failed.")
print("%s vs %s" % (video_duration, SHORT_VIDEO_LENGTH))
else:
print("Short video duration check passed.")
short_video_duration_passed = True
video_status = VideoStatus.objects.get(video_slug=slug)
duration_difference = math.fabs(
Decimal(str(video_status.encode_duration)) -
Decimal(str(SHORT_VIDEO_ENCODING)))
percent_error = (duration_difference / SHORT_VIDEO_ENCODING) * 100
if percent_error > 30.0:
print("Short video encoding time check failed.")
else:
print("Short video encoding time check passed.")
short_video_encode_time_passed = True
print("Percent error: %s" % percent_error)
video_file_len = len(get_from_s3(slug))
size_difference = math.fabs(video_file_len - SHORT_VIDEO_SIZE)
percent_error = (size_difference / SHORT_VIDEO_SIZE) * 100
if percent_error > 5.0:
print("Short video encoded size check failed.")
else:
print("Short video encoded size check passed.")
short_video_size_passed = True
print("Percent error: %s" % percent_error)
if not ((long_video_duration_passed and long_video_encode_time_passed
and long_video_size_passed) and
(short_video_duration_passed and short_video_encode_time_passed
and short_video_size_passed)):
print("Encoding tests failed!")
message = \
"""
5-minute video with audio
Video time check passed: %s
Video encoding time check passed: %s
Video size check passed: %s
2-minute video with audio
Video time check passed: %s
Video encoding time check passed: %s
Video size check passed: %s
""" % (long_video_duration_passed, long_video_encode_time_passed, long_video_size_passed, short_video_duration_passed, short_video_encode_time_passed, short_video_size_passed)
email = EmailMessage(
subject="Encoding tests failed",
body=message,
from_email="*****@*****.**",
# to = ["*****@*****.**" % settings.ASSEMBLA_SPACE,],
to=[
"*****@*****.**",
])
else:
print("Encoding tests passed!")
video.save()
logger.info("Video slug: %s" % slug)
slug_tmp = slug + '_tmp'
logger.info("Uploading file to s3...")
upload_to_s3(slug_tmp, LONG_VIDEO)
logger.info("Upload process successful.")
#should indicate that video was uploaded and is stored in s3
logger.info("Checking if video is already in s3 (should be there)...")
check = check_from_s3(slug_tmp)
if check != 'Video is in s3':
raise AssertionError("Video was not uploaded to s3 properly.")
logger.info("Enqueuing slug to rabbitmq...")
enqueue(slug_tmp)
logger.info("Enqueue process successful.")
# Mark reserved slug as used
try:
reserved_slug = ReservedSlug.objects.get(slug=slug)
except:
pass
else:
reserved_slug.used = True
reserved_slug.save()
# Simulating waiting for encoding to finish
logger.info("Waiting for encoding to finish.")
wait_time = math.ceil((LONG_VIDEO_ENCODING * 2) * 60)
logger.info("ETA: %s" % wait_time)
def handle_noargs(self, **options):
short_video_size_passed = False
short_video_duration_passed = False
short_video_encode_time_passed = False
long_video_size_passed = False
long_video_duration_passed = False
long_video_encode_time_passed = False
#check if rabbitmq is working
try:
connection = pika.BlockingConnection(pika.ConnectionParameters(settings.RABBITMQ_SERVER))
except:
logger.critical("We are experiencing a connection problem and cannot perform uploads for now. Please try again later.")
# Get uploader
uploader, created = User.objects.get_or_create(username='******')
# Get video
slug = reserve_slug(SLUG_LENGTH)
title = "Encode Test A"
description = "5-minute video with audio"
video = Video(title=title, slug=slug, uploader=uploader, description=description)
video.save()
print ("Video slug: %s" % slug)
slug_tmp = slug + '_tmp'
print ("Uploading file to s3...")
upload_to_s3(slug_tmp, LONG_VIDEO)
print ("Upload process successful.")
print ("Enqueuing slug to rabbitmq...")
enqueue(slug_tmp)
print ("Enqueue process successful.")
# Mark reserved slug as used
try:
reserved_slug = ReservedSlug.objects.get(slug=slug)
except:
pass
else:
reserved_slug.used = True
reserved_slug.save()
while not is_encoded(slug):
time.sleep(3)
print ("Encoded!")
# Checks
video = Video.objects.get(slug=slug)
video_duration = video.video_duration
if Decimal(str(video_duration)) != Decimal(str(LONG_VIDEO_LENGTH)):
print ("Long video duration check failed.")
print ("%s vs %s" % (video_duration, LONG_VIDEO_LENGTH))
else:
print ("Long video duration check passed.")
long_video_duration_passed = True
video_status = VideoStatus.objects.get(video_slug=slug)
duration_difference = math.fabs(Decimal(str(video_status.encode_duration)) - Decimal(str(LONG_VIDEO_ENCODING)))
percent_error = (duration_difference / LONG_VIDEO_ENCODING) * 100
if percent_error > 30.0:
print ("Long video encoding time check failed.")
else:
print ("Long video encoding time check passed.")
long_video_encode_time_passed = True
print ("Percent error: %s" % percent_error)
video_file_len = len(get_from_s3(slug))
size_difference = math.fabs(video_file_len - LONG_VIDEO_SIZE)
percent_error = (size_difference / LONG_VIDEO_SIZE) * 100
if percent_error > 5.0:
print ("Long video encoded size check failed.")
else:
print ("Long video encoded size check passed.")
long_video_size_passed = True
print ("Percent error: %s" % percent_error)
# Get video
slug = reserve_slug(SLUG_LENGTH)
title = "Encode Test B"
description = "2-minute video with no audio"
video = Video(title=title, slug=slug, uploader=uploader, description=description)
video.save()
print ("Video slug: %s" % slug)
slug_tmp = slug + '_tmp'
print ("Uploading file to s3...")
upload_to_s3(slug_tmp, SHORT_VIDEO)
print ("Upload process successful.")
print ("Enqueuing slug to rabbitmq...")
enqueue(slug_tmp)
print ("Enqueue process successful.")
# Mark reserved slug as used
try:
reserved_slug = ReservedSlug.objects.get(slug=slug)
except:
pass
else:
reserved_slug.used = True
reserved_slug.save()
while not is_encoded(slug):
time.sleep(3)
print ("Encoded!")
# Checks
video = Video.objects.get(slug=slug)
video_duration = video.video_duration
if Decimal(str(video_duration)) != Decimal(str(SHORT_VIDEO_LENGTH)):
print ("Short video duration check failed.")
print ("%s vs %s" % (video_duration, SHORT_VIDEO_LENGTH))
else:
print ("Short video duration check passed.")
short_video_duration_passed = True
video_status = VideoStatus.objects.get(video_slug=slug)
duration_difference = math.fabs(Decimal(str(video_status.encode_duration)) - Decimal(str(SHORT_VIDEO_ENCODING)))
percent_error = (duration_difference / SHORT_VIDEO_ENCODING) * 100
if percent_error > 30.0:
print ("Short video encoding time check failed.")
else:
print ("Short video encoding time check passed.")
short_video_encode_time_passed = True
print ("Percent error: %s" % percent_error)
video_file_len = len(get_from_s3(slug))
size_difference = math.fabs(video_file_len - SHORT_VIDEO_SIZE)
percent_error = (size_difference / SHORT_VIDEO_SIZE) * 100
if percent_error > 5.0:
print ("Short video encoded size check failed.")
else:
print ("Short video encoded size check passed.")
short_video_size_passed = True
print ("Percent error: %s" % percent_error)
if not ((long_video_duration_passed and long_video_encode_time_passed and long_video_size_passed) and (short_video_duration_passed and short_video_encode_time_passed and short_video_size_passed)):
print ("Encoding tests failed!")
message = \
"""
5-minute video with audio
Video time check passed: %s
Video encoding time check passed: %s
Video size check passed: %s
2-minute video with audio
Video time check passed: %s
Video encoding time check passed: %s
Video size check passed: %s
""" % (long_video_duration_passed, long_video_encode_time_passed, long_video_size_passed, short_video_duration_passed, short_video_encode_time_passed, short_video_size_passed)
email = EmailMessage(
subject = "Encoding tests failed",
body = message,
from_email = "*****@*****.**",
# to = ["*****@*****.**" % settings.ASSEMBLA_SPACE,],
to = ["*****@*****.**",]
)
else:
print ("Encoding tests passed!")
