def about_video_ingest_daemon(self):
x = 0
while True:
FD = FileDiscovery(node_work_directory=WORK_DIRECTORY)
FD.about_video_ingest()
reset_queries()
x += 1
if x >= 100:
LOGGER.info('Memory usage: %s (kb)' %
resource.getrusage(resource.RUSAGE_SELF).ru_maxrss)
x = 0
def _ingest_from_s3_bucket(self, request_body):
"""
Handle ingest from s3 bucket.
"""
bucket_name = CONFIG['edx_s3_ingest_bucket']
status = 400
reason = ''
request_message = request_body.get('Message')
try:
message_json = json.loads(request_message)
s3_object = message_json.get('Records')[0].get('s3')
video_s3_key = s3_object.get('object').get('key')
except TypeError:
reason = 'Request message body does not contain expected output'
LOGGER.error('[HTTP INGEST] {reason}'.format(reason=reason, ))
return status, reason
if not video_s3_key:
return status, 'Video does not contain s3 key'
try:
connection = boto.connect_s3()
bucket = connection.get_bucket(bucket_name)
bucket_location = bucket.get_location()
if bucket_location:
conn = boto.s3.connect_to_region(bucket_location)
bucket = conn.get_bucket(bucket_name)
vd_key = bucket.get_key(video_s3_key)
if vd_key is None:
LOGGER.error(
'[INGEST] Video key {vd_key} supplied in notification but not found in the s3 bucket.'
'Returning 200 to avoid retry attempts.'.format(
vd_key=vd_key))
successful_ingest = 200
else:
file_discovery = FileDiscovery()
file_discovery.bucket = bucket
successful_ingest = file_discovery.validate_metadata_and_feed_to_ingest(
video_s3_key=vd_key)
status = 200 if successful_ingest else 400
except S3ResponseError:
LOGGER.error('[INGEST] S3 Ingest Connection Failure')
reason = 'S3 ingest connection failure'
return status, reason
def ingest_video_and_upload_to_hotstore(requested_key):
LOGGER.info('ingest_video_and_upload_to_hotstore key %s' % requested_key)
bucket_name = auth_dict['edx_s3_ingest_bucket']
try:
connection = boto.connect_s3()
bucket = connection.get_bucket(bucket_name)
s3_key = bucket.get_key(requested_key)
except S3ResponseError:
LOGGER.error('[INGEST CELERY TASK] Could not connect to S3, key %s' % requested_key)
return
file_discovery = FileDiscovery()
file_discovery.bucket = bucket
successful_ingest = file_discovery.validate_metadata_and_feed_to_ingest(video_s3_key=s3_key)
if not successful_ingest:
LOGGER.error('[INGEST CELERY TASK] Ingest failed for key %s' % requested_key)
return
def handle(self, *args, **options):
"""
handle method for command class.
"""
LOGGER.info('[Re-ingest from hotstore] Process started.')
start_id = options.get('start_video_id')
end_id = options.get('end_video_id')
query = Video.objects.filter(id__range=(start_id, end_id))
conn = boto.connect_s3()
bucket = conn.get_bucket(BUCKET_NAME)
for vd in query:
keyname = vd.edx_id + '.' + vd.video_orig_extension
filename = './' + keyname
vd_key = bucket.get_key(keyname)
file_discovery = FileDiscovery()
file_discovery.validate_metadata_and_feed_to_ingest(vd_key)
LOGGER.info('Ingest completed for {}'.format(vd.edx_id))