def test_fetch_duration_oserror(self, mock_popen, rhead):
def mocked_head(url, **options):
return Response(
'',
200
)
rhead.side_effect = mocked_head
def mocked_popen(command, **kwargs):
class Inner:
def communicate(self):
raise OSError('Something Bad')
return Inner()
mock_popen.side_effect = mocked_popen
event = Event.objects.get(title='Test event')
video_url = 'https://example.com/file.mov'
try:
videoinfo.fetch_duration(event, video_url=video_url)
raise AssertionError("not supposed to happen")
except OSError as exception:
message = str(exception)
ok_('Something Bad' in message)
ok_('ffmpeg -i %s' % video_url in message)
def vidly_media_webhook(request):
if not request.POST.get('xml'):
return http.HttpResponseBadRequest("no 'xml'")
xml_string = request.POST['xml'].strip()
try:
struct = xmltodict.parse(xml_string)
except ExpatError:
return http.HttpResponseBadRequest("Bad 'xml'")
try:
task = struct['Response']['Result']['Task']
try:
vidly_submission = VidlySubmission.objects.get(
url=task['SourceFile'],
tag=task['MediaShortLink']
)
if task['Status'] == 'Finished':
if not vidly_submission.finished:
vidly_submission.finished = timezone.now()
vidly_submission.save()
event = vidly_submission.event
# Awesome!
# This event now has a fully working transcoded piece of
# media.
event.archive_time = timezone.now()
event.save()
# More awesome! We can start processing the transcoded media.
if not event.duration:
videoinfo.fetch_duration(
event,
save=True,
verbose=settings.DEBUG
)
event = Event.objects.get(id=event.id)
if event.duration:
if not Picture.objects.filter(event=event):
videoinfo.fetch_screencapture(
event,
save=True,
verbose=settings.DEBUG,
set_first_available=True,
)
elif task['Status'] == 'Error':
if not vidly_submission.errored:
vidly_submission.errored = timezone.now()
vidly_submission.save()
except VidlySubmission.DoesNotExist:
# remember, we can't trust the XML since it's publically
# available and exposed as a webhook
pass
except KeyError:
# If it doesn't have a "Result" or "Task", it was just a notification
# that the media was added.
pass
return http.HttpResponse('OK\n')
def event_screencaptures(request, event):
if event.status != Event.STATUS_INITIATED:
return http.HttpResponseBadRequest(
"Events NOT in the state of initiated."
)
upload = event.upload
video_url = upload.url
context = {}
cache_key = 'fetching-{0}'.format(event.id)
# This function sets the cache `fetching-{id}` before and after calling
# those functions in the videoinfo module.
# The reason is that those calls might take many many seconds
# and the webapp might send async calls to the event_picture view
# which will inform the webapp that the slow videoinfo processes
# are running and thus that the webapp shouldn't kick if off yet.
seconds = event.duration
if not event.duration:
# it's a poor man's lock
if not cache.get(cache_key):
cache.set(cache_key, True, 60)
seconds = videoinfo.fetch_duration(
event,
video_url=video_url,
save=True,
verbose=settings.DEBUG
)
cache.delete(cache_key)
event = Event.objects.get(id=event.id)
context['seconds'] = seconds
# The reason we can't use `if event.duration:` is because the
# fetch_duration() does an inline-update instead of modifying
# the instance object.
no_pictures = Picture.objects.filter(event=event).count()
if event.duration and not no_pictures:
if not cache.get(cache_key):
cache.set(cache_key, True, 60)
event = Event.objects.get(id=event.id)
no_pictures = videoinfo.fetch_screencapture(
event,
video_url=video_url,
save=True,
verbose=settings.DEBUG,
set_first_available=not event.picture,
import_immediately=True,
)
cache.delete(cache_key)
event = Event.objects.get(id=event.id)
if no_pictures and not event.picture:
# no picture has been chosen previously
pictures = Picture.objects.filter(event=event).order_by('created')[:1]
for picture in pictures:
event.picture = picture
event.save()
break
context['no_pictures'] = no_pictures
return context
def handle(self, *args, **options):
identifiers = options['slug-or-url-or-id']
if not identifiers:
raise CommandError('slug-or-url-or-id')
verbose = int(options['verbosity']) > 1
for arg in identifiers:
if arg.isdigit():
event = Event.objects.get(pk=arg)
else:
if '://' in arg:
slug = urlparse(arg).path.split('/')[1]
else:
slug = arg
event = Event.objects.get(slug=slug)
result = fetch_duration(
event,
save=not options['dry_run'],
save_locally=options['save_locally'],
verbose=verbose,
)
if verbose:
print result
def handle(self, *args, **options):
identifiers = options['slug-or-url-or-id']
if not identifiers:
raise CommandError('slug-or-url-or-id')
verbose = int(options['verbosity']) > 1
for arg in identifiers:
if arg.isdigit():
event = Event.objects.get(pk=arg)
else:
if '://' in arg:
slug = urlparse(arg).path.split('/')[1]
else:
slug = arg
event = Event.objects.get(slug=slug)
result = fetch_duration(
event,
save=not options['dry_run'],
save_locally=options['save_locally'],
verbose=verbose,
)
if verbose:
print result
def event_screencaptures(request, event):
if event.status != Event.STATUS_INITIATED:
return http.HttpResponseBadRequest(
"Events NOT in the state of initiated."
)
upload = event.upload
video_url = upload.url
context = {}
cache_key = 'fetching-{0}'.format(event.id)
# This function sets the cache `fetching-{id}` before and after calling
# those functions in the videoinfo module.
# The reason is that those calls might take many many seconds
# and the webapp might send async calls to the event_picture view
# which will inform the webapp that the slow videoinfo processes
# are running and thus that the webapp shouldn't kick if off yet.
seconds = event.duration
if not event.duration:
# it's a poor man's lock
if not cache.get(cache_key):
cache.set(cache_key, True, 60)
seconds = videoinfo.fetch_duration(
event,
video_url=video_url,
save=True,
verbose=settings.DEBUG
)
cache.delete(cache_key)
event = Event.objects.get(id=event.id)
context['seconds'] = seconds
# The reason we can't use `if event.duration:` is because the
# fetch_duration() does an inline-update instead of modifying
# the instance object.
no_pictures = Picture.objects.filter(event=event).count()
if event.duration and not no_pictures:
if not cache.get(cache_key):
cache.set(cache_key, True, 60)
event = Event.objects.get(id=event.id)
no_pictures = videoinfo.fetch_screencapture(
event,
video_url=video_url,
save=True,
verbose=settings.DEBUG,
set_first_available=not event.picture,
import_immediately=True,
)
cache.delete(cache_key)
event = Event.objects.get(id=event.id)
if no_pictures and not event.picture:
# no picture has been chosen previously
pictures = Picture.objects.filter(event=event).order_by('created')[:1]
for picture in pictures:
event.picture = picture
event.save()
break
context['no_pictures'] = no_pictures
return context
def test_fetch_duration_oserror(self, mock_popen, rhead):
def mocked_head(url, **options):
return Response('', 200)
rhead.side_effect = mocked_head
def mocked_popen(command, **kwargs):
class Inner:
def communicate(self):
raise OSError('Something Bad')
return Inner()
mock_popen.side_effect = mocked_popen
event = Event.objects.get(title='Test event')
video_url = 'https://example.com/file.mov'
try:
videoinfo.fetch_duration(event, video_url=video_url)
raise AssertionError("not supposed to happen")
except OSError as exception:
message = str(exception)
ok_('Something Bad' in message)
ok_('ffmpeg -i %s' % video_url in message)
def handle(self, *args, **options):
if not args:
raise CommandError(self.args)
verbose = int(options["verbosity"]) > 1
for arg in args:
if arg.isdigit():
event = Event.objects.get(pk=arg)
else:
if "://" in arg:
slug = urlparse(arg).path.split("/")[1]
else:
slug = arg
event = Event.objects.get(slug=slug)
result = fetch_duration(
event, save=not options["dry_run"], save_locally=options["save_locally"], verbose=verbose
)
if verbose:
print result
def vidly_media_webhook(request):
if not request.POST.get('xml'):
return http.HttpResponseBadRequest("no 'xml'")
xml_string = request.POST['xml'].strip()
try:
struct = xmltodict.parse(xml_string)
except ExpatError:
return http.HttpResponseBadRequest("Bad 'xml'")
try:
task = struct['Response']['Result']['Task']
try:
vidly_submission = VidlySubmission.objects.get(
url=task['SourceFile'], tag=task['MediaShortLink'])
if task['Status'] == 'Finished':
if not vidly_submission.finished:
vidly_submission.finished = timezone.now()
vidly_submission.save()
event = vidly_submission.event
if (task['Private'] == 'false'
and event.privacy != Event.PRIVACY_PUBLIC):
# the event is private but the video is not
vidly.update_media_protection(
vidly_submission.tag,
True # make it private
)
if not vidly_submission.token_protection:
vidly_submission.token_protection = True
vidly_submission.save()
# Awesome!
# This event now has a fully working transcoded piece of
# media.
if (event.status == Event.STATUS_PENDING
or event.status == Event.STATUS_PROCESSING):
event.status = Event.STATUS_SCHEDULED
event.archive_time = timezone.now()
event.save()
# More awesome! We can start processing the transcoded media.
# XXX move this to a background task.
if not event.duration:
videoinfo.fetch_duration(event,
save=True,
verbose=settings.DEBUG)
event = Event.objects.get(id=event.id)
if event.duration:
if not Picture.objects.filter(event=event):
videoinfo.fetch_screencapture(
event,
save=True,
verbose=settings.DEBUG,
set_first_available=True,
)
elif task['Status'] == 'Error':
if not vidly_submission.errored:
vidly_submission.errored = timezone.now()
vidly_submission.save()
except VidlySubmission.DoesNotExist:
# remember, we can't trust the XML since it's publicly
# available and exposed as a webhook
pass
except KeyError:
# If it doesn't have a "Result" or "Task", it was just a notification
# that the media was added.
pass
return http.HttpResponse('OK\n')
def legacy_video_migration(request): # pragma: no cover
"""for one off mass vid.ly submission"""
class VideoURLError(Exception):
pass
def redirect_recurse(url):
"""return the URL only when it's not redirecting.
Raise an error on all other statuses >= 400
"""
response = requests.head(url)
if response.status_code in (301, 302):
return redirect_recurse(response.headers['Location'])
elif response.status_code >= 400:
raise VideoURLError('{} => {}'.format(
url, response.status_code
))
return url
if request.method == 'POST':
events = Event.objects.filter(id__in=request.POST.getlist('ids'))
template, = Template.objects.filter(default_archive_template=True)
for event in events:
try:
url = event.template_environment['url']
url = redirect_recurse(url)
base_url = get_base_url(request)
webhook_url = base_url + reverse('manage:vidly_media_webhook')
url = prepare_vidly_video_url(url)
token_protection = event.privacy != Event.PRIVACY_PUBLIC
shortcode, error = vidly.add_media(
url,
hd=True,
token_protection=token_protection,
notify_url=webhook_url,
)
VidlySubmission.objects.create(
event=event,
url=url,
token_protection=token_protection,
hd=True,
tag=shortcode,
submission_error=error
)
event.template_environment = {
'tag': shortcode,
}
if shortcode:
event.template = template
event.archive_time = None
event.status = Event.STATUS_PROCESSING
event.save()
videoinfo.fetch_duration(
event,
video_url=url,
save=True,
verbose=settings.DEBUG
)
if Event.objects.get(id=event.id).duration:
create_all_event_pictures.delay(
event.id,
video_url=url,
)
create_all_timestamp_pictures.delay(
event.id,
video_url=url,
)
except Exception as exception:
error = str(exception)
messages.error(
request,
'Failed to submit "{}". Error: {}'.format(
event.title,
error,
)
)
messages.success(
request,
'Submitted {} events for Vid.ly processing'.format(
events.count()
)
)
return redirect('manage:legacy_video_migration')
search = request.GET.get('search', 'http://videos.mozilla.org/')
if search:
events = Event.objects.filter(
template_environment__icontains=search
)
else:
events = Event.objects.none()
context = {
'events': events,
'search': search,
}
return render(request, 'manage/legacy_video_migration.html', context)
def legacy_video_migration(request): # pragma: no cover
"""for one off mass vid.ly submission"""
class VideoURLError(Exception):
pass
def redirect_recurse(url):
"""return the URL only when it's not redirecting.
Raise an error on all other statuses >= 400
"""
response = requests.head(url)
if response.status_code in (301, 302):
return redirect_recurse(response.headers['Location'])
elif response.status_code >= 400:
raise VideoURLError('{} => {}'.format(url, response.status_code))
return url
if request.method == 'POST':
events = Event.objects.filter(id__in=request.POST.getlist('ids'))
template, = Template.objects.filter(default_archive_template=True)
for event in events:
try:
url = event.template_environment['url']
url = redirect_recurse(url)
base_url = get_base_url(request)
webhook_url = base_url + reverse('manage:vidly_media_webhook')
url = prepare_vidly_video_url(url)
token_protection = event.privacy != Event.PRIVACY_PUBLIC
shortcode, error = vidly.add_media(
url,
hd=True,
token_protection=token_protection,
notify_url=webhook_url,
)
VidlySubmission.objects.create(
event=event,
url=url,
token_protection=token_protection,
hd=True,
tag=shortcode,
submission_error=error)
event.template_environment = {
'tag': shortcode,
}
if shortcode:
event.template = template
event.archive_time = None
event.status = Event.STATUS_PROCESSING
event.save()
videoinfo.fetch_duration(event,
video_url=url,
save=True,
verbose=settings.DEBUG)
if Event.objects.get(id=event.id).duration:
create_all_event_pictures.delay(
event.id,
video_url=url,
)
create_all_timestamp_pictures.delay(
event.id,
video_url=url,
)
except Exception as exception:
error = str(exception)
messages.error(
request, 'Failed to submit "{}". Error: {}'.format(
event.title,
error,
))
messages.success(
request,
'Submitted {} events for Vid.ly processing'.format(events.count()))
return redirect('manage:legacy_video_migration')
search = request.GET.get('search', 'http://videos.mozilla.org/')
if search:
events = Event.objects.filter(template_environment__icontains=search)
else:
events = Event.objects.none()
context = {
'events': events,
'search': search,
}
return render(request, 'manage/legacy_video_migration.html', context)