def vidly_media(request): events = Event.objects.filter( Q(template__name__contains='Vid.ly') | Q(pk__in=VidlySubmission.objects.all() .values_list('event_id', flat=True)) ) status = request.GET.get('status') repeated = request.GET.get('repeated') == 'event' repeats = {} if status: if status not in ('New', 'Processing', 'Finished', 'Error'): return http.HttpResponseBadRequest("Invalid 'status' value") # make a list of all tags -> events _tags = {} for event in events: environment = event.template_environment or {} if not environment.get('tag') or environment.get('tag') == 'None': continue _tags[environment['tag']] = event.id event_ids = [] for tag in vidly.medialist(status): try: event_ids.append(_tags[tag]) except KeyError: # it's on vid.ly but not in this database logging.debug("Unknown event with tag=%r", tag) events = events.filter(id__in=event_ids) elif repeated: repeats = dict( (x['event_id'], x['event__id__count']) for x in VidlySubmission.objects .values('event_id') .annotate(Count('event__id')) .filter(event__id__count__gt=1) ) events = Event.objects.filter(id__in=repeats.keys()) def get_repeats(event): return repeats[event.id] events = events.order_by('-start_time') events = events.select_related('template') paged = paginate(events, request.GET.get('page'), 15) vidly_resubmit_form = forms.VidlyResubmitForm() context = { 'paginate': paged, 'status': status, 'vidly_resubmit_form': vidly_resubmit_form, 'repeated': repeated, 'get_repeats': get_repeats, } return render(request, 'manage/vidly_media.html', context)
def vidly_media(request): events = Event.objects.filter( Q(template__name__contains='Vid.ly') | Q(pk__in=VidlySubmission.objects.all().values_list('event_id', flat=True))) status = request.GET.get('status') repeated = request.GET.get('repeated') == 'event' repeats = {} if status: if status not in ('New', 'Processing', 'Finished', 'Error'): return http.HttpResponseBadRequest("Invalid 'status' value") # make a list of all tags -> events _tags = {} for event in events: environment = event.template_environment or {} if not environment.get('tag') or environment.get('tag') == 'None': continue _tags[environment['tag']] = event.id event_ids = [] for tag in vidly.medialist(status): try: event_ids.append(_tags[tag]) except KeyError: # it's on vid.ly but not in this database logging.debug("Unknown event with tag=%r", tag) events = events.filter(id__in=event_ids) elif repeated: repeats = dict( (x['event_id'], x['event__id__count']) for x in VidlySubmission.objects.values('event_id').annotate( Count('event__id')).filter(event__id__count__gt=1)) events = Event.objects.filter(id__in=repeats.keys()) def get_repeats(event): return repeats[event.id] events = events.order_by('-start_time') events = events.select_related('template') paged = paginate(events, request.GET.get('page'), 15) submissions = defaultdict(list) for submission in VidlySubmission.objects.filter(event__in=paged): submissions[submission.event_id].append(submission) vidly_resubmit_form = forms.VidlyResubmitForm() context = { 'paginate': paged, 'status': status, 'vidly_resubmit_form': vidly_resubmit_form, 'repeated': repeated, 'get_repeats': get_repeats, 'submissions': submissions, } return render(request, 'manage/vidly_media.html', context)
def test_medialist(self, p_urlopen): def mocked_urlopen(request): return StringIO(SAMPLE_MEDIALIST_XML.strip()) p_urlopen.side_effect = mocked_urlopen results = vidly.medialist('Error') ok_(results['abc123']) ok_(results['xyz987'])
def test_medialist(self, p_urlopen): def mocked_urlopen(request): return StringIO(SAMPLE_MEDIALIST_XML.strip()) p_urlopen.side_effect = mocked_urlopen results = vidly.medialist("Error") ok_(results["abc123"]) ok_(results["xyz987"])
def report_all(request): events = ( Event.objects.archived() .filter(template__name__icontains='vid.ly') .filter(template_environment__icontains='tag') .order_by('-start_time') )[:1000] # Vid.ly's GetMediaList is capped at 1000 most recent submissions vidly_durations = {} for tag, information in vidly.medialist('Finished').items(): try: vidly_durations[tag] = float(information['Duration']) except KeyError: pass def equalish_duration(duration1, duration2): return abs(duration1 - duration2) <= 1 context = { 'events': events, 'vidly_durations': vidly_durations, 'equalish_duration': equalish_duration, } return render(request, 'manage/durations.html', context)
def resubmit_failures(max_attempts=1, verbose=False): failed = vidly.medialist('Error') resubmitted = [] for shortcode in failed: try: submission = VidlySubmission.objects.exclude( event__status=Event.STATUS_REMOVED, ).get(tag=shortcode) except VidlySubmission.DoesNotExist: # If we have no record of submissions with that shortcode, # it's probably a piece of video on Vid.ly that came from # some other instance. continue if verbose: # pragma: no cover print repr(shortcode), "has failed" # print submissions.count(), "known vidly submissions in our DB" if not submission.errored: # That's weird and nearly impossible. # It can happen that the transcoding *did* fail but we # were never informed (or failed to acknowledge being # informed). results = vidly.query(shortcode)[shortcode] if results['Status'] == 'Error': submission.errored = parse_non_iso_date(results['Updated']) submission.save() assert submission.errored # If we can find any submissions that are submitted after # this failed one that has not errored, then bail out. non_failures = VidlySubmission.objects.filter( event=submission.event, errored__isnull=True, submission_time__gt=submission.submission_time, ) if non_failures.exists(): if verbose: # pragma: no cover print ( "Found at least one non-failure submission more recent." ) continue # How many failed attempts have there been? # If there's too many resubmissions, the bail out of fear of # re-submitting something that'll never work. failures = VidlySubmission.objects.filter( event=submission.event, errored__isnull=False, ).exclude( id=submission.id ) if failures.count() >= max_attempts: if verbose: # pragma: no cover print ( "Already been {} failed attempts.".format(failures.count()) ) continue if verbose: # pragma: no cover print "Resubmitting! {!r}".format(submission.event) error = resubmit(submission) if verbose: # pragma: no cover print "Resubmission error", error print "\n" resubmitted.append(submission) return resubmitted
def resubmit_failures(max_attempts=1, verbose=False): failed = vidly.medialist('Error') resubmitted = [] for shortcode in failed: try: submission = VidlySubmission.objects.exclude( event__status=Event.STATUS_REMOVED, ).get(tag=shortcode) except VidlySubmission.DoesNotExist: # If we have no record of submissions with that shortcode, # it's probably a piece of video on Vid.ly that came from # some other instance. continue if verbose: # pragma: no cover print repr(shortcode), "has failed" # print submissions.count(), "known vidly submissions in our DB" if not submission.errored: # That's weird and nearly impossible. # It can happen that the transcoding *did* fail but we # were never informed (or failed to acknowledge being # informed). results = vidly.query(shortcode)[shortcode] if results['Status'] == 'Error': submission.errored = parse_non_iso_date(results['Updated']) submission.save() assert submission.errored # If we can find any submissions that are submitted after # this failed one that has not errored, then bail out. non_failures = VidlySubmission.objects.filter( event=submission.event, errored__isnull=True, submission_time__gt=submission.submission_time, ) if non_failures.exists(): if verbose: # pragma: no cover print("Found at least one non-failure submission more recent.") continue # How many failed attempts have there been? # If there's too many resubmissions, the bail out of fear of # re-submitting something that'll never work. failures = VidlySubmission.objects.filter( event=submission.event, errored__isnull=False, ).exclude(id=submission.id) if failures.count() >= max_attempts: if verbose: # pragma: no cover print("Already been {} failed attempts.".format( failures.count())) continue if verbose: # pragma: no cover print "Resubmitting! {!r}".format(submission.event) error = resubmit(submission) if verbose: # pragma: no cover print "Resubmission error", error print "\n" resubmitted.append(submission) return resubmitted