def check_smil(video): _, record = video.fetch_published() master = CDSVideosFilesIterator.get_master_video_file(record) playlist = master['playlist'] assert playlist[0]['key'] == 'test.smil' assert playlist[0]['content_type'] == 'smil' assert playlist[0]['context_type'] == 'playlist' assert playlist[0]['media_type'] == 'text' assert playlist[0]['tags']['master'] == master['version_id'] # check bucket dump is done correctly master_video = CDSVideosFilesIterator.get_master_video_file(video) assert master_video['version_id'] != master['version_id']
def check_files(video): bucket = CDSRecordDumpLoader._get_bucket(record=video) files = [ dump_object(obj) for obj in ObjectVersion.get_by_bucket(bucket=bucket) ] for file_ in files: assert as_bucket(file_['bucket_id']) is not None assert 'checksum' in file_ assert 'content_type' in file_ assert 'context_type' in file_ assert FileInstance.query.filter_by( id=file_['file_id']) is not None assert 'key' in file_ assert 'links' in file_ assert 'content_type' in file_ assert 'context_type' in file_ assert 'media_type' in file_ assert 'tags' in file_ # check extracted metadata master_video = CDSVideosFilesIterator.get_master_video_file(video) assert any([ key in master_video['tags'] for key in ExtractMetadataTask._all_keys ]) assert any([ key in video['_cds']['extracted_metadata'] for key in ExtractMetadataTask._all_keys ])
def _get_master_video(record): """Return master video.""" master = CDSVideosFilesIterator.get_master_video_file(record) if not master: raise Exception("No master video found for the given record") return master, master['tags']['display_aspect_ratio'], \ int(master['tags']['width']), int(master['tags']['height'])
def check_tag_master(record): bucket = CDSRecordDumpLoader._get_bucket(record=record) master = CDSVideosFilesIterator.get_master_video_file(record) files = [dump_object(obj) for obj in ObjectVersion.get_by_bucket(bucket=bucket) if obj.get_tags().get('master')] assert all([file_['tags']['master'] == master['version_id'] for file_ in files])
def check_gif(video, mock_gif): # called only once for deposit (_, _, mock_args) = mock_gif.mock_calls[0] # check gif record video = CDSRecord(dict(video), video.model) # check gif deposit deposit = deposit_video_resolver(video['_deposit']['id']) master_video = CDSVideosFilesIterator.get_master_video_file(deposit) assert mock_args['master_id'] == master_video['version_id'] assert str(deposit.files.bucket.id) == mock_args['bucket'] # assert mock_args['bucket'].id == deposit.files.bucket.id assert len(mock_args['frames']) == 10 assert 'output_dir' in mock_args
def extract_frames(recid, depid): """Re-trigger the extract frames task.""" if not recid and not depid: raise ClickException('Missing option "--recid" or "--depid"') if recid: _, record = record_resolver.resolve(recid) depid = record['_deposit']['id'] video_deposit = deposit_video_resolver(depid) master = CDSVideosFilesIterator.get_master_video_file(video_deposit) ExtractFramesTask().si(version_id=master['version_id'], deposit_id=depid).apply_async()
def _force_sync_deposit_bucket(record): """Replace deposit bucket with a copy of the record bucket.""" deposit = Video.get_record(record.depid.object_uuid) # if deposit['_deposit']['status'] == 'draft': # raise RuntimeError('Deposit in edit mode: {0}'.format(deposit.id)) deposit_old_bucket = deposit.files.bucket # create a copy of record bucket new_bucket = record.files.bucket.snapshot() new_bucket.locked = False db.session.commit() rb = RecordsBuckets.query.filter( RecordsBuckets.bucket_id == deposit_old_bucket.id).one() rb.bucket = new_bucket db.session.add(rb) db.session.commit() # Put tags correctly pointing to the right object master_file = CDSVideosFilesIterator.get_master_video_file(record) if master_file: master_deposit_obj = ObjectVersion.get(new_bucket, master_file['key']) for slave in ObjectVersion.query_heads_by_bucket( bucket=new_bucket).join(ObjectVersion.tags).filter( ObjectVersion.file_id.isnot(None), ObjectVersionTag.key == 'master'): ObjectVersionTag.create_or_update( slave, 'master', str(master_deposit_obj.version_id)) db.session.add(slave) db.session.commit() # Delete the old bucket deposit_old_bucket.locked = False _ = deposit_old_bucket.remove() deposit['_buckets']['deposit'] = str(new_bucket.id) record['_buckets']['deposit'] = str(new_bucket.id) record['_deposit'] = deposit['_deposit'] deposit['_files'] = deposit.files.dumps() deposit.commit() record.commit() db.session.commit() return deposit_old_bucket.id, new_bucket.id
def test_video_name_after_publish(api_app, db, api_project, users): project, video_1, video_2 = api_project video_1_depid = video_1['_deposit']['id'] master_video_filename = 'test.mp4' # insert a master file inside the video add_master_to_video(video_deposit=video_1, filename=master_video_filename, stream=BytesIO(b'1234'), video_duration='15') # publish the video prepare_videos_for_publish([video_1]) video_1 = deposit_video_resolver(video_1_depid) login_user(User.query.get(users[0])) video_1 = video_1.publish() _, record = video_1.fetch_published() master = CDSVideosFilesIterator.get_master_video_file(record) assert master['key'] == '{}.mp4'.format(record['report_number'][0])
def create_all_missing_subformats(id_type, id_value): """Create all missing subformats.""" _validate(id_type=id_type) video_deposit, dep_uuid = _resolve_deposit(id_type, id_value) master, ar, w, h = _get_master_video(video_deposit) subformats = CDSVideosFilesIterator.get_video_subformats(master) dones = [subformat['tags']['preset_quality'] for subformat in subformats] missing = set(get_all_distinct_qualities()) - set(dones) transcodables = list( filter(lambda q: can_be_transcoded(q, ar, w, h), missing)) # sequential (and immutable) transcoding to avoid MergeConflicts on bucket if transcodables: chain([ MaintenanceTranscodeVideoTask().si(version_id=master['version_id'], preset_quality=quality, deposit_id=dep_uuid) for quality in transcodables ]).apply_async() return transcodables
def missing_subformats_report(start_date=None, end_date=None): """Send a report of missing subformats to CDS admins.""" report = [] def _get_master_video(record): """Return master video.""" master = CDSVideosFilesIterator.get_master_video_file(record) if not master: raise Exception("No master video found for the given record") return master, master['tags']['display_aspect_ratio'], \ int(master['tags']['width']), int(master['tags']['height']) def _get_missing_subformats(subformats, ar, w, h): """Return missing and transcodable subformats.""" dones = [ subformat['tags']['preset_quality'] for subformat in subformats ] missing = set(get_all_distinct_qualities()) - set(dones) transcodables = list( filter(lambda q: can_be_transcoded(q, ar, w, h), missing)) return transcodables def _format_report(report): """Format the email body for the file integrity report.""" lines = [] for entry in report: lines.append('Message: {}'.format(entry.get('message'))) lines.append(u'Record: {}'.format( format_pid_link(current_app.config['RECORDS_UI_ENDPOINT'], entry.get('recid')))) lines.append('Report number: {}'.format( entry.get('report_number'))) lines.append('Missing subformats: {}'.format( entry.get('missing_subformats'))) lines.append(('-' * 80) + '\n') return '\n'.join(lines) cache = current_cache.get('task_missing_subformats:details') or {} if 'end_date' not in cache: # Set the end date to 7 days ago cache['end_date'] = datetime.utcnow() - timedelta(days=7) record_uuids = _filter_by_last_created(_get_all_records_with_bucket(), start_date, end_date or cache['end_date']) for record_uuid in record_uuids: record = CDSRecord.get_record(record_uuid.id) master, ar, w, h = _get_master_video(record) if not master: report.append({ 'message': 'No master video found for the given record', 'recid': record.get('recid'), 'report_number': record['report_number'][0] }) continue # check missing subformats subformats = CDSVideosFilesIterator.get_video_subformats(master) missing = _get_missing_subformats(subformats, ar, w, h) if missing: report.append({ 'message': 'Missing subformats for the given record', 'recid': record.get('recid'), 'report_number': record['report_number'][0], 'missing_subformats': missing }) # check bucket ids consistency bucket_id = master['bucket_id'] for f in \ subformats + CDSVideosFilesIterator.get_video_frames(master) + \ CDSVideosFilesIterator.get_video_subtitles(record): if f['bucket_id'] != bucket_id: report.append({ 'message': 'Different buckets in the same record', 'recid': record.get('recid'), 'report_number': record['report_number'][0], 'buckets': 'Master: {0} - {1}: {2}'.format(bucket_id, f['key'], f['bucket_id']) }) cache['end_date'] = datetime.utcnow() current_cache.set('task_missing_subformats:details', cache, timeout=-1) if report: # Format and send the email subject = u'[CDS Videos] Missing subformats report [{}]'.format( datetime.now()) body = _format_report(report) sender = current_app.config['NOREPLY_EMAIL'] recipients = [current_app.config['CDS_ADMIN_EMAIL']] _send_email(subject, body, sender, recipients)
def subformats_integrity_report(start_date=None, end_date=None): """Send a report of all corrupted subformats to CDS admins.""" report = [] update_cache = True def _probe_video_file(obj, record): """Run ffmpeg on a video file Return a touple containing (report, accessible) """ file_report = {} path = obj.file.uri.replace( current_app.config['VIDEOS_XROOTD_ENDPOINT'], '') if not os.path.exists(path): # Check if the file exists on disk file_report = { 'file_name': obj.key, 'message': 'The file cannot be accessed', 'error': repr(e) } # Return the file report and the file accessibility return (file_report, False) try: # Expecting the storage to be mounted on the machine probe = ff_probe_all(path) if not probe.get('streams'): file_report = { 'file_name': obj.key, 'message': 'No video stream' } except Exception as e: file_report = { 'file_name': obj.key, 'message': 'Error while running ff_probe_all', 'error': repr(e) } # Return the file report and the file accessibility return (file_report, True) def _format_report(report): """Format the email body for the subformats integrity report.""" lines = [] for entry in report: lines.append(u'Record: {}'.format( format_pid_link(current_app.config['RECORDS_UI_ENDPOINT'], entry.get('recid')))) lines.append('Message: {}'.format(entry.get('message'))) if entry.get('report_number'): lines.append('Report number: {}'.format( entry.get('report_number'))) subreports = entry.get('subreports') if subreports: lines.append(('-' * 10) + '\n') for subreport in subreports: lines.append(' File name: {}'.format( subreport.get('file_name'))) lines.append(' Message: {}'.format( subreport.get('message'))) if subreport.get('error'): lines.append(' Error: {}'.format( subreport.get('error'))) lines.append(('-' * 80) + '\n') return '\n'.join(lines) cache = current_cache.get('task_subformats_integrity:details') or {} two_days_ago = datetime.utcnow() - timedelta(days=2) if 'start_date' not in cache: # Set the start date to 4 days ago cache['start_date'] = datetime.utcnow() - timedelta(days=4) record_uuids = _filter_by_last_created(_get_all_records_with_bucket(), start_date or cache['start_date'], end_date or two_days_ago) for record_uuid in record_uuids: record = CDSRecord.get_record(record_uuid.id) master = CDSVideosFilesIterator.get_master_video_file(record) if not master: report.append({ 'recid': record['recid'], 'message': 'No master video found for the given record', 'report_number': record['report_number'][0] }) continue master_obj = as_object_version(master['version_id']) subreport_master, accessible = _probe_video_file(master_obj, record) if not accessible: update_cache = False if subreport_master: report.append({ 'recid': record['recid'], 'message': 'Master file issue report', 'report_number': record['report_number'][0], 'subreports': subreport_master }) subformats = CDSVideosFilesIterator.get_video_subformats(master) if not subformats: report.append({ 'recid': record['recid'], 'message': 'No subformats found' }) continue subformats_subreport = [] for subformat in subformats: subformat_obj = as_object_version(subformat['version_id']) subformat_subreport, accessible = _probe_video_file( subformat_obj, record) if not accessible: update_cache = False if subformat_subreport: subformats_subreport.append(subformat_subreport) if subformats_subreport: report.append({ 'recid': record['recid'], 'message': 'Subformats issues found', 'report_number': record['report_number'][0], 'subreports': subformats_subreport }) if update_cache: # Set the start date for next time when the task will run cache['start_date'] = two_days_ago current_cache.set('task_subformats_integrity:details', cache, timeout=-1) if report: # Format and send the email subject = u'[CDS Videos] Subformats integrity report [{}]'.format( datetime.now()) body = _format_report(report) sender = current_app.config['NOREPLY_EMAIL'] recipients = [current_app.config['CDS_ADMIN_EMAIL']] _send_email(subject, body, sender, recipients)