Ejemplo n.º 1
0
def log_results(checksum_mismatches, csv_report_filename, parent_basename,
                judgement, start, finish, transcode_start, transcode_finish,
                transcode_time, sequence_length, fps, total_size, ffv1_size,
                pix_fmt, container, width, height, comp_ratio):
    if len(checksum_mismatches) == 0:
        print 'LOSSLESS'
        append_csv(csv_report_filename,
                   (parent_basename, judgement, start, finish, transcode_start,
                    transcode_finish, transcode_time, sequence_length, fps,
                    total_size, ffv1_size, pix_fmt, container, width, height,
                    comp_ratio))

    elif len(checksum_mismatches) == 1:
        if checksum_mismatches[0] == 'sar':
            print 'Image content is lossless, Pixel Aspect Ratio has been altered - This is mostly likely because your source had no pixel aspect ratio information, and Matroska is specifying 1:1. https://www.ietf.org/mail-archive/web/cellar/current/msg00739.html '
            append_csv(
                csv_report_filename,
                (parent_basename, 'LOSSLESS - different PAR', start, finish,
                 transcode_start, transcode_finish, transcode_time,
                 sequence_length, fps, total_size, ffv1_size, pix_fmt,
                 container, width, height, comp_ratio))
    elif len(checksum_mismatches) > 1:
        print 'NOT LOSSLESS'
        append_csv(csv_report_filename,
                   (parent_basename, judgement, start, finish, transcode_start,
                    transcode_finish, transcode_time, sequence_length, fps,
                    total_size, ffv1_size, pix_fmt, container, width, height,
                    comp_ratio))
Ejemplo n.º 2
0
def main():
    '''
    Launches the commands that generate the CSV error report
    '''
    args = parse_args()
    source = args.input
    csv_filename = os.path.join(
        ififuncs.make_desktop_logs_dir(),
        time.strftime("%Y-%m-%dT%H_%M_%S_videoerrors.csv"))
    print('Report stored as %s' % csv_filename)
    if not os.path.isfile(csv_filename):
        ififuncs.create_csv(
            csv_filename,
            ['filename', 'start_time', 'timestamp', 'error', 'notes'])
    for root, _, filenames in os.walk(source):
        for filename in filenames:
            if filename.endswith('.m2t'):
                with open(csv_filename, 'r') as fo:
                    if not filename in fo.read():
                        try:
                            start_time = bitc.getffprobe(
                                'start_time', 'stream=start_time',
                                os.path.join(root, filename)).rsplit()[0]
                            json_output = subprocess.check_output([
                                'ffprobe', '-sexagesimal',
                                os.path.join(root, filename), '-show_error',
                                '-show_log', '16', '-show_frames', '-of',
                                'json'
                            ])
                            errors = False
                            ffprobe_dict = json.loads(json_output)
                            for values in ffprobe_dict:
                                for more in ffprobe_dict[values]:
                                    if 'logs' in more:
                                        errors = True
                                        print(more['pkt_pts_time'],
                                              more['logs'])
                                        ififuncs.append_csv(
                                            csv_filename, [
                                                filename, start_time,
                                                more['pkt_pts_time'],
                                                more['logs'], ''
                                            ])
                            if errors == False:
                                ififuncs.append_csv(csv_filename, [
                                    filename, start_time, 'no errors',
                                    'no errors', ''
                                ])
                        except subprocess.CalledProcessError:
                            ififuncs.append_csv(csv_filename, [
                                filename, start_time,
                                'script error - process file manually', '', ''
                            ])
    print('Report stored as %s' % csv_filename)
Ejemplo n.º 3
0
        ffv1_size                           = os.path.getsize(ffv1_path)
        comp_ratio                          = float(total_size) / float(os.path.getsize(ffv1_path))
        judgement                           = diff_textfiles(source_textfile, ffv1_md5)
        fps                                 = float(sequence_length) / float(transcode_time)
        checksum_mismatches                 = []
        with open(source_textfile) as f1:
            with open(ffv1_md5) as f2:
                for (lineno1, line1), (lineno2, line2) in itertools.izip(
                               read_non_comment_lines(f1), read_non_comment_lines(f2)):
                    if line1 != line2:
                        if 'sar' in line1:
                            checksum_mismatches = ['sar']
                        else:
                            checksum_mismatches.append(1)
        if len(checksum_mismatches) == 0:
            print 'LOSSLESS'
            append_csv(csv_report_filename, (parent_basename,judgement, start, finish,transcode_start, transcode_finish,transcode_time, sequence_length, fps, total_size, ffv1_size, pix_fmt, container, width, height,comp_ratio))

        elif len(checksum_mismatches) == 1:
            if checksum_mismatches[0] == 'sar':
                print 'Image content is lossless, Pixel Aspect Ratio has been altered - This is mostly likely because your source had no pixel aspect ratio information, and Matroska is specifying 1:1. https://www.ietf.org/mail-archive/web/cellar/current/msg00739.html '
                append_csv(csv_report_filename, (parent_basename,'LOSSLESS - different PAR',start, finish,transcode_start, transcode_finish,transcode_time,sequence_length, fps,total_size, ffv1_size, pix_fmt, container,width, height,comp_ratio))
        elif len(checksum_mismatches) > 1:
            print 'NOT LOSSLESS'
            print csv_report_filename
            append_csv(csv_report_filename, (parent_basename,judgement, start, finish,transcode_start, transcode_finish,transcode_time,sequence_length, fps,total_size, ffv1_size, pix_fmt,container, width, height,comp_ratio))
        make_manifest(output_parent_directory, os.path.basename(output_dirname), manifest_textfile)


#send_gmail(emails, csv_report_filename, 'makedpx completed', 'Hi,\n Please the attached log for details of the makedpx job, \nSincerely yours,\nIFIROBOT', config[2].rstrip(), config[3].rstrip())
Ejemplo n.º 4
0
def main(args_):
    # if multiple file are present, this script will treat them as a single
    # instantiation/representation and get aggregate metadata about the whole
    # package. For now, this will be a clumsy implementation - the first file
    # will provide most metadata. Things like duration/bitrate/filesize
    # will be calculated as a whole.
    # Although another way would be that every call is looped, and if
    # this could catch files that should not be in the package, eg. a 4:2:2
    # file in a 4:2:0 package..
    # yup - do it that way!
    args = parse_args(args_)
    all_files = ififuncs.recursive_file_list(args.input)
    silence = True
    if args.user:
        user = args.user
    else:
        user = ififuncs.get_user()
    acquisition_type = ''
    if args.acquisition_type:
        acquisition_type = ififuncs.get_acquisition_type(
            args.acquisition_type)[0]
    for dirs in os.listdir(args.input):
        if ififuncs.validate_uuid4(dirs) is None:
            instantiationIdentif = dirs
    Accession_Number = get_accession_number(args.input)
    if args.reference:
        Reference_Number = args.reference.upper()
    else:
        Reference_Number = get_reference_number(args.input)
    if args.p:
        for root, _, filenames in os.walk(args.input):
            if os.path.basename(root) == 'metadata':
                metadata_dir = root
            elif os.path.basename(root) == 'logs':
                logs_dir = root
        csv_filename = os.path.join(metadata_dir,
                                    Accession_Number + '_pbcore.csv')
        sipcreator_log = os.path.join(logs_dir,
                                      instantiationIdentif + '_sip_log.log')
        ififuncs.generate_log(sipcreator_log, 'EVENT = makepbcore.py started')
        ififuncs.generate_log(
            sipcreator_log, 'eventDetail=makepbcore.py %s' %
            ififuncs.get_script_version('makepbcore.py'))
        ififuncs.generate_log(sipcreator_log,
                              'Command line arguments: %s' % args)
        ififuncs.generate_log(sipcreator_log, 'EVENT = agentName=%s' % user)
    else:
        csv_filename = 'blaa.csv'
    print ' - Metadata will be stored in %s' % csv_filename
    for filenames in os.listdir(args.input):
        if '_manifest.md5' in filenames:
            md5_manifest = os.path.join(args.input, filenames)
        elif 'manifest-sha512.txt' in filenames:
            sha512_manifest = os.path.join(args.input, filenames)
    make_csv(csv_filename)
    ms = 0
    FrameCount = 0
    instantFileSize_byte = 0
    instantFileSize_gigs = 0
    scan_types = []
    matrix_list = []
    transfer_list = []
    colour_primaries_list = []
    color_spaces = []
    chroma = []
    frame_sizes = []
    par_list = []
    container_list = []
    fps_list = []
    sample_rate_list = []
    track_count_list = []
    interlace_list = []
    compression_list = []
    pix_fmt_list = []
    audio_fmt_list = []
    audio_codecid_list = []
    audio_codec_list = []
    au_bitdepth_list = []
    video_codecid_list = []
    video_codec_version_list = []
    video_codec_profile_list = []
    timecode_list = []
    channels_list = []
    for source in all_files:
        metadata = subprocess.check_output(
            ['mediainfo', '--Output=PBCore2', source])
        root = etree.fromstring(metadata)
        print(' - Analysing  %s') % source
        pbcore_namespace = root.xpath('namespace-uri(.)')
        track_type = root.xpath('//ns:essenceTrackType',
                                namespaces={'ns': pbcore_namespace})
        if len(track_type) > 0:
            for track in track_type:
                if track.text == 'Video':
                    essenceTrackEncodvid = get_metadata(
                        "ns:essenceTrackEncoding", track.getparent(),
                        pbcore_namespace)
                    vcodec_attributes = get_attributes(track.getparent(),
                                                       pbcore_namespace)
                elif track.text == 'Audio':
                    silence = False
                    essenceTrackEncod_au = get_metadata(
                        "//ns:essenceTrackEncoding", track.getparent(),
                        pbcore_namespace)
                    audio_codec_list.append(essenceTrackEncod_au)
                    acodec_attributes = get_attributes(track.getparent(),
                                                       pbcore_namespace)
                    try:
                        audio_codecid = acodec_attributes['ref']
                    except KeyError:
                        audio_codecid = 'n/a'
                    essenceTrackSampling = ififuncs.get_mediainfo(
                        'samplerate', '--inform=Audio;%SamplingRate_String%',
                        source)
                    sample_rate_list.append(essenceTrackSampling)
                    essenceBitDepth_au = get_metadata(
                        "//ns:essenceTrackBitDepth", root, pbcore_namespace)
                    audio_codecid_list.append(audio_codecid)
                    au_bitdepth_list.append(essenceBitDepth_au)
                    channels = get_metadata(
                        "//ns:essenceTrackAnnotation[@annotationType='Channel(s)']",
                        track.getparent(), pbcore_namespace)
                    channels_list.append(channels)
        ScanType = get_metadata(
            "//ns:essenceTrackAnnotation[@annotationType='ScanType']", root,
            pbcore_namespace)
        scan_types.append(ScanType)
        matrix_coefficients = get_metadata(
            "//ns:essenceTrackAnnotation[@annotationType='matrix_coefficients']",
            root, pbcore_namespace)
        timecode_source, starting_timecode = get_timecode(
            pbcore_namespace, root, source)
        timecode_list.append(starting_timecode)
        matrix_list.append(matrix_coefficients)
        transfer_characteris = get_metadata(
            "//ns:essenceTrackAnnotation[@annotationType='transfer_characteristics']",
            root, pbcore_namespace)
        transfer_list.append(transfer_characteris)
        colour_primaries = get_metadata(
            "//ns:essenceTrackAnnotation[@annotationType='color_primaries']",
            root, pbcore_namespace)
        colour_primaries_list.append(colour_primaries)
        FrameCount += int(
            get_metadata(
                "//ns:essenceTrackAnnotation[@annotationType='FrameCount']",
                root, pbcore_namespace))
        instantFileSize_byte += int(
            get_metadata("//ns:instantiationFileSize", root, pbcore_namespace))
        instantDataRate = round(
            float(
                ififuncs.get_mediainfo('OverallBitRate',
                                       '--inform=General;%OverallBitRate%',
                                       source)) / 1000 / 1000, 2)
        instantTracks = ififuncs.get_number_of_tracks(source)
        track_count_list.append(instantTracks)
        ms += ififuncs.get_milliseconds(source)
        ColorSpace = get_metadata(
            "//ns:essenceTrackAnnotation[@annotationType='ColorSpace']", root,
            pbcore_namespace)
        color_spaces.append(ColorSpace)
        ChromaSubsampling = get_metadata(
            "//ns:essenceTrackAnnotation[@annotationType='ChromaSubsampling']",
            root, pbcore_namespace)
        chroma.append(ChromaSubsampling)
        instantMediaty = get_metadata("//ns:instantiationMediaType", root,
                                      pbcore_namespace)
        essenceFrameSize = get_metadata("//ns:essenceTrackFrameSize", root,
                                        pbcore_namespace)
        frame_sizes.append(essenceFrameSize)
        PixelAspectRatio = get_metadata(
            "//ns:essenceTrackAnnotation[@annotationType='PixelAspectRatio']",
            root, pbcore_namespace)
        par_list.append(PixelAspectRatio)
        instantiationStandar = get_metadata(
            "//ns:instantiationAnnotation[@annotationType='Format']", root,
            pbcore_namespace)
        container_list.append(instantiationStandar)
        essenceFrameRate = get_metadata("//ns:essenceTrackFrameRate", root,
                                        pbcore_namespace)
        fps_list.append(essenceFrameRate)
        essenceAspectRatio = ififuncs.get_mediainfo(
            'DAR', '--inform=Video;%DisplayAspectRatio_String%', source)
        Interlacement = get_metadata(
            "//ns:instantiationAnnotation[@annotationType='Interlacement']",
            root, pbcore_namespace)
        interlace_list.append(Interlacement)
        Compression_Mode = get_metadata(
            "//ns:instantiationAnnotation[@annotationType='Compression_Mode']",
            root, pbcore_namespace)
        colour_range = get_metadata(
            "//ns:essenceTrackAnnotation[@annotationType='colour_range']",
            root, pbcore_namespace)
        format_version = get_metadata(
            "//ns:instantiationAnnotation[@annotationType='Format_Version']",
            root, pbcore_namespace)
        app_company_name = get_metadata(
            "//ns:instantiationAnnotation[@annotationType='Encoded_Application_CompanyName']",
            root, pbcore_namespace)
        app_name = get_metadata(
            "//ns:instantiationAnnotation[@annotationType='Encoded_Application_Name']",
            root, pbcore_namespace)
        app_version = get_metadata(
            "//ns:instantiationAnnotation[@annotationType='Encoded_Application_Version']",
            root, pbcore_namespace)
        library_name = get_metadata(
            "//ns:instantiationAnnotation[@annotationType='Encoded_Library_Name']",
            root, pbcore_namespace)
        library_version = get_metadata(
            "//ns:instantiationAnnotation[@annotationType='Encoded_Library_Version']",
            root, pbcore_namespace)
        compression_list.append(Compression_Mode)
        instantiationDate_mo = get_metadata(
            "//ns:instantiationDate[@dateType='file modification']", root,
            pbcore_namespace)
        instantDate_other = 'n/a'
        instantDate_type = 'n/a'
        pix_fmt = ififuncs.get_ffmpeg_fmt(source, 'video')
        pix_fmt_list.append(pix_fmt)
        audio_fmt = ififuncs.get_ffmpeg_fmt(source, 'audio')
        audio_fmt_list.append(audio_fmt)
    if silence:
        audio_codecid = 'n/a'
        essenceBitDepth_au = 'n/a'
        essenceTrackEncod_au = 'n/a'
        essenceTrackSampling = 'n/a'
    video_codecid = vcodec_attributes['ref']
    video_codecid_list.append(video_codecid)
    try:
        video_codec_version = vcodec_attributes['version']
    except KeyError:
        video_codec_version = 'n/a'
    try:
        video_codec_profile = vcodec_attributes['annotation'][8:]
    except KeyError:
        video_codec_profile = 'n/a'
    video_codec_version_list.append(video_codec_version)
    video_codec_profile_list.append(video_codec_profile)
    metadata_error = ''
    metadata_list = [
        scan_types, matrix_list, transfer_list, colour_primaries_list,
        color_spaces, chroma, frame_sizes, par_list, container_list, fps_list,
        sample_rate_list, track_count_list, interlace_list, compression_list,
        pix_fmt_list, audio_fmt_list, audio_codecid_list, audio_codec_list,
        au_bitdepth_list, video_codecid_list, video_codec_version_list,
        video_codec_profile_list, channels_list, timecode_list
    ]
    for i in metadata_list:
        if len(set(i)) > 1:
            metadata_error += 'WARNING - Your metadata values are not the same for all files: %s\n' % set(
                i)
            print metadata_error
            if args.p:
                ififuncs.generate_log(
                    sipcreator_log,
                    'EVENT = Metadata mismatch - Your metadata values are not the same for all files: %s'
                    % set(i))
    tc = ififuncs.convert_millis(ms)
    instantiationDuratio = ififuncs.convert_timecode(25, tc)
    if args.donor:
        Donor = args.donor
    else:
        Donor = ''
    Edited_By = user
    Date_Created = ''
    Date_Last_Modified = ''
    Film_Or_Tape = 'Digital AV Object'
    Date_Of_Donation = ''
    reproduction_creator = ''
    if args.acquisition_type:
        if acquisition_type == 'Reproduction':
            Date_Of_Donation = instantiationDate_mo.split('T')[0]
            # if a reproduction, then there's no Donor/transfer of title.
            Donor = 'n/a'
            if ififuncs.find_concat_user(
                    sipcreator_log) == 'Aoife Fitzmaurice':
                reproduction_creator = 'Aoife Fitzmaurice (2016)'
            elif ififuncs.find_concat_user(
                    sipcreator_log) == 'Kieran O\'Leary':
                reproduction_creator = 'Kieran O\'Leary (2013)'
    Habitat = ''
    backup_habitat = ''
    Type_Of_Deposit = acquisition_type
    if args.depositor_reference:
        Depositor_Reference = args.depositor_reference
    else:
        Depositor_Reference = ''
    Master_Viewing = 'Preservation Object'
    Language_Version = ''
    Condition_Rating = ''
    Companion_Elements = ''
    TTape_Origin = args.parent
    EditedNew = user
    FIO = 'In'
    CollectionTitle = ''
    Created_By = user
    instantTimeStart = 'n/a'
    instantFileSize_gigs = round(
        float(instantFileSize_byte) / 1024 / 1024 / 1024, 3)
    instantColors = 'n/a'
    instantLanguage = 'n/a'
    instantAltMo = 'n/a'
    essenceBitDepth_vid = ififuncs.get_mediainfo('duration',
                                                 '--inform=Video;%BitDepth%',
                                                 source)
    instantiationChanCon = 'n/a'
    colour_range = colour_range
    format_version = format_version
    TimeCode_FirstFrame = process_mixed_values(timecode_list)
    TimeCode_Source = timecode_source
    reproduction_reason = ''
    dig_object_descrip = ififuncs.get_digital_object_descriptor(args.input)
    ififuncs.append_csv(csv_filename, [
        Reference_Number,
        Donor,
        Edited_By,
        Date_Created,
        Date_Last_Modified,
        Film_Or_Tape,
        Date_Of_Donation,
        Accession_Number,
        Habitat,
        backup_habitat,
        TTape_Origin,
        Type_Of_Deposit,
        Depositor_Reference,
        Master_Viewing,
        Language_Version,
        Condition_Rating,
        Companion_Elements,
        EditedNew,
        FIO,
        CollectionTitle,
        Created_By,
        instantiationIdentif,
        instantDate_other,
        instantDate_type,
        instantiationDate_mo,
        instantiationStandar,
        instantMediaty,
        instantFileSize_byte,
        instantFileSize_gigs,
        instantTimeStart,
        instantDataRate,
        instantTracks,
        instantColors,
        instantLanguage,
        instantAltMo,
        essenceTrackEncodvid,
        essenceFrameRate,
        essenceTrackSampling,
        essenceBitDepth_vid,
        essenceFrameSize,
        essenceAspectRatio,
        essenceTrackEncod_au,
        essenceBitDepth_au,
        instantiationDuratio,
        instantiationChanCon,
        PixelAspectRatio,
        FrameCount,
        ColorSpace,
        ChromaSubsampling,
        ScanType,
        Interlacement,
        Compression_Mode,
        colour_primaries,
        transfer_characteris,
        matrix_coefficients,
        pix_fmt,
        audio_fmt,
        audio_codecid,
        video_codecid,
        video_codec_version,
        video_codec_profile,
        channels,
        colour_range,
        format_version,
        TimeCode_FirstFrame,
        TimeCode_Source,
        app_company_name,
        app_name,
        app_version,
        library_name,
        library_version,
        reproduction_creator,
        reproduction_reason,
        dig_object_descrip,
    ])
    if args.p:
        ififuncs.generate_log(
            sipcreator_log,
            'EVENT = Metadata extraction - eventDetail=Technical record creation using PBCore, eventOutcome=%s, agentName=makepbcore'
            % (csv_filename))
        ififuncs.generate_log(sipcreator_log, 'EVENT = makepbcore.py finished')
        ififuncs.checksum_replace(md5_manifest, sipcreator_log, 'md5')
        ififuncs.checksum_replace(sha512_manifest, sipcreator_log, 'sha512')
        ififuncs.manifest_update(md5_manifest, csv_filename)
        print ' - Updating %s with %s' % (md5_manifest, csv_filename)
        ififuncs.sha512_update(sha512_manifest, csv_filename)
        print ' - Updating %s with %s' % (sha512_manifest, csv_filename)
        print metadata_error
Ejemplo n.º 5
0
def main():
    desktop_logdir = os.path.expanduser("~/Desktop/") + 'seq_csv_reports'
    if not os.path.isdir(desktop_logdir):
        os.makedirs(desktop_logdir)
    all_files = sys.argv[1:]
    permission = ''
    if not permission == 'y' or permission == 'Y':
        print '\n\n**** All image sequences within these directories will be converted the input for this script.\n'
        for i in all_files:
            print i
        permission =  raw_input('\n**** All image sequences within these directories will be converted the input for this script \n**** If this looks ok, please press Y, otherwise, type N\n' )
        while permission not in ('Y','y','N','n'):
            permission =  raw_input('\n**** All image sequences within these directories will be converted the input for this script \n**** If this looks ok, please press Y, otherwise, type N\n')
        if permission == 'n' or permission == 'N':
            print 'Exiting at your command- Cheerio for now'
            sys.exit()
        elif permission =='y' or permission == 'Y':
            print 'Ok so!'
    csv_report_filename = desktop_logdir + '/seq2prores_report' + time.strftime("_%Y_%m_%dT%H_%M_%S") + '.csv'
    user = get_user()
    create_csv(csv_report_filename, ('Sequence Name', 'Start time', 'Finish Time'))
    for source_directory in all_files:
        for root,dirnames,filenames in os.walk(source_directory):
                #if "tiff_scans"  in dirnames:
                source_directory = root# + '/tiff_scans'
                total_size = 0
                remove_bad_files(source_directory)
                source_parent_dir    = os.path.dirname(source_directory)
                normpath             = os.path.normpath(source_directory)
                relative_path        = normpath.split(os.sep)[-1]
                split_path           = os.path.split(os.path.basename(source_directory))[1]
                start = datetime.datetime.now()

                info = get_filenames(source_directory, 'dpx_framemd5')
                if info == 'none':
                    continue
                for files in filenames:
                    total_size += os.path.getsize(os.path.join(root,files))
                master_parent_dir     = os.path.dirname(source_parent_dir)
                master_object_dir     = master_parent_dir + '/objects/image'
                master_metadata_dir = master_parent_dir + '/' + 'metadata'
                middle =  os.listdir(os.path.dirname(os.path.dirname(master_parent_dir)) + '/mezzanine')[0]
                mezzanine_object_dir            =  os.path.dirname(os.path.dirname(master_parent_dir)) + '/mezzanine/%s/objects' % middle
                mezzanine_parent_dir   = os.path.dirname(os.path.dirname(master_parent_dir)) + '/mezzanine/%s' % middle
                mezzanine_metadata_dir = mezzanine_parent_dir + '/metadata'
                source_manifest =  master_parent_dir + '/' + os.path.basename( master_parent_dir) +  '_manifest.md5'
                mezzanine_manifest =   mezzanine_parent_dir + '/' + os.path.basename( mezzanine_parent_dir) +  '_manifest.md5'
                master_audio =  master_parent_dir + '/objects/audio/' + os.listdir(master_parent_dir + '/objects/audio')[0]
                mezzanine_file =  mezzanine_object_dir + '/' + os.path.basename(mezzanine_parent_dir) + '_mezzanine.mov'
                if os.path.isfile(mezzanine_file):
                    print 'Mezzanine file already exists so this script has most likely already been run.. skipping.'
                    continue
                image_seq_without_container = info[0]
                start_number                = info[1]
                container                   = info[2]
                start_number_length = len(start_number)
                number_regex = "%0" + str(start_number_length) + 'd.'
                audio_dir            = source_parent_dir + '/audio'
                logs_dir            =  mezzanine_parent_dir + '/logs'

                source_representation_uuid = premis_description(master_object_dir, master_parent_dir + '/objects/audio', user)

                os.chdir(audio_dir)
                audio_file_list = glob('*.wav')
                audio_file = os.path.join(audio_dir,audio_file_list[0])
                dpx_filename                = image_seq_without_container + number_regex + container
                logfile = logs_dir + '/%s_prores.log' % os.path.basename(mezzanine_parent_dir)
                env_dict = os.environ.copy()
                # https://github.com/imdn/scripts/blob/0dd89a002d38d1ff6c938d6f70764e6dd8815fdd/ffmpy.py#L272
                logfile = "\'" + logfile + "\'"
                env_dict['FFREPORT'] = 'file={}:level=48'.format(logfile)
                seq2prores= ['ffmpeg','-y','-f','image2','-framerate','24', '-start_number', start_number, '-i', root + '/' + dpx_filename ,'-i', audio_file,'-c:v','prores','-profile:v', '3','-c:a','pcm_s24le', '-ar', '48000', mezzanine_object_dir + '/' + os.path.basename(mezzanine_parent_dir) + '_mezzanine.mov','-f', 'framemd5', '-an', master_metadata_dir + '/image/' + os.path.basename(master_parent_dir) + '.framemd5', '-c:a', 'pcm_s24le', '-f', 'framemd5', '-vn', master_metadata_dir + '/audio/' + os.path.basename(master_parent_dir) + '.framemd5']
                print seq2prores
                subprocess.call(seq2prores,env=env_dict)
                representation_uuid = str(uuid.uuid4())
                split_list = os.path.basename(mezzanine_parent_dir).split('_')
                premisxml, premis_namespace, doc, premis = setup_xml(mezzanine_file)
                items = {"workflow":"seq2prores","oe":'n/a', "filmographic":split_list[0], "sourceAccession":split_list[1], "interventions":['placeholder'], "prepList":['placeholder'], "user":user}
                premis = doc.getroot()
                xml_info    = make_premis(mezzanine_file, items, premis, premis_namespace,premisxml, representation_uuid, '????')
                sequence = xml_info[3]

                linking_representation_uuids = []
                linking_representation_uuids.append(xml_info[2])
                linking_representation_uuids.append(xml_info[2]) # the duplicate does nothing btw, they are a placeholder from a hardcoded function
                linking_representation_uuids.append(source_representation_uuid)
                create_representation(premisxml, premis_namespace, doc, premis, items,linking_representation_uuids, representation_uuid,sequence )
                doc         = xml_info[0]
                premisxml   = xml_info[1]
                final_sip_manifest_uuid                     = str(uuid.uuid4())
                prores_event_uuid                           = str(uuid.uuid4())

                macMiniTelecineMachineAgent_events          = [prores_event_uuid,final_sip_manifest_uuid  ]
                macMiniTelecineMachineAgent                 = make_agent(premis,macMiniTelecineMachineAgent_events, '230d72da-07e7-4a79-96ca-998b9f7a3e41')
                macMiniTelecineMachineOSAgent_events        = [prores_event_uuid,final_sip_manifest_uuid ]
                macMiniTelecineOSAgent                      = make_agent(premis,macMiniTelecineMachineOSAgent_events, '9486b779-907c-4cc4-802c-22e07dc1242f')

                hashlib_events                              = [final_sip_manifest_uuid ]
                hashlibAgent                                = make_agent(premis,hashlib_events, '9430725d-7523-4071-9063-e8a6ac4f84c4')
                ffmpegAgent_events                          = [prores_event_uuid ]
                ffmpegAgent                                 = make_agent(premis,ffmpegAgent_events , 'ee83e19e-cdb1-4d83-91fb-7faf7eff738e')
                operatorEvents                              = [final_sip_manifest_uuid,prores_event_uuid]
                operatorAgent                               = make_agent(premis,operatorEvents ,user)
                #ffmpegAgent                                 = make_agent(premis,[framemd5_uuid ], 'ee83e19e-cdb1-4d83-91fb-7faf7eff738e')
                make_event(premis, 'creation', 'Image Sequence and WAV re-encoded to Apple Pro Res 422 HQ with 48khz 24-bit PCM audio', [macMiniTelecineMachineAgent ,macMiniTelecineOSAgent, ffmpegAgent, operatorAgent ],prores_event_uuid,[representation_uuid], 'outcome')

                print premisxml
                mezzanine_mediainfoxml =  "%s/%s_mediainfo.xml" % (mezzanine_metadata_dir,os.path.basename(mezzanine_parent_dir) )
                tracexml =  "%s/%s_mediatrace.xml" % (mezzanine_metadata_dir,os.path.basename(mezzanine_parent_dir) )
                audio_mediainfoxml = "%s/%s_mediainfo.xml" % (master_metadata_dir + '/audio',os.path.basename(master_audio) )
                audio_mediatracexml = "%s/%s_mediatrace.xml" % (master_metadata_dir + '/audio',os.path.basename(master_audio) )
                if not os.path.isfile(audio_mediainfoxml):
                    make_mediainfo(audio_mediainfoxml,'audiomediaxmlinput',master_audio)
                if not os.path.isfile(audio_mediatracexml):
                    make_mediainfo(audio_mediatracexml,'audiomediatraceinput',master_audio)
                if not os.path.isfile(mezzanine_mediainfoxml):
                    make_mediainfo(mezzanine_mediainfoxml,'mediaxmlinput',mezzanine_object_dir + '/' + os.path.basename(mezzanine_parent_dir) + '_mezzanine.mov')
                if not os.path.isfile(tracexml):
                    make_mediatrace(tracexml,'mediatracexmlinput',mezzanine_object_dir + '/' + os.path.basename(mezzanine_parent_dir) + '_mezzanine.mov')
                hashlib_manifest(master_parent_dir, source_manifest, master_parent_dir)
                hashlib_manifest(mezzanine_parent_dir, mezzanine_manifest, mezzanine_parent_dir)
                make_event(premis, 'message digest calculation', 'Checksum manifest for whole package created', [macMiniTelecineMachineAgent ,macMiniTelecineOSAgent, operatorAgent],final_sip_manifest_uuid,[representation_uuid], 'source')
                write_premis(doc, premisxml)
                finish = datetime.datetime.now()
                append_csv(csv_report_filename, (os.path.basename( master_parent_dir), start, finish))
                '''
Ejemplo n.º 6
0
def main(args_):
    # if multiple file are present, this script will treat them as a single
    # instantiation/representation and get aggregate metadata about the whole
    # package. For now, this will be a clumsy implementation - the first file
    # will provide most metadata. Things like duration/bitrate/filesize
    # will be calculated as a whole.
    # Although another way would be that every call is looped, and if
    # this could catch files that should not be in the package, eg. a 4:2:2
    # file in a 4:2:0 package..
    # yup - do it that way!
    args = parse_args(args_)
    all_files = ififuncs.recursive_file_list(args.input)
    silence = True
    audio_only = True
    if args.user:
        user = args.user
    else:
        user = ififuncs.get_user()
    acquisition_type = ''
    if args.acquisition_type:
        acquisition_type = ififuncs.get_acquisition_type(
            args.acquisition_type)[0]
    instantiationIdentif = ''
    for dirs in os.listdir(args.input):
        if ififuncs.validate_uuid4(dirs) is None:
            instantiationIdentif = dirs
    Accession_Number = get_accession_number(args.input)
    if args.reference:
        Reference_Number = args.reference.upper()
    else:
        Reference_Number = get_reference_number(args.input)
    if args.p:
        for root, _, filenames in os.walk(args.input):
            if os.path.basename(root) == 'metadata':
                metadata_dir = root
            elif os.path.basename(root) == 'logs':
                logs_dir = root
        csv_filename = os.path.join(
            metadata_dir,
            Accession_Number + '_%s_pbcore.csv' % Reference_Number)
        sipcreator_log = os.path.join(logs_dir,
                                      instantiationIdentif + '_sip_log.log')
        ififuncs.generate_log(sipcreator_log, 'EVENT = makepbcore.py started')
        ififuncs.generate_log(
            sipcreator_log, 'eventDetail=makepbcore.py %s' %
            ififuncs.get_script_version('makepbcore.py'))
        ififuncs.generate_log(sipcreator_log,
                              'Command line arguments: %s' % args)
        ififuncs.generate_log(sipcreator_log, 'EVENT = agentName=%s' % user)
    else:
        csv_filename = 'blaa.csv'
    print((' - Metadata will be stored in %s' % csv_filename))
    for filenames in os.listdir(args.input):
        if '_manifest.md5' in filenames:
            md5_manifest = os.path.join(args.input, filenames)
        elif 'manifest-sha512.txt' in filenames:
            sha512_manifest = os.path.join(args.input, filenames)
    make_csv(csv_filename)
    ms = 0
    FrameCount = 0
    instantFileSize_byte = 0
    instantFileSize_gigs = 0
    scan_types = []
    matrix_list = []
    transfer_list = []
    colour_primaries_list = []
    color_spaces = []
    chroma = []
    frame_sizes = []
    par_list = []
    container_list = []
    fps_list = []
    sample_rate_list = []
    track_count_list = []
    interlace_list = []
    compression_list = []
    pix_fmt_list = []
    audio_fmt_list = []
    audio_codecid_list = []
    audio_codec_list = []
    au_bitdepth_list = []
    video_codecid_list = []
    video_codec_version_list = []
    video_codec_profile_list = []
    timecode_list = []
    channels_list = []
    stl = False
    subtitle_check = ififuncs.get_digital_object_descriptor(args.input)
    if 'STL' in subtitle_check:
        stl = True
    for source in all_files:
        metadata = subprocess.check_output(
            ['mediainfo', '--Output=PBCore2', source])
        new_metadata = subprocess.check_output(
            ['mediainfo', '--Output=XML', source])
        try:
            root = etree.fromstring(metadata)
            new_root = etree.fromstring(new_metadata)
        except lxml.etree.XMLSyntaxError:
            print('Windows encoding detected - transforming into utf-8')
            root = etree.fromstring(metadata.decode('cp1252').encode('utf-8'))
            new_root = etree.fromstring(
                new_metadata.decode('cp1252').encode('utf-8'))
        print(((' - Analysing  %s') % source))
        pbcore_namespace = root.xpath('namespace-uri(.)')
        mediainfo_namespace = new_root.xpath('namespace-uri(.)')
        track_type = root.xpath('//ns:essenceTrackType',
                                namespaces={'ns': pbcore_namespace})
        new_track_type = new_root.xpath('//ns:track',
                                        namespaces={'ns': mediainfo_namespace})
        if len(new_track_type) > 0:
            for track in new_track_type:
                if track.attrib['type'] == 'Video':
                    audio_only = False
                    essenceTrackEncodvid = ififuncs.get_metadata(
                        "ns:Format", track, mediainfo_namespace)
                    #vcodec_attributes = get_attributes(track.getparent(), pbcore_namespace)
                    #vcodec_attributes = 'TODO'
                    video_codecid = ififuncs.get_metadata(
                        "ns:CodecID", track, mediainfo_namespace)
                    video_codec_version = ififuncs.get_metadata(
                        "ns:Format_Version", track, mediainfo_namespace)
                    video_codec_profile = ififuncs.get_metadata(
                        "ns:Format_Profile", track, mediainfo_namespace)
                    video_codec_version_list.append(video_codec_version)
                    video_codec_profile_list.append(video_codec_profile)
                elif track.attrib['type'] == 'Audio':
                    silence = False
                    essenceTrackEncod_au = ififuncs.get_metadata(
                        "ns:Format", track, mediainfo_namespace)
                    audio_codec_list.append(essenceTrackEncod_au)
                    #acodec_attributes = get_attributes(track.getparent(), pbcore_namespace)

                    audio_codecid = ififuncs.get_metadata(
                        "ns:CodecID", track, mediainfo_namespace)
                    essenceTrackSampling = ififuncs.get_mediainfo(
                        'samplerate', '--inform=Audio;%SamplingRate_String%',
                        source)
                    sample_rate_list.append(essenceTrackSampling)
                    essenceBitDepth_au = ififuncs.get_metadata(
                        "ns:BitDepth", track, mediainfo_namespace)
                    audio_codecid_list.append(audio_codecid)
                    au_bitdepth_list.append(essenceBitDepth_au)
                    channels = ififuncs.get_metadata("//ns:Channels", track,
                                                     mediainfo_namespace)
                    channels_list.append(channels)
        if audio_only:
            essenceTrackEncodvid = 'n/a'
            video_codecid = 'n/a'
            video_codec_version = 'n/a'
            video_codec_profile = 'n/a'
        ScanType = ififuncs.get_metadata("//ns:ScanType", new_root,
                                         mediainfo_namespace)
        scan_types.append(ScanType)
        matrix_coefficients = ififuncs.get_metadata("//ns:matrix_coefficients",
                                                    new_root,
                                                    mediainfo_namespace)
        timecode_source, starting_timecode = get_timecode(
            pbcore_namespace, root, source)
        timecode_list.append(starting_timecode)
        matrix_list.append(matrix_coefficients)
        transfer_characteris = ififuncs.get_metadata(
            "//ns:transfer_characteristics", new_root, mediainfo_namespace)
        transfer_list.append(transfer_characteris)
        colour_primaries = ififuncs.get_metadata("//ns:colour_primaries",
                                                 new_root, mediainfo_namespace)
        colour_primaries_list.append(colour_primaries)
        try:
            if audio_only:
                FrameCount = 'n/a'
            else:
                # increment if multiple objects are present
                try:
                    FrameCount += int(
                        ififuncs.get_metadata("//ns:FrameCount", new_root,
                                              mediainfo_namespace))
                except ValueError:
                    # don't increment if multiple values are returned as str
                    FrameCount = ififuncs.get_metadata("//ns:FrameCount",
                                                       new_root,
                                                       mediainfo_namespace)
        except TypeError:
            # workaround for silent pic in DCP
            FrameCount = 'n/a'
        instantFileSize_byte += int(
            ififuncs.get_metadata("//ns:FileSize", new_root,
                                  mediainfo_namespace))
        instantDataRate = round(
            float(
                ififuncs.get_mediainfo('OverallBitRate',
                                       '--inform=General;%OverallBitRate%',
                                       source)) / 1000 / 1000, 2)
        instantTracks = ififuncs.get_number_of_tracks(source)
        track_count_list.append(instantTracks)
        if stl is True:
            track_count_list.append('STL sidecar')
        ms += ififuncs.get_milliseconds(source)
        ColorSpace = ififuncs.get_metadata("//ns:ColorSpace", new_root,
                                           mediainfo_namespace)
        color_spaces.append(ColorSpace)
        ChromaSubsampling = get_metadata("//ns:ChromaSubsampling", new_root,
                                         mediainfo_namespace)
        chroma.append(ChromaSubsampling)
        instantMediaty = get_metadata("//ns:instantiationMediaType", root,
                                      pbcore_namespace)
        if audio_only:
            essenceFrameSize = 'n/a'
        else:
            essenceFrameSize = get_metadata("//ns:essenceTrackFrameSize", root,
                                            pbcore_namespace)
        frame_sizes.append(essenceFrameSize)
        PixelAspectRatio = ififuncs.get_metadata("//ns:PixelAspectRatio",
                                                 new_root, mediainfo_namespace)
        par_list.append(PixelAspectRatio)
        general_root = new_root.xpath("//ns:track[@type='General']",
                                      namespaces={'ns':
                                                  mediainfo_namespace})[0]
        instantiationStandar = ififuncs.get_metadata("ns:Format", general_root,
                                                     mediainfo_namespace)
        container_list.append(instantiationStandar)
        essenceFrameRate = ififuncs.get_metadata("//ns:FrameRate", new_root,
                                                 mediainfo_namespace)
        fps_list.append(essenceFrameRate)
        essenceAspectRatio = ififuncs.get_mediainfo(
            'DAR', '--inform=Video;%DisplayAspectRatio_String%', source)
        Interlacement = ififuncs.get_metadata("//ns:ScanOrder", new_root,
                                              mediainfo_namespace)
        # FFV1/MKV seems to have this scanorder metadata here rather than Interlacement
        # FFV1/MKV is the only example I've seen so far that behaves like this :|
        # It could be that Interlacement is set at a codec level for FFV1, but others are
        # declared at the container level..
        if Interlacement == 'n/a':
            Interlacement = get_metadata(
                "//ns:essenceTrackAnnotation[@annotationType='ScanOrder']",
                root, pbcore_namespace)
        interlace_list.append(Interlacement)
        Compression_Mode = ififuncs.get_metadata("//ns:Compression_Mode",
                                                 new_root, mediainfo_namespace)
        colour_range = ififuncs.get_metadata("//ns:colour_range", new_root,
                                             mediainfo_namespace)
        # this needs to be clarified as it exists in general and codec
        format_version = ififuncs.get_metadata("ns:Format_Version",
                                               general_root,
                                               mediainfo_namespace)
        app_company_name = ififuncs.get_metadata(
            "//ns:Encoded_Application_CompanyName", new_root,
            mediainfo_namespace)
        app_name = ififuncs.get_metadata("//ns:Encoded_Application_Name",
                                         new_root, mediainfo_namespace)
        app_version = ififuncs.get_metadata("//ns:Encoded_Application_Version",
                                            new_root, mediainfo_namespace)
        library_name = ififuncs.get_metadata("//ns:Encoded_Library_Name",
                                             new_root, mediainfo_namespace)
        if library_name == 'n/a':
            library_name = ififuncs.get_metadata("//ns:Encoded_Library",
                                                 general_root,
                                                 mediainfo_namespace)
        library_version = ififuncs.get_metadata("//ns:Encoded_Library_Version",
                                                new_root, mediainfo_namespace)
        compression_list.append(Compression_Mode)
        instantiationDate_mo = get_metadata(
            "//ns:instantiationDate[@dateType='file modification']", root,
            pbcore_namespace)
        instantDate_other = 'n/a'
        instantDate_type = 'n/a'
        pix_fmt = ififuncs.get_ffmpeg_fmt(source, 'video')
        pix_fmt_list.append(pix_fmt)
        audio_fmt = ififuncs.get_ffmpeg_fmt(source, 'audio')
        audio_fmt_list.append(audio_fmt)
        essenceBitDepth_vid = ififuncs.get_mediainfo(
            'duration', '--inform=Video;%BitDepth%', source)
    if silence:
        audio_codecid = 'n/a'
        essenceBitDepth_au = 'n/a'
        essenceTrackEncod_au = 'n/a'
        essenceTrackSampling = 'n/a'
        channels = 'n/a'
    '''
    video_codecid = vcodec_attributes['ref']
    video_codecid_list.append(video_codecid)
    try:
        video_codec_version = vcodec_attributes['version']
    except KeyError:
        video_codec_version = 'n/a'
    try:
        video_codec_profile = vcodec_attributes['annotation'][8:]
    except KeyError:
        video_codec_profile = 'n/a'
    '''
    metadata_error = ''
    metadata_list = [
        scan_types, matrix_list, transfer_list, colour_primaries_list,
        color_spaces, chroma, frame_sizes, par_list, container_list, fps_list,
        sample_rate_list, track_count_list, interlace_list, compression_list,
        pix_fmt_list, audio_fmt_list, audio_codecid_list, audio_codec_list,
        au_bitdepth_list, video_codecid_list, video_codec_version_list,
        video_codec_profile_list, channels_list, timecode_list
    ]
    for i in metadata_list:
        if len(set(i)) > 1:
            metadata_error += 'WARNING - Your metadata values are not the same for all files - but this could be a false positive if dealing with atomised audio and video as with DCP: %s\n' % set(
                i)
            if args.p:
                ififuncs.generate_log(
                    sipcreator_log,
                    'EVENT = Metadata mismatch - Your metadata values are not the same for all files - but this could be a false positive if dealing with atomised audio and video as with DCP: %s'
                    % set(i))
    print(metadata_error)
    tc = ififuncs.convert_millis(ms)
    instantiationDuratio = ififuncs.convert_timecode(25, tc)
    if args.donor:
        Donor = args.donor
    else:
        Donor = ''
    Edited_By = user
    Date_Created = ''
    Date_Last_Modified = ''
    Film_Or_Tape = 'Digital AV Object'
    Date_Of_Donation = ''
    if args.reproduction_creator:
        reproduction_creator = args.reproduction_creator
    else:
        reproduction_creator = ''
    if args.acquisition_type:
        if acquisition_type == 'Reproduction':
            Date_Of_Donation = instantiationDate_mo.split('T')[0]
            # if a reproduction, then there's no Donor/transfer of title.
            Donor = 'n/a'
        else:
            Date_Of_Donation = args.donation_date
    Habitat = ''
    backup_habitat = ''
    Type_Of_Deposit = acquisition_type
    if args.depositor_reference:
        Depositor_Reference = args.depositor_reference
    else:
        Depositor_Reference = ''
    Master_Viewing = 'Preservation Object'
    Language_Version = ''
    Condition_Rating = ''
    Companion_Elements = ''
    TTape_Origin = args.parent
    EditedNew = user
    FIO = 'In'
    CollectionTitle = ''
    Created_By = user
    instantTimeStart = 'n/a'
    instantFileSize_gigs = round(
        float(instantFileSize_byte) / 1024 / 1024 / 1024, 3)
    instantColors = 'n/a'
    instantLanguage = 'n/a'
    instantAltMo = 'n/a'

    instantiationChanCon = 'n/a'
    '''
    no idea why these are here
    colour_range = colour_range
    format_version = format_version
    '''
    TimeCode_FirstFrame = process_mixed_values(timecode_list)
    pix_fmt = process_mixed_values(pix_fmt_list)
    audio_fmt = process_mixed_values(audio_fmt_list)
    instantTracks = process_mixed_values(track_count_list)
    TimeCode_Source = timecode_source
    reproduction_reason = ''
    dig_object_descrip = ififuncs.get_digital_object_descriptor(args.input)
    if 'STL' in dig_object_descrip:
        dig_object_descrip = 'AS-11 package'
    dcp_check = ififuncs.find_cpl(args.input)
    if dcp_check is not None:
        essenceFrameSize, ChromaSubsampling, ColorSpace, FrameCount, essenceAspectRatio, instantiationDuratio, PixelAspectRatio, ScanType, dig_object_descrip, instantTracks, instantDataRate, essenceBitDepth_vid, instantMediaty = check_dcp(
            dcp_check)
    ififuncs.append_csv(csv_filename, [
        Reference_Number,
        Donor,
        Edited_By,
        Date_Created,
        Date_Last_Modified,
        Film_Or_Tape,
        Date_Of_Donation,
        Accession_Number,
        Habitat,
        backup_habitat,
        TTape_Origin,
        Type_Of_Deposit,
        Depositor_Reference,
        Master_Viewing,
        Language_Version,
        Condition_Rating,
        Companion_Elements,
        EditedNew,
        FIO,
        CollectionTitle,
        Created_By,
        instantiationIdentif,
        instantDate_other,
        instantDate_type,
        instantiationDate_mo,
        instantiationStandar,
        instantMediaty,
        instantFileSize_byte,
        instantFileSize_gigs,
        instantTimeStart,
        instantDataRate,
        instantTracks,
        instantColors,
        instantLanguage,
        instantAltMo,
        essenceTrackEncodvid,
        essenceFrameRate,
        essenceTrackSampling,
        essenceBitDepth_vid,
        essenceFrameSize,
        essenceAspectRatio,
        essenceTrackEncod_au,
        essenceBitDepth_au,
        instantiationDuratio,
        instantiationChanCon,
        PixelAspectRatio,
        FrameCount,
        ColorSpace,
        ChromaSubsampling,
        ScanType,
        Interlacement,
        Compression_Mode,
        colour_primaries,
        transfer_characteris,
        matrix_coefficients,
        pix_fmt,
        audio_fmt,
        audio_codecid,
        video_codecid,
        video_codec_version,
        video_codec_profile,
        channels,
        colour_range,
        format_version,
        TimeCode_FirstFrame,
        TimeCode_Source,
        app_company_name,
        app_name,
        app_version,
        library_name,
        library_version,
        reproduction_creator,
        reproduction_reason,
        dig_object_descrip,
    ])
    if args.p:
        ififuncs.generate_log(
            sipcreator_log,
            'EVENT = Metadata extraction - eventDetail=Technical record creation using PBCore, eventOutcome=%s, agentName=makepbcore'
            % (csv_filename))
        ififuncs.generate_log(sipcreator_log, 'EVENT = makepbcore.py finished')
        ififuncs.checksum_replace(md5_manifest, sipcreator_log, 'md5')
        ififuncs.checksum_replace(sha512_manifest, sipcreator_log, 'sha512')
        ififuncs.manifest_update(md5_manifest, csv_filename)
        print((' - Updating %s with %s' % (md5_manifest, csv_filename)))
        ififuncs.sha512_update(sha512_manifest, csv_filename)
        print((' - Updating %s with %s' % (sha512_manifest, csv_filename)))
        print(metadata_error)
Ejemplo n.º 7
0
def main(args_):

    # if multiple file are present, this script will treat them as a single
    # instantiation/representation and get aggregate metadata about the whole
    # package. For now, this will be a clumsy implementation - the first file
    # will provide most metadata. Things like duration/bitrate/filesize
    # will be calculated as a whole.
    # Although another way would be that every call is looped, and if
    # this could catch files that should not be in the package, eg. a 4:2:2
    # file in a 4:2:0 package..
    # yup - do it that way!
    args = parse_args(args_)
    all_files = ififuncs.recursive_file_list(args.input)
    silence = True
    if args.user:
        user = args.user
    else:
        user = ififuncs.get_user()
    for dirs in os.listdir(args.input):
        if ififuncs.validate_uuid4(dirs) is None:
            instantiationIdentif = dirs
    Accession_Number = get_accession_number(args.input)
    Reference_Number = get_reference_number(args.input)
    if args.p:
        for root, _, filenames in os.walk(args.input):
            if os.path.basename(root) == 'metadata':
                metadata_dir = root
            elif os.path.basename(root) == 'logs':
                logs_dir = root
        csv_filename = os.path.join(metadata_dir, Accession_Number + '.csv')
        sipcreator_log = os.path.join(logs_dir,
                                      instantiationIdentif + '_sip_log.log')
        ififuncs.generate_log(sipcreator_log, 'EVENT = makepbcore.py started')
        ififuncs.generate_log(
            sipcreator_log, 'eventDetail=makepbcore.py %s' %
            ififuncs.get_script_version('makepbcore.py'))
        ififuncs.generate_log(sipcreator_log,
                              'Command line arguments: %s' % args)
        ififuncs.generate_log(sipcreator_log, 'EVENT = agentName=%s' % user)
    else:
        csv_filename = 'blaa.csv'
    print ' - Metadata will be stored in %s' % csv_filename
    for filenames in os.listdir(args.input):
        if '_manifest.md5' in filenames:
            md5_manifest = os.path.join(args.input, filenames)
        elif 'manifest-sha512.txt' in filenames:
            sha512_manifest = os.path.join(args.input, filenames)
    make_csv(csv_filename)
    ms = 0
    FrameCount = 0
    instantFileSize = 0
    instantFileSize_gigs = 0
    for source in all_files:
        metadata = subprocess.check_output(
            ['mediainfo', '--Output=PBCore2', source])
        root = etree.fromstring(metadata)
        print(' - Analysing  %s') % source
        pbcore_namespace = root.xpath('namespace-uri(.)')
        track_type = root.xpath('//ns:essenceTrackType',
                                namespaces={'ns': pbcore_namespace})
        if len(track_type) > 0:
            for track in track_type:
                if track.text == 'Video':
                    essenceTrackEncodvid = get_metadata(
                        "ns:essenceTrackEncoding", track.getparent(),
                        pbcore_namespace)
                    vcodec_attributes = get_attributes(track.getparent(),
                                                       pbcore_namespace)
                elif track.text == 'Audio':
                    silence = False
                    essenceTrackEncod_au = get_metadata(
                        "ns:essenceTrackEncoding", track.getparent(),
                        pbcore_namespace)
                    acodec_attributes = get_attributes(track.getparent(),
                                                       pbcore_namespace)
        ScanType = get_metadata(
            "//ns:essenceTrackAnnotation[@annotationType='ScanType']", root,
            pbcore_namespace)
        matrix_coefficients = get_metadata(
            "//ns:essenceTrackAnnotation[@annotationType='matrix_coefficients']",
            root, pbcore_namespace)
        transfer_characteris = get_metadata(
            "//ns:essenceTrackAnnotation[@annotationType='transfer_characteristics']",
            root, pbcore_namespace)
        colour_primaries = get_metadata(
            "//ns:essenceTrackAnnotation[@annotationType='color_primaries']",
            root, pbcore_namespace)
        FrameCount += int(
            get_metadata(
                "//ns:essenceTrackAnnotation[@annotationType='FrameCount']",
                root, pbcore_namespace))
        instantFileSize += int(
            get_metadata("//ns:instantiationFileSize", root, pbcore_namespace))
        instantDataRate = round(
            float(
                ififuncs.get_mediainfo('OverallBitRate',
                                       '--inform=General;%OverallBitRate%',
                                       source)) / 1000 / 1000, 2)
        ms += ififuncs.get_milliseconds(source)
        ColorSpace = get_metadata(
            "//ns:essenceTrackAnnotation[@annotationType='ColorSpace']", root,
            pbcore_namespace)
        ChromaSubsampling = get_metadata(
            "//ns:essenceTrackAnnotation[@annotationType='ChromaSubsampling']",
            root, pbcore_namespace)
        instantMediaty = get_metadata("//ns:instantiationMediaType", root,
                                      pbcore_namespace)
        essenceFrameSize = get_metadata("//ns:essenceTrackFrameSize", root,
                                        pbcore_namespace)
        essenceAspectRatio = get_metadata("//ns:essenceTrackAspectRatio", root,
                                          pbcore_namespace)
        PixelAspectRatio = get_metadata(
            "//ns:essenceTrackAnnotation[@annotationType='PixelAspectRatio']",
            root, pbcore_namespace)
        instantiationStandar = get_metadata(
            "//ns:instantiationAnnotation[@annotationType='Format']", root,
            pbcore_namespace)
        essenceFrameRate = get_metadata("//ns:essenceTrackFrameRate", root,
                                        pbcore_namespace)
        essenceTrackSampling = get_metadata("//ns:essenceTrackSamplingRate",
                                            root, pbcore_namespace)
        Interlacement = get_metadata(
            "//ns:instantiationAnnotation[@annotationType='Interlacement']",
            root, pbcore_namespace)
        Compression_Mode = get_metadata(
            "//ns:instantiationAnnotation[@annotationType='Compression_Mode']",
            root, pbcore_namespace)
        instantiationDate_modified = get_metadata(
            "//ns:instantiationDate[@dateType='file modification']", root,
            pbcore_namespace)
        pix_fmt = ififuncs.get_ffmpeg_fmt(source, 'video')
        audio_fmt = ififuncs.get_ffmpeg_fmt(source, 'audio')
    if not silence:
        audio_codecid = acodec_attributes['ref']
        essenceBitDepth_au = ififuncs.get_mediainfo(
            'duration', '--inform=Audio;%BitDepth%', source)
    else:
        audio_codecid = 'n/a'
        essenceBitDepth_au = 'n/a'
        essenceTrackEncod_au = 'n/a'
    video_codecid = vcodec_attributes['ref']
    try:
        video_codec_version = vcodec_attributes['version']
    except KeyError:
        video_codec_version = 'n/a'
    try:
        video_codec_profile = vcodec_attributes['annotation'][8:]
    except KeyError:
        video_codec_profile = 'n/a'
    tc = ififuncs.convert_millis(ms)
    instantiationDuratio = ififuncs.convert_timecode(25, tc)
    Donor = ''
    Edited_By = user
    Date_Created = ''
    Date_Last_Modified = ''
    Film_Or_Tape = 'Digital File'
    Date_Of_Donation = ''
    Habitat = ''
    Type_Of_Deposit = ''
    Depositor_Reference = ''
    Master_Viewing = 'Preservation Master'
    Language_Version = ''
    Condition_Rating = ''
    Companion_Elements = ''
    EditedNew = user
    FIO = 'In'
    CollectionTitle = ''
    Created_By = user
    instantiationDimensi = ''
    instantiationLocatio = ''
    instantTimeStart = ''
    instantFileSize_gigs = round(
        float(instantFileSize) / 1024 / 1024 / 1024, 3)
    instantColors = ''
    instantLanguage = ''
    instantAltMo = 'n/a'
    essenceBitDepth_vid = ififuncs.get_mediainfo('duration',
                                                 '--inform=Video;%BitDepth%',
                                                 source)
    instantiationChanCon = ''
    ififuncs.append_csv(csv_filename, [
        Reference_Number, Donor, Edited_By, Date_Created, Date_Last_Modified,
        Film_Or_Tape, Date_Of_Donation, Accession_Number, Habitat,
        Type_Of_Deposit, Depositor_Reference, Master_Viewing, Language_Version,
        Condition_Rating, Companion_Elements, EditedNew, FIO, CollectionTitle,
        Created_By, instantiationIdentif, instantiationDate_modified,
        instantiationDimensi, instantiationStandar, instantiationLocatio,
        instantMediaty, instantFileSize, instantFileSize_gigs,
        instantTimeStart, instantDataRate, instantColors, instantLanguage,
        instantAltMo, essenceTrackEncodvid, essenceFrameRate,
        essenceTrackSampling, essenceBitDepth_vid, essenceFrameSize,
        essenceAspectRatio, essenceTrackEncod_au, essenceBitDepth_au,
        instantiationDuratio, instantiationChanCon, PixelAspectRatio,
        FrameCount, ColorSpace, ChromaSubsampling, ScanType, Interlacement,
        Compression_Mode, colour_primaries, transfer_characteris,
        matrix_coefficients, pix_fmt, audio_fmt, audio_codecid, video_codecid,
        video_codec_version, video_codec_profile
    ])
    if args.p:
        ififuncs.generate_log(
            sipcreator_log,
            'EVENT = Metadata extraction - eventDetail=Technical record creation using PBCore, eventOutcome=%s, agentName=makepbcore'
            % (csv_filename))
        ififuncs.generate_log(sipcreator_log, 'EVENT = makepbcore.py finished')
        ififuncs.checksum_replace(md5_manifest, sipcreator_log, 'md5')
        ififuncs.checksum_replace(sha512_manifest, sipcreator_log, 'sha512')
        ififuncs.manifest_update(md5_manifest, csv_filename)
        print ' - Updating %s with %s' % (md5_manifest, csv_filename)
        ififuncs.sha512_update(sha512_manifest, csv_filename)
        print ' - Updating %s with %s' % (sha512_manifest, csv_filename)
Ejemplo n.º 8
0
def main(args_):
    '''
    Launches the various functions that will accession a package
    '''
    args = parse_args(args_)
    source = args.input
    uuid_directory = ififuncs.check_for_sip([source])
    if uuid_directory is not None:
        oe_path = os.path.dirname(uuid_directory)
        oe_number = os.path.basename(oe_path)
        if args.user:
            user = args.user
        else:
            user = ififuncs.get_user()
        if args.number:
            if args.number[:3] != 'aaa':
                print 'First three characters must be \'aaa\' and last four characters must be four digits'
                accession_number = ififuncs.get_accession_number()
            elif len(args.number[3:]) != 4:
                accession_number = ififuncs.get_accession_number()
                print 'First three characters must be \'aaa\' and last four characters must be four digits'
            elif not args.number[3:].isdigit():
                accession_number = ififuncs.get_accession_number()
                print 'First three characters must be \'aaa\' and last four characters must be four digits'
            else:
                accession_number = args.number
        else:
            accession_number = ififuncs.get_accession_number()
        if args.pbcore:
            if args.reference:
                Reference_Number = args.reference.upper()
            else:
                Reference_Number = ififuncs.get_reference_number()
        if args.acquisition_type:
            acquisition_type = ififuncs.get_acquisition_type(
                args.acquisition_type)
            print acquisition_type
        accession_path = os.path.join(os.path.dirname(oe_path),
                                      accession_number)
        uuid = os.path.basename(uuid_directory)
        new_uuid_path = os.path.join(accession_path, uuid)
        logs_dir = os.path.join(new_uuid_path, 'logs')
        sipcreator_log = os.path.join(logs_dir, uuid) + '_sip_log.log'
        if args.force:
            proceed = 'Y'
        else:
            proceed = ififuncs.ask_yes_no('Do you want to rename %s with %s' %
                                          (oe_number, accession_number))
        if proceed == 'Y':
            os.rename(oe_path, accession_path)
        if args.register:
            register = args.register
        else:
            register = make_register()
        ififuncs.append_csv(register,
                            (oe_number.upper()[:2] + '-' + oe_number[2:],
                             accession_number, '', '', '', '', '', ''))
        ififuncs.generate_log(sipcreator_log, 'EVENT = accession.py started')
        ififuncs.generate_log(
            sipcreator_log, 'eventDetail=accession.py %s' %
            ififuncs.get_script_version('accession.py'))
        ififuncs.generate_log(sipcreator_log,
                              'Command line arguments: %s' % args)
        ififuncs.generate_log(sipcreator_log, 'EVENT = agentName=%s' % user)
        ififuncs.generate_log(
            sipcreator_log, 'EVENT = eventType=Identifier assignment,'
            ' eventIdentifierType=accession number, value=%s' %
            accession_number)
        ififuncs.generate_log(
            sipcreator_log, 'EVENT = eventType=accession,'
            ' eventIdentifierType=accession number, value=%s' %
            accession_number)
        sip_manifest = os.path.join(accession_path, uuid) + '_manifest.md5'
        sha512_log = manifest.main([new_uuid_path, '-sha512', '-s'])
        sha512_manifest = os.path.join(os.path.dirname(new_uuid_path),
                                       uuid + '_manifest-sha512.txt')
        ififuncs.merge_logs_append(sha512_log, sipcreator_log, sip_manifest)
        os.remove(sha512_log)
        print('Generating Digital Forensics XML')
        dfxml = make_dfxml(args, new_uuid_path, uuid)
        ififuncs.generate_log(
            sipcreator_log,
            'EVENT = Metadata extraction - eventDetail=File system metadata extraction using Digital Forensics XML, eventOutcome=%s, agentName=makedfxml'
            % (dfxml))
        # this is inefficient. The script should not have to ask for reference
        # number twice if someone wants to insert the filmographic but do not
        # want to make the pbcore csv, perhaps because the latter already exists.
        if args.csv:
            metadata_dir = os.path.join(new_uuid_path, 'metadata')
            package_filmographic = os.path.join(
                metadata_dir, Reference_Number + '_filmographic.csv')
            insert_filmographic(args.csv, Reference_Number,
                                package_filmographic)
            ififuncs.generate_log(
                sipcreator_log,
                'EVENT = Metadata extraction - eventDetail=Filmographic descriptive metadata added to metadata folder, eventOutcome=%s, agentName=accession.py'
                % (package_filmographic))
            ififuncs.manifest_update(sip_manifest, package_filmographic)
            ififuncs.sha512_update(sha512_manifest, package_filmographic)
            print('Filmographic descriptive metadata added to metadata folder')
        ififuncs.generate_log(sipcreator_log, 'EVENT = accession.py finished')
        ififuncs.checksum_replace(sip_manifest, sipcreator_log, 'md5')
        ififuncs.checksum_replace(sha512_manifest, sipcreator_log, 'sha512')
        ififuncs.manifest_update(sip_manifest, dfxml)
        ififuncs.sha512_update(sha512_manifest, dfxml)
        if args.pbcore:
            makepbcore_cmd = [
                accession_path, '-p', '-user', user, '-reference',
                Reference_Number
            ]
            if args.parent:
                makepbcore_cmd.extend(['-parent', args.parent])
            if args.acquisition_type:
                makepbcore_cmd.extend(
                    ['-acquisition_type', args.acquisition_type])
            if args.donor:
                makepbcore_cmd.extend(['-donor', args.donor])
            if args.donor:
                makepbcore_cmd.extend(
                    ['-depositor_reference', args.depositor_reference])
            if args.donation_date:
                makepbcore_cmd.extend(['-donation_date', args.donation_date])
            makepbcore.main(makepbcore_cmd)
    else:
        print 'not a valid package. The input should include a package that has been through Object Entry'
Ejemplo n.º 9
0
    user = '******'
    print 'Hi Brian, Congratulations on becoming a father!!!'
elif user == '2':
    user = '******'
    print 'Hi Gavin, Have you renewed your subscription to American Cinematographer?'
elif user == '3':
    user = '******'
    print 'Hi Raelene, Brian Cash must be on paternity leave'
    time.sleep(1)

create_csv(csv_report_filename, ('Sequence Name', 'Lossless?', 'Start time', 'Finish Time'))
for source_directory in all_files:
    for root,dirnames,filenames in os.walk(source_directory):
            source_directory = root
            if not file_check(source_directory) == 'TIFF':
                append_csv(csv_report_filename, (source_directory,'EMPTY DIRECTORY - SKIPPED', 'n/a', 'n/a'))
                continue

            root_dir = os.path.dirname(os.path.dirname(root))
            general_log = root_dir + '/logs/image/%s_image_log.log' % os.path.basename(root_dir)
            generate_log(general_log, 'Input = %s' % root)
            remove_bad_files(source_directory)
            source_parent_dir           = os.path.dirname(source_directory)
            normpath                    = os.path.normpath(source_directory)
            relative_path               = normpath.split(os.sep)[-1]
            split_path                  = os.path.split(os.path.basename(source_directory))[1]
            start                       = datetime.datetime.now()
            source_manifest             = root_dir + '/%s_manifest.md5' % relative_path
            generate_log(general_log, 'Generating source manifest via md5deep and storing as  %s' % source_manifest)
            make_manifest(root_dir, root_dir, source_manifest)
            info                        = make_framemd5(source_directory, 'tiff', 'tiff_framemd5')
Ejemplo n.º 10
0
def make_ffv1(video_files, csv_report_filename):
    for filename in video_files:  #loop all files in directory
        filenoext = os.path.splitext(filename)[0]
        # Generate new directory names
        metadata_dir = "%s/metadata" % filenoext
        log_dir = "%s/logs" % filenoext
        data_dir = "%s/objects" % filenoext
        # Actually create the directories.
        os.makedirs(metadata_dir)
        os.makedirs(data_dir)
        os.makedirs(log_dir)
        #Generate filenames for new files.
        inputxml = "%s/%s_source_mediainfo.xml" % (metadata_dir,
                                                   os.path.basename(filename))
        inputtracexml = "%s/%s_source_mediatrace.xml" % (
            metadata_dir, os.path.basename(filename))
        output = "%s/%s.mkv" % (
            data_dir, os.path.splitext(os.path.basename(filename))[0])
        # Generate filename of ffv1.mkv without the path.
        outputfilename = os.path.basename(output)
        outputxml = "%s/%s_mediainfo.xml" % (metadata_dir, outputfilename)
        outputtracexml = "%s/%s_mediatrace.xml" % (metadata_dir,
                                                   outputfilename)
        fmd5 = "%s/%s_source.framemd5" % (metadata_dir,
                                          os.path.basename(filename))
        fmd5ffv1 = "%s/%s_ffv1.framemd5" % (metadata_dir, outputfilename)
        log = "%s/%s_log.log" % (log_dir, filename)
        generate_log(log, 'Input = %s' % filename)
        generate_log(log, 'Output = %s' % output)
        generate_log(
            log,
            'makeffv1.py transcode to FFV1 and framemd5 generation of source started.'
        )
        ffv1_logfile = log_dir + '/%s_ffv1_transcode.log' % filename
        ffv1_env_dict = set_environment(ffv1_logfile)
        par = subprocess.check_output([
            'mediainfo', '--Language=raw', '--Full',
            "--Inform=Video;%PixelAspectRatio%", filename
        ]).rstrip()
        field_order = subprocess.check_output([
            'mediainfo', '--Language=raw', '--Full',
            "--Inform=Video;%ScanType%", filename
        ]).rstrip()
        height = subprocess.check_output([
            'mediainfo', '--Language=raw', '--Full', "--Inform=Video;%Height%",
            filename
        ]).rstrip()
        # Transcode video file writing frame md5 and output appropriately
        ffv1_command = [
            'ffmpeg',
            '-i',
            filename,
            '-c:v',
            'ffv1',  # Use FFv1 codec
            '-g',
            '1',  # Use intra-frame only aka ALL-I aka GOP=1
            '-level',
            '3',  # Use Version 3 of FFv1
            '-c:a',
            'copy',  # Copy and paste audio bitsream with no transcoding
            '-map',
            '0',
            '-dn',
            '-report',
            '-slicecrc',
            '1',
            '-slices',
            '16',
        ]
        # check for FCP7 lack of description and PAL
        if par == '1.000':
            if field_order == '':
                if height == '576':
                    ffv1_command += ['-vf', 'setfield=tff, setdar=4/3']
        ffv1_command += [
            output,
            '-f',
            'framemd5',
            '-an',  # Create decoded md5 checksums for every frame of the input. -an ignores audio
            fmd5
        ]
        print ffv1_command
        subprocess.call(ffv1_command, env=ffv1_env_dict)
        generate_log(
            log,
            'makeffv1.py transcode to FFV1 and framemd5 generation completed.')
        generate_log(
            log, 'makeffv1.py Framemd5 generation of output file started.')
        fmd5_logfile = log_dir + '/%s_framemd5.log' % outputfilename
        fmd5_env_dict = set_environment(fmd5_logfile)
        pix_fmt = get_ffmpeg_fmt(filename, 'video')
        fmd5_command = [
            'ffmpeg',  # Create decoded md5 checksums for every frame
            '-i',
            output,
            '-report',
            '-pix_fmt',
            pix_fmt,
            '-f',
            'framemd5',
            '-an',
            fmd5ffv1
        ]
        print fmd5_command
        subprocess.call(fmd5_command, env=fmd5_env_dict)
        generate_log(
            log, 'makeffv1.py Framemd5 generation of output file completed')
        source_video_size = get_mediainfo('source_video_size',
                                          "--inform=General;%FileSize%",
                                          filename)
        ffv1_video_size = get_mediainfo('ffv1_video_size',
                                        '--inform=General;%FileSize%', output)
        compression_ratio = float(source_video_size) / float(ffv1_video_size)
        if os.path.basename(sys.argv[0]) == 'makeffv1.py':
            try:
                shutil.copy(sys.argv[0], log_dir)
            except IOError:
                pass
        print 'Generating mediainfo xml of input file and saving it in %s' % inputxml
        make_mediainfo(inputxml, 'mediaxmlinput', filename)
        print 'Generating mediainfo xml of output file and saving it in %s' % outputxml
        make_mediainfo(outputxml, 'mediaxmloutput', output)
        print 'Generating mediatrace xml of input file and saving it in %s' % inputtracexml
        make_mediatrace(inputtracexml, 'mediatracexmlinput', filename)
        print 'Generating mediatrace xml of output file and saving it in %s' % outputtracexml
        make_mediatrace(outputtracexml, 'mediatracexmloutput', output)
        source_parent_dir = os.path.dirname(os.path.abspath(filename))
        manifest = '%s/%s_manifest.md5' % (source_parent_dir, filenoext)
        generate_log(log, 'makeffv1.py MD5 manifest started')
        checksum_mismatches = []
        with open(fmd5) as f1:
            with open(fmd5ffv1) as f2:
                for (lineno1, line1), (lineno2, line2) in itertools.izip(
                        read_non_comment_lines(f1),
                        read_non_comment_lines(f2)):
                    if line1 != line2:
                        if 'sar' in line1:
                            checksum_mismatches = ['sar']
                        else:
                            checksum_mismatches.append(1)
        if len(checksum_mismatches) == 0:
            print 'LOSSLESS'
            append_csv(csv_report_filename,
                       (output, 'LOSSLESS', source_video_size, ffv1_video_size,
                        compression_ratio))
            generate_log(log, 'makeffv1.py Transcode was lossless')
        elif len(checksum_mismatches) == 1:
            if checksum_mismatches[0] == 'sar':
                print 'Image content is lossless,'
                ' Pixel Aspect Ratio has been altered.'
                ' Update ffmpeg in order to resolve the PAR issue.'
                append_csv(
                    csv_report_filename,
                    (output, 'LOSSLESS - different PAR', source_video_size,
                     ffv1_video_size, compression_ratio))
                generate_log(
                    log,
                    'makeffv1.py Image content is lossless but Pixel Aspect Ratio has been altered.Update ffmpeg in order to resolve the PAR issue.'
                )
        elif len(checksum_mismatches) > 1:
            print 'NOT LOSSLESS'
            append_csv(csv_report_filename,
                       (output, 'NOT LOSSLESS', source_video_size,
                        ffv1_video_size, compression_ratio))
            generate_log(log, 'makeffv1.py Not Lossless.')
        hashlib_manifest(filenoext, manifest, source_parent_dir)
        if filecmp.cmp(fmd5, fmd5ffv1, shallow=False):
            print "YOUR FILES ARE LOSSLESS YOU SHOULD BE SO HAPPY!!!"
        else:
            print "The framemd5 text files are not completely identical."
            " This may be because of a lossy transcode,"
            " or a change in metadata, most likely pixel aspect ratio."
            " Please analyse the framemd5 files for source and output."
Ejemplo n.º 11
0
def main(args_):
    '''
    Launches the various functions that will accession a package
    '''
    args = parse_args(args_)
    input = args.input
    uuid_directory = ififuncs.check_for_sip([input])
    if uuid_directory is not None:
        oe_path = os.path.dirname(uuid_directory)
        oe_number = os.path.basename(oe_path)
        if args.user:
            user = args.user
        else:
            user = ififuncs.get_user()
        if args.number:
            if args.number[:3] != 'aaa':
                print 'First three characters must be \'aaa\' and last four characters must be four digits'
                accession_number = ififuncs.get_accession_number()
            elif len(args.number[3:]) != 4:
                accession_number = ififuncs.get_accession_number()
                print 'First three characters must be \'aaa\' and last four characters must be four digits'
            elif not args.number[3:].isdigit():
                accession_number = ififuncs.get_accession_number()
                print 'First three characters must be \'aaa\' and last four characters must be four digits'
            else:
                accession_number = args.number
        else:
            accession_number = ififuncs.get_accession_number()
        accession_path = os.path.join(
            os.path.dirname(oe_path), accession_number
        )
        uuid = os.path.basename(uuid_directory)
        new_uuid_path = os.path.join(accession_path, uuid)
        logs_dir = os.path.join(new_uuid_path, 'logs')
        sipcreator_log = os.path.join(logs_dir, uuid) + '_sip_log.log'
        if args.force:
            proceed = 'Y'
        else:
            proceed = ififuncs.ask_yes_no(
                'Do you want to rename %s with %s' % (oe_number, accession_number)
            )
        if proceed == 'Y':
            os.rename(oe_path, accession_path)
        register = make_register()
        ififuncs.append_csv(register, (oe_number.upper()[:2] + '-' + oe_number[2:6], accession_number, '','','','', ''))
        ififuncs.generate_log(
            sipcreator_log,
            'EVENT = accession.py started'
        )
        ififuncs.generate_log(
            sipcreator_log,
            'eventDetail=accession.py %s' % ififuncs.get_script_version('accession.py')
        )
        ififuncs.generate_log(
            sipcreator_log,
            'Command line arguments: %s' % args
        )
        ififuncs.generate_log(
            sipcreator_log,
            'EVENT = agentName=%s' % user
        )
        ififuncs.generate_log(
            sipcreator_log,
            'EVENT = eventType=Identifier assignment,'
            ' eventIdentifierType=accession number, value=%s'
            % accession_number
        )
        ififuncs.generate_log(
            sipcreator_log,
            'EVENT = eventType=accession,'
            ' eventIdentifierType=accession number, value=%s'
            % accession_number
        )
        sip_manifest = os.path.join(
            accession_path, uuid
            ) + '_manifest.md5'
        sha512_log = manifest.main([new_uuid_path, '-sha512', '-s'])
        sha512_manifest = os.path.join(
            os.path.dirname(new_uuid_path), uuid + '_manifest-sha512.txt'
        )
        ififuncs.merge_logs_append(sha512_log, sipcreator_log, sip_manifest)
        os.remove(sha512_log)
        dfxml = make_dfxml(args, new_uuid_path, uuid)
        ififuncs.generate_log(
            sipcreator_log,
            'EVENT = Metadata extraction - eventDetail=File system metadata extraction using Digital Forensics XML, eventOutcome=%s, agentName=makedfxml' % (dfxml)
        )
        ififuncs.generate_log(
            sipcreator_log,
            'EVENT = accession.py finished'
        )
        ififuncs.checksum_replace(sip_manifest, sipcreator_log, 'md5')
        ififuncs.checksum_replace(sha512_manifest, sipcreator_log, 'sha512')
        ififuncs.manifest_update(sip_manifest, dfxml)
        ififuncs.sha512_update(sha512_manifest, dfxml)
        if args.pbcore:
            makepbcore.main([accession_path, '-p', '-user', user])
    else:
        print 'not a valid package. The input should include a package that has been through Object Entry'
Ejemplo n.º 12
0
def main(args_):
    '''
    Retrospectively updates older FFV1/DV packages in order to meet our current
    packaging requirements. This should allow accession.py and makepbcore.py to run as
    expected. This script should work on files created by:
    makeffv1.py
    dvsip.py
    loopline.py
    '''
    args = parse_args(args_)
    user = ififuncs.get_user()
    new_object_entry = get_numbers(args)
    filmographic_csv = args.filmographic
    technical_csv = args.technical
    filmographic_oe_list = []
    filmo_csv_extraction = ififuncs.extract_metadata(filmographic_csv)
    tech_csv_extraction = ififuncs.extract_metadata(technical_csv)
    register = make_register()
    for line_item in filmo_csv_extraction[0]:
        dictionary = {}
        oe_number = line_item['Object Entry'].lower()
        dictionary['title'] = line_item['Title']
        if dictionary['title'] == '':
            dictionary['title'] = '%s - %s' % (line_item['TitleSeries'],
                                               line_item['EpisodeNo'])
        dictionary['uppercase_dashed_oe'] = oe_number.upper()
        for tech_record in tech_csv_extraction[0]:
            if tech_record['Reference Number'] == dictionary[
                    'uppercase_dashed_oe']:
                dictionary['source_accession_number'] = tech_record[
                    'Accession Number']
                dictionary['filmographic_reference_number'] = tech_record[
                    'new_ref']
                # this transforms OE-#### to oe####
                dictionary['old_oe'] = oe_number[:2] + oe_number[3:]
                filmographic_oe_list.append(dictionary)
    for oe_package in filmographic_oe_list:
        for root, _, filenames in os.walk(args.input):
            if os.path.basename(root) == oe_package['old_oe']:
                old_oe_path = root
                old_oe = os.path.basename(root)
                log_dir = os.path.join(root, 'logs')
                for files in os.listdir(log_dir):
                    if '.mov_log.log' in files:
                        log = os.path.join(log_dir, files)
                manifest = os.path.join(os.path.dirname(root),
                                        old_oe + '_manifest.md5')
                uuid = ififuncs.create_uuid()
                uuid_event = (
                    'EVENT = eventType=Identifier assignement,'
                    ' eventIdentifierType=UUID, value=%s, module=uuid.uuid4'
                ) % uuid
                ififuncs.generate_log(log,
                                      'EVENT = loopline_repackage.py started')
                ififuncs.generate_log(
                    log, 'eventDetail=loopline_repackage.py %s' %
                    ififuncs.get_script_version('loopline_repackage.py'))
                ififuncs.generate_log(log, 'Command line arguments: %s' % args)
                ififuncs.generate_log(log, 'EVENT = agentName=%s' % user)
                ififuncs.generate_log(log, uuid_event)
                ififuncs.generate_log(
                    log, 'EVENT = eventType=Identifier assignement,'
                    ' eventIdentifierType=object entry, value=%s' %
                    new_object_entry)
                ififuncs.generate_log(
                    log, 'EVENT = eventType=Identifier assignement,'
                    ' eventIdentifierType=Filmographic reference number , value=%s'
                    % oe_package['filmographic_reference_number'])
                oe_package['new_object_entry'] = new_object_entry
                print('Transforming %s into %s' %
                      (oe_package['old_oe'], oe_package['new_object_entry']))
                ififuncs.generate_log(
                    log, 'Relationship, derivation, has source=%s' %
                    oe_package['source_accession_number'])
                old_uuid_path = os.path.join(os.path.dirname(root), uuid)
                new_oe_path, new_uuid_path = move_files(
                    root, new_object_entry, old_oe_path, old_uuid_path, uuid)
                updated_lines = update_manifest(manifest, old_oe, uuid)
                new_manifest = os.path.join(new_oe_path,
                                            uuid) + '_manifest.md5'
                shutil.move(manifest, new_manifest)
                with open(new_manifest, 'w') as fo:
                    for lines in updated_lines:
                        fo.write(lines)
                new_logs_path = os.path.join(new_uuid_path, 'logs')
                for files in os.listdir(new_logs_path):
                    if '.mov_log.log' in files:
                        log = os.path.join(new_logs_path, files)
                logname = rename_files(new_uuid_path, old_oe, uuid,
                                       new_manifest, log)
                date_modified, extension = get_date_modified(new_uuid_path)
                # This normally would be bad practise, but this project only has two formats. MOV/DV and FFv1/MKV
                if extension == '.mkv':
                    av_format = 'FFV1/PCM/Matroska'
                elif extension == '.mov':
                    av_format = 'DV/PCM/QuickTime'
                provenance_string = 'Reproduction of %s' % oe_package[
                    'source_accession_number']
                ififuncs.append_csv(
                    register,
                    (oe_package['new_object_entry'].upper()[:2] + '-' +
                     oe_package['new_object_entry'][2:], date_modified, '1',
                     av_format, oe_package['title'], 'contact_name',
                     'Reproduction', '', provenance_string, '', ''))
                ififuncs.generate_log(
                    logname, 'EVENT = loopline_repackage.py finished')
                ififuncs.checksum_replace(new_manifest, logname, 'md5')
                oe_digits = int(os.path.basename(new_oe_path)[2:]) + 1
                new_object_entry = 'oe' + str(oe_digits)
Ejemplo n.º 13
0
        os.path.abspath(dpx_filename), '-strict', '-2', '-c:v', 'ffv1',
        '-level', '3', '-pix_fmt', 'rgb48le',
        output_dirname + '/video/' + os.path.basename(root) + '.mkv'
    ]
    print ffv12dpx

    subprocess.call(ffv12dpx, env=env_dict)
    parent_basename = os.path.basename(output_dirname)
    manifest_textfile = os.path.dirname(
        output_dirname) + '/' + parent_basename + '_manifest.md5'
    ffv1_path = output_dirname + '/video/' + os.path.basename(root) + '.mkv'
    ffv1_md5 = output_dirname + '/md5/' + os.path.basename(
        root) + 'ffv1.framemd5'
    subprocess.call([
        'ffmpeg', '-i', ffv1_path, '-pix_fmt', 'rgb48le', '-f', 'framemd5',
        ffv1_md5
    ])
    #other_textfile = other[1]
    judgement = diff_textfiles(source_textfile, ffv1_md5)
    make_manifest(output_parent_directory, os.path.basename(output_dirname),
                  manifest_textfile)
    finish = datetime.datetime.now()

    print total_size
    ffv1_size = os.path.getsize(ffv1_path)
    comp_ratio = float(total_size) / float(os.path.getsize(ffv1_path))
    append_csv(csv_report_filename, (parent_basename, judgement, start, finish,
                                     total_size, ffv1_size, comp_ratio))

#send_gmail(emails, csv_report_filename, 'makedpx completed', 'Hi,\n Please the attached log for details of the makedpx job, \nSincerely yours,\nIFIROBOT', config[2].rstrip(), config[3].rstrip())
Ejemplo n.º 14
0
        source_directory = root + '/tiff_scans'
        remove_bad_files(source_directory)
        source_parent_dir    = os.path.dirname(source_directory)
        normpath             = os.path.normpath(source_directory) 
        relative_path        = normpath.split(os.sep)[-1]
        split_path           = os.path.split(os.path.basename(source_directory))[1]
        start = datetime.datetime.now()
        source_manifest = source_parent_dir + '/%s_manifest.md5' % relative_path
        make_manifest(os.path.dirname(source_directory), os.path.basename(source_directory), source_manifest)
        info = make_framemd5(source_directory, 'tiff', 'tiff_framemd5')
        output_dirname = info[0]  
        source_textfile = info[1]
        image_seq_without_container = info[2]
        tiff_filename = image_seq_without_container + "%06d.tiff" 
        dpx_filename = image_seq_without_container + "%06d.dpx" 

        logfile = output_dirname + '/image/logs/%sdpx_transcode.log' % image_seq_without_container
        env_dict = set_environment(logfile)
        tiff2dpx = ['ffmpegnometadata','-report','-f','image2','-framerate','24', '-i', tiff_filename ,output_dirname +  '/image/dpx_files' '/' + dpx_filename]
        print tiff2dpx

        subprocess.call(tiff2dpx,env=env_dict)
        parent_basename =  os.path.basename(output_dirname)
        manifest_textfile = os.path.dirname(output_dirname) + '/' +  parent_basename + '_manifest.md5'
        other = make_framemd5(output_dirname + '/image/dpx_files', 'dpx', 'dpx_framemd5')
        other_textfile = other[1]
        judgement = diff_textfiles(source_textfile, other_textfile)
        make_manifest(output_parent_directory, os.path.basename(output_dirname), manifest_textfile)
        finish = datetime.datetime.now()
        append_csv(csv_report_filename, (parent_basename,judgement, start, finish))
#send_gmail(emails, csv_report_filename, 'makedpx completed', 'Hi,\n Please the attached log for details of the makedpx job, \nSincerely yours,\nIFIROBOT', config[2].rstrip(), config[3].rstrip())
Ejemplo n.º 15
0
def make_ffv1(video_files, csv_report_filename):
    for filename in video_files: #loop all files in directory
        filenoext = os.path.splitext(filename)[0]
        # Generate new directory names
        metadata_dir = "%s/metadata" % filenoext
        log_dir = "%s/logs" % filenoext
        data_dir = "%s/objects" % filenoext
        # Actually create the directories.
        os.makedirs(metadata_dir)
        os.makedirs(data_dir)
        os.makedirs(log_dir)
        #Generate filenames for new files.
        inputxml = "%s/%s_source_mediainfo.xml" % (
            metadata_dir, os.path.basename(filename)
            )
        inputtracexml = "%s/%s_source_mediatrace.xml" % (
            metadata_dir, os.path.basename(filename)
            )
        output = "%s/%s.mkv" % (
            data_dir, os.path.splitext(os.path.basename(filename))[0]
            )
        # Generate filename of ffv1.mkv without the path.
        outputfilename = os.path.basename(output)
        outputxml = "%s/%s_mediainfo.xml" % (metadata_dir, outputfilename)
        outputtracexml = "%s/%s_mediatrace.xml" % (metadata_dir, outputfilename)
        fmd5 = "%s/%s_source.framemd5" % (
            metadata_dir, os.path.basename(filename)
            )
        fmd5ffv1 = "%s/%s_ffv1.framemd5" % (metadata_dir, outputfilename)
        log = "%s/%s_log.log" %  (log_dir, filename)
        generate_log(log, 'Input = %s' % filename)
        generate_log(log, 'Output = %s' % output)
        generate_log(
            log, 'makeffv1.py transcode to FFV1 and framemd5 generation of source started.'
            )
        ffv1_logfile = log_dir + '/%s_ffv1_transcode.log' % filename
        ffv1_env_dict = set_environment(ffv1_logfile)
        par = subprocess.check_output(
            [
                'mediainfo', '--Language=raw', '--Full',
                "--Inform=Video;%PixelAspectRatio%", filename
            ]
            ).rstrip()
        field_order = subprocess.check_output(
            [
                'mediainfo', '--Language=raw',
                '--Full', "--Inform=Video;%ScanType%", filename
            ]
            ).rstrip()
        height = subprocess.check_output(
            [
                'mediainfo', '--Language=raw',
                '--Full', "--Inform=Video;%Height%",
                filename
            ]
            ).rstrip()
        # Transcode video file writing frame md5 and output appropriately
        ffv1_command = [
            'ffmpeg',
            '-i', filename,
            '-c:v', 'ffv1',        # Use FFv1 codec
            '-g', '1',              # Use intra-frame only aka ALL-I aka GOP=1
            '-level', '3',          # Use Version 3 of FFv1
            '-c:a', 'copy',         # Copy and paste audio bitsream with no transcoding
            '-map', '0',
            '-dn',
            '-report',
            '-slicecrc', '1',
            '-slices', '16',
            ]
        # check for FCP7 lack of description and PAL
        if par == '1.000':
            if field_order == '':
                if height == '576':
                    ffv1_command += [
                        '-vf',
                        'setfield=tff, setdar=4/3'
                        ]
        ffv1_command += [
            output,
            '-f', 'framemd5', '-an',  # Create decoded md5 checksums for every frame of the input. -an ignores audio
            fmd5
            ]
        print ffv1_command
        subprocess.call(ffv1_command, env=ffv1_env_dict)
        generate_log(
            log, 'makeffv1.py transcode to FFV1 and framemd5 generation completed.'
            )
        generate_log(
            log, 'makeffv1.py Framemd5 generation of output file started.'
            )
        fmd5_logfile = log_dir + '/%s_framemd5.log' % outputfilename
        fmd5_env_dict = set_environment(fmd5_logfile)
        fmd5_command = [
            'ffmpeg',    # Create decoded md5 checksums for every frame
            '-i', output,
            '-report',
            '-f', 'framemd5', '-an',
            fmd5ffv1
            ]
        print fmd5_command
        subprocess.call(fmd5_command, env=fmd5_env_dict)
        generate_log(
            log,
            'makeffv1.py Framemd5 generation of output file completed'
            )
        source_video_size = get_mediainfo(
            'source_video_size', "--inform=General;%FileSize%", filename
            )
        ffv1_video_size = get_mediainfo(
            'ffv1_video_size', '--inform=General;%FileSize%', output
            )
        compression_ratio = float(source_video_size) / float(ffv1_video_size)
        if os.path.basename(sys.argv[0]) == 'makeffv1.py':
            shutil.copy(sys.argv[0], log_dir)
        print 'Generating mediainfo xml of input file and saving it in %s' % inputxml
        make_mediainfo(inputxml, 'mediaxmlinput', filename)
        print 'Generating mediainfo xml of output file and saving it in %s' % outputxml
        make_mediainfo(outputxml, 'mediaxmloutput', output)
        print 'Generating mediatrace xml of input file and saving it in %s' % inputtracexml
        make_mediatrace(inputtracexml, 'mediatracexmlinput', filename)
        print 'Generating mediatrace xml of output file and saving it in %s' % outputtracexml
        make_mediatrace(outputtracexml, 'mediatracexmloutput', output)
        source_parent_dir = os.path.dirname(os.path.abspath(filename))
        manifest = '%s/%s_manifest.md5' % (source_parent_dir, filenoext)
        generate_log(log, 'makeffv1.py MD5 manifest started')
        checksum_mismatches = []
        with open(fmd5) as f1:
            with open(fmd5ffv1) as f2:
                for (lineno1, line1), (lineno2, line2) in itertools.izip(
                        read_non_comment_lines(f1),
                        read_non_comment_lines(f2)
                        ):
                    if line1 != line2:
                        if 'sar' in line1:
                            checksum_mismatches = ['sar']
                        else:
                            checksum_mismatches.append(1)
        if len(checksum_mismatches) == 0:
            print 'LOSSLESS'
            append_csv(
                csv_report_filename, (
                    output,
                    'LOSSLESS', source_video_size,
                    ffv1_video_size, compression_ratio
                    )
                )
            generate_log(log, 'makeffv1.py Transcode was lossless')
        elif len(checksum_mismatches) == 1:
            if checksum_mismatches[0] == 'sar':
                print 'Image content is lossless,'
                ' Pixel Aspect Ratio has been altered.'
                ' Update ffmpeg in order to resolve the PAR issue.'
                append_csv(
                    csv_report_filename,
                    (
                        output,
                        'LOSSLESS - different PAR',
                        source_video_size, ffv1_video_size, compression_ratio
                        )
                    )
                generate_log(
                    log,
                    'makeffv1.py Image content is lossless but Pixel Aspect Ratio has been altered.Update ffmpeg in order to resolve the PAR issue.'
                    )
        elif len(checksum_mismatches) > 1:
            print 'NOT LOSSLESS'
            append_csv(
                csv_report_filename,
                (
                    output, 'NOT LOSSLESS',
                    source_video_size, ffv1_video_size, compression_ratio
                    )
                )
            generate_log(log, 'makeffv1.py Not Lossless.')
        hashlib_manifest(filenoext, manifest, source_parent_dir)
        if filecmp.cmp(fmd5, fmd5ffv1, shallow=False):
            print "YOUR FILES ARE LOSSLESS YOU SHOULD BE SO HAPPY!!!"
        else:
            print "The framemd5 text files are not completely identical."
            " This may be because of a lossy transcode,"
            " or a change in metadata, most likely pixel aspect ratio."
            " Please analyse the framemd5 files for source and output."
Ejemplo n.º 16
0
    user = '******'
    print 'Hi Brian, Congratulations on becoming a father!!!'
elif user == '2':
    user = '******'
    print 'Hi Gavin, Have you renewed your subscription to American Cinematographer?'
elif user == '3':
    user = '******'
    print 'Hi Raelene, Brian must be out of the office'
    time.sleep(1)

create_csv(csv_report_filename, ('Sequence Name', 'Lossless?', 'Start time', 'Finish Time'))
for source_directory in all_files:
    for root,dirnames,filenames in os.walk(source_directory):
            source_directory = root
            if not file_check(source_directory) == 'TIFF':
                append_csv(csv_report_filename, (source_directory,'EMPTY DIRECTORY - SKIPPED', 'n/a', 'n/a'))
                continue

            root_dir = os.path.dirname(os.path.dirname(root))
            general_log = root_dir + '/logs/image/%s_image_log.log' % os.path.basename(root_dir)
            generate_log(general_log, 'Input = %s' % root)
            remove_bad_files(source_directory)
            source_parent_dir           = os.path.dirname(source_directory)
            normpath                    = os.path.normpath(source_directory)
            relative_path               = normpath.split(os.sep)[-1]
            split_path                  = os.path.split(os.path.basename(source_directory))[1]
            start                       = datetime.datetime.now()
            source_manifest             = root_dir + '/%s_manifest.md5' % relative_path
            generate_log(general_log, 'Generating source manifest via md5deep and storing as  %s' % source_manifest)
            make_manifest(root_dir, root_dir, source_manifest)
            info                        = make_framemd5(source_directory, 'tiff', 'tiff_framemd5')
Ejemplo n.º 17
0
                      os.path.basename(source_directory), source_manifest)
        info = make_framemd5(source_directory, 'tiff', 'tiff_framemd5')
        output_dirname = info[0]
        source_textfile = info[1]
        image_seq_without_container = info[2]
        tiff_filename = image_seq_without_container + "%06d.tiff"
        dpx_filename = image_seq_without_container + "%06d.dpx"

        logfile = output_dirname + '/image/logs/%sdpx_transcode.log' % image_seq_without_container
        env_dict = set_environment(logfile)
        tiff2dpx = [
            'ffmpegnometadata', '-report', '-f', 'image2', '-framerate', '24',
            '-i', tiff_filename, output_dirname + '/image/dpx_files'
            '/' + dpx_filename
        ]
        print tiff2dpx

        subprocess.call(tiff2dpx, env=env_dict)
        parent_basename = os.path.basename(output_dirname)
        manifest_textfile = os.path.dirname(
            output_dirname) + '/' + parent_basename + '_manifest.md5'
        other = make_framemd5(output_dirname + '/image/dpx_files', 'dpx',
                              'dpx_framemd5')
        other_textfile = other[1]
        judgement = diff_textfiles(source_textfile, other_textfile)
        make_manifest(output_parent_directory,
                      os.path.basename(output_dirname), manifest_textfile)
        finish = datetime.datetime.now()
        append_csv(csv_report_filename,
                   (parent_basename, judgement, start, finish))
#send_gmail(emails, csv_report_filename, 'makedpx completed', 'Hi,\n Please the attached log for details of the makedpx job, \nSincerely yours,\nIFIROBOT', config[2].rstrip(), config[3].rstrip())
Ejemplo n.º 18
0
def representation_uuid_csv(filmographic, source_accession, uuid):
    uuid_csv = os.path.expanduser('~/Desktop/uuid.csv')
    print os.path.abspath(uuid_csv)
    if not os.path.isfile(uuid_csv):
        create_csv(uuid_csv, ('reference number','source accession number' 'uuid'))
    append_csv(uuid_csv, (filmographic, source_accession, uuid) )
Ejemplo n.º 19
0
    with open(source_textfile) as f1:
        with open(ffv1_md5) as f2:
            for (lineno1,
                 line1), (lineno2,
                          line2) in itertools.izip(read_non_comment_lines(f1),
                                                   read_non_comment_lines(f2)):
                if line1 != line2:
                    if 'sar' in line1:
                        checksum_mismatches = ['sar']
                    else:
                        checksum_mismatches.append(1)
    if len(checksum_mismatches) == 0:
        print 'LOSSLESS'
        append_csv(csv_report_filename,
                   (parent_basename, judgement, start, finish, transcode_start,
                    transcode_finish, transcode_time, sequence_length, fps,
                    total_size, ffv1_size, pix_fmt, container, width, height,
                    comp_ratio))

    elif len(checksum_mismatches) == 1:
        if checksum_mismatches[0] == 'sar':
            print 'Image content is lossless, Pixel Aspect Ratio has been altered - This is mostly likely because your source had no pixel aspect ratio information, and Matroska is specifying 1:1. https://www.ietf.org/mail-archive/web/cellar/current/msg00739.html '
            append_csv(
                csv_report_filename,
                (parent_basename, 'LOSSLESS - different PAR', start, finish,
                 transcode_start, transcode_finish, transcode_time,
                 sequence_length, fps, total_size, ffv1_size, pix_fmt,
                 container, width, height, comp_ratio))
    elif len(checksum_mismatches) > 1:
        print 'NOT LOSSLESS'
        print csv_report_filename
Ejemplo n.º 20
0
def main():
    desktop_logdir = os.path.expanduser("~/Desktop/") + 'seq_csv_reports'
    if not os.path.isdir(desktop_logdir):
        os.makedirs(desktop_logdir)
    all_files = sys.argv[1:]
    permission = ''
    if not permission == 'y' or permission == 'Y':
        print '\n\n**** All image sequences within these directories will be converted the input for this script.\n'
        for i in all_files:
            print i
        permission = raw_input(
            '\n**** All image sequences within these directories will be converted the input for this script \n**** If this looks ok, please press Y, otherwise, type N\n'
        )
        while permission not in ('Y', 'y', 'N', 'n'):
            permission = raw_input(
                '\n**** All image sequences within these directories will be converted the input for this script \n**** If this looks ok, please press Y, otherwise, type N\n'
            )
        if permission == 'n' or permission == 'N':
            print 'Exiting at your command- Cheerio for now'
            sys.exit()
        elif permission == 'y' or permission == 'Y':
            print 'Ok so!'
    csv_report_filename = desktop_logdir + '/seq2prores_report' + time.strftime(
        "_%Y_%m_%dT%H_%M_%S") + '.csv'
    user = get_user()
    create_csv(csv_report_filename,
               ('Sequence Name', 'Start time', 'Finish Time'))
    for source_directory in all_files:
        for root, dirnames, filenames in os.walk(source_directory):
            #if "tiff_scans"  in dirnames:
            source_directory = root  # + '/tiff_scans'
            total_size = 0
            remove_bad_files(source_directory)
            source_parent_dir = os.path.dirname(source_directory)
            normpath = os.path.normpath(source_directory)
            relative_path = normpath.split(os.sep)[-1]
            split_path = os.path.split(os.path.basename(source_directory))[1]
            start = datetime.datetime.now()

            info = get_filenames(source_directory, 'dpx_framemd5')
            if info == 'none':
                continue
            for files in filenames:
                total_size += os.path.getsize(os.path.join(root, files))
            master_parent_dir = os.path.dirname(source_parent_dir)
            master_object_dir = master_parent_dir + '/objects/image'
            master_metadata_dir = master_parent_dir + '/' + 'metadata'
            middle = os.listdir(
                os.path.dirname(os.path.dirname(master_parent_dir)) +
                '/mezzanine')[0]
            mezzanine_object_dir = os.path.dirname(
                os.path.dirname(
                    master_parent_dir)) + '/mezzanine/%s/objects' % middle
            mezzanine_parent_dir = os.path.dirname(
                os.path.dirname(master_parent_dir)) + '/mezzanine/%s' % middle
            mezzanine_metadata_dir = mezzanine_parent_dir + '/metadata'
            source_manifest = master_parent_dir + '/' + os.path.basename(
                master_parent_dir) + '_manifest.md5'
            mezzanine_manifest = mezzanine_parent_dir + '/' + os.path.basename(
                mezzanine_parent_dir) + '_manifest.md5'
            audio_dir_list = os.listdir(master_parent_dir + '/objects/audio')
            for audio_files in audio_dir_list:
                if not audio_files[0] == '.':
                    if audio_files.endswith('.wav'):
                        master_audio = master_parent_dir + '/objects/audio/' + audio_files
                        audio_date_modified = get_date_modified(master_audio)
            mezzanine_file = mezzanine_object_dir + '/' + os.path.basename(
                mezzanine_parent_dir) + '_mezzanine.mov'
            if os.path.isfile(mezzanine_file):
                print 'Mezzanine file already exists so this script has most likely already been run.. skipping.'
                continue
            image_seq_without_container = info[0]
            start_number = info[1]
            container = info[2]
            image_date_modified = info[3]
            start_number_length = len(start_number)
            number_regex = "%0" + str(start_number_length) + 'd.'
            audio_dir = source_parent_dir + '/audio'
            logs_dir = mezzanine_parent_dir + '/logs'
            intellectual_entity_uuid = str(uuid.uuid4())
            source_representation_uuid = premis_description(
                master_object_dir, master_parent_dir + '/objects/audio', user,
                image_date_modified, audio_date_modified,
                intellectual_entity_uuid)

            os.chdir(audio_dir)
            audio_file_list = glob('*.wav')
            audio_file = os.path.join(audio_dir, audio_file_list[0])
            dpx_filename = image_seq_without_container + number_regex + container
            logfile = logs_dir + '/%s_prores.log' % os.path.basename(
                mezzanine_parent_dir)
            env_dict = os.environ.copy()
            # https://github.com/imdn/scripts/blob/0dd89a002d38d1ff6c938d6f70764e6dd8815fdd/ffmpy.py#L272
            logfile = "\'" + logfile + "\'"
            env_dict['FFREPORT'] = 'file={}:level=48'.format(logfile)
            seq2prores = [
                'ffmpeg', '-y', '-f', 'image2', '-framerate', '24',
                '-start_number', start_number, '-i', root + '/' + dpx_filename,
                '-i', audio_file, '-c:v', 'prores', '-profile:v', '3', '-c:a',
                'pcm_s24le', '-ar', '48000', mezzanine_object_dir + '/' +
                os.path.basename(mezzanine_parent_dir) + '_mezzanine.mov',
                '-f', 'framemd5', '-an', master_metadata_dir + '/image/' +
                os.path.basename(master_parent_dir) + '.framemd5', '-c:a',
                'pcm_s24le', '-f', 'framemd5', '-vn', master_metadata_dir +
                '/audio/' + os.path.basename(master_parent_dir) + '.framemd5'
            ]
            print seq2prores
            subprocess.call(seq2prores, env=env_dict)
            representation_uuid = str(uuid.uuid4())
            split_list = os.path.basename(mezzanine_parent_dir).split('_')
            premisxml, premis_namespace, doc, premis = setup_xml(
                mezzanine_file)
            items = {
                "workflow": "seq2prores",
                "oe": 'n/a',
                "filmographic": split_list[0],
                "sourceAccession": split_list[1],
                "interventions": ['placeholder'],
                "prepList": ['placeholder'],
                "user": user
            }
            premis = doc.getroot()
            xml_info = make_premis(mezzanine_file, items, premis,
                                   premis_namespace, premisxml,
                                   representation_uuid, '????')
            sequence = xml_info[3]

            linking_representation_uuids = []
            linking_representation_uuids.append(xml_info[2])
            linking_representation_uuids.append(
                xml_info[2]
            )  # the duplicate does nothing btw, they are a placeholder from a hardcoded function
            linking_representation_uuids.append(source_representation_uuid)
            create_intellectual_entity(premisxml, premis_namespace, doc,
                                       premis, items, intellectual_entity_uuid)
            create_representation(premisxml, premis_namespace, doc, premis,
                                  items, linking_representation_uuids,
                                  representation_uuid, sequence,
                                  intellectual_entity_uuid)
            doc = xml_info[0]
            premisxml = xml_info[1]
            final_sip_manifest_uuid = str(uuid.uuid4())
            prores_event_uuid = str(uuid.uuid4())

            macMiniTelecineMachineAgent_events = [
                prores_event_uuid, final_sip_manifest_uuid
            ]
            macMiniTelecineMachineAgent = make_agent(
                premis, macMiniTelecineMachineAgent_events,
                '230d72da-07e7-4a79-96ca-998b9f7a3e41')
            macMiniTelecineMachineOSAgent_events = [
                prores_event_uuid, final_sip_manifest_uuid
            ]
            macMiniTelecineOSAgent = make_agent(
                premis, macMiniTelecineMachineOSAgent_events,
                '9486b779-907c-4cc4-802c-22e07dc1242f')

            hashlib_events = [final_sip_manifest_uuid]
            hashlibAgent = make_agent(premis, hashlib_events,
                                      '9430725d-7523-4071-9063-e8a6ac4f84c4')
            ffmpegAgent_events = [prores_event_uuid]
            ffmpegAgent = make_agent(premis, ffmpegAgent_events,
                                     'ee83e19e-cdb1-4d83-91fb-7faf7eff738e')
            operatorEvents = [final_sip_manifest_uuid, prores_event_uuid]
            operatorAgent = make_agent(premis, operatorEvents, user)
            #ffmpegAgent                                 = make_agent(premis,[framemd5_uuid ], 'ee83e19e-cdb1-4d83-91fb-7faf7eff738e')
            make_event(
                premis, 'creation',
                'Image Sequence and WAV re-encoded to Apple Pro Res 422 HQ with 48khz 24-bit PCM audio',
                [
                    macMiniTelecineMachineAgent, macMiniTelecineOSAgent,
                    ffmpegAgent, operatorAgent
                ], prores_event_uuid, [representation_uuid], 'outcome', 'now')

            print premisxml
            mezzanine_mediainfoxml = "%s/%s_mediainfo.xml" % (
                mezzanine_metadata_dir, os.path.basename(mezzanine_parent_dir))
            tracexml = "%s/%s_mediatrace.xml" % (
                mezzanine_metadata_dir, os.path.basename(mezzanine_parent_dir))
            audio_mediainfoxml = "%s/%s_mediainfo.xml" % (
                master_metadata_dir + '/audio', os.path.basename(master_audio))
            audio_mediatracexml = "%s/%s_mediatrace.xml" % (
                master_metadata_dir + '/audio', os.path.basename(master_audio))
            if not os.path.isfile(audio_mediainfoxml):
                make_mediainfo(audio_mediainfoxml, 'audiomediaxmlinput',
                               master_audio)
            if not os.path.isfile(audio_mediatracexml):
                make_mediainfo(audio_mediatracexml, 'audiomediatraceinput',
                               master_audio)
            if not os.path.isfile(mezzanine_mediainfoxml):
                make_mediainfo(
                    mezzanine_mediainfoxml, 'mediaxmlinput',
                    mezzanine_object_dir + '/' +
                    os.path.basename(mezzanine_parent_dir) + '_mezzanine.mov')
            if not os.path.isfile(tracexml):
                make_mediatrace(
                    tracexml, 'mediatracexmlinput',
                    mezzanine_object_dir + '/' +
                    os.path.basename(mezzanine_parent_dir) + '_mezzanine.mov')
            hashlib_manifest(master_parent_dir, source_manifest,
                             master_parent_dir)
            hashlib_manifest(mezzanine_parent_dir, mezzanine_manifest,
                             mezzanine_parent_dir)
            make_event(premis, 'message digest calculation',
                       'Checksum manifest for whole package created', [
                           macMiniTelecineMachineAgent, macMiniTelecineOSAgent,
                           operatorAgent
                       ], final_sip_manifest_uuid, [representation_uuid],
                       'source', 'now')
            write_premis(doc, premisxml)
            finish = datetime.datetime.now()
            append_csv(csv_report_filename,
                       (os.path.basename(master_parent_dir), start, finish))
            '''
Ejemplo n.º 21
0
def representation_uuid_csv(filmographic, source_accession, uuid):
    uuid_csv = os.path.expanduser('~/Desktop/uuid.csv')
    if not os.path.isfile(uuid_csv):
        create_csv(uuid_csv, ('reference number', 'source accession number'
                              'uuid'))
    append_csv(uuid_csv, (filmographic, source_accession, uuid))
Ejemplo n.º 22
0
def main(args_):
    '''
    Launch all the functions for creating an IFI SIP.
    '''
    args = parse_args(args_)
    start = datetime.datetime.now()
    inputs = args.i
    if args.d:
        try:
            import clairmeta
            clairmeta_version = clairmeta.__version__
        except ImportError:
            print(
                'Exiting as Clairmeta is not installed. If there is a case for not using clairmeta, please let me know and i can make a workaround'
            )
            sys.exit()
    print(args)
    user = ififuncs.determine_user(args)
    object_entry = get_object_entry(args)
    sip_path = make_folder_path(os.path.join(args.o), args, object_entry)
    uuid, uuid_event = determine_uuid(args, sip_path)
    new_log_textfile = os.path.join(sip_path,
                                    'logs' + '/' + uuid + '_sip_log.log')
    if args.d:
        content_title = create_content_title_text(sip_path, args)
    ififuncs.generate_log(new_log_textfile, 'EVENT = sipcreator.py started')
    ififuncs.generate_log(
        new_log_textfile, 'eventDetail=sipcreator.py %s' %
        ififuncs.get_script_version('sipcreator.py'))
    ififuncs.generate_log(new_log_textfile,
                          'Command line arguments: %s' % args)
    ififuncs.generate_log(new_log_textfile, 'EVENT = agentName=%s' % user)
    ififuncs.generate_log(new_log_textfile, uuid_event)
    if not args.sc:
        ififuncs.generate_log(
            new_log_textfile, 'EVENT = eventType=Identifier assignement,'
            ' eventIdentifierType=object entry, value=%s' % object_entry)
    metadata_dir = os.path.join(sip_path, 'metadata')
    supplemental_dir = os.path.join(metadata_dir, 'supplemental')
    logs_dir = os.path.join(sip_path, 'logs')
    if args.accession:
        accession_number = ififuncs.get_accession_number()
        reference_number = ififuncs.get_reference_number()
        parent = ififuncs.ask_question(
            'What is the parent record? eg MV 1234. Enter n/a if this is a born digital acquisition with no parent.'
        )
        donor = ififuncs.ask_question(
            'Who is the source of acquisition, as appears on the donor agreement? This will not affect Reproductions.'
        )
        depositor_reference = ififuncs.ask_question(
            'What is the donor/depositor number? This will not affect Reproductions.'
        )
        acquisition_type = ififuncs.get_acquisition_type('')
        donation_date = ififuncs.ask_question(
            'When was the donation date in DD/MM/YYYY format? Eg. 31/12/1999 - Unfortunately this is NOT using ISO 8601.'
        )
    if args.zip:
        inputxml, inputtracexml, dfxml = ififuncs.generate_mediainfo_xmls(
            inputs[0], args.o, uuid, new_log_textfile)
        if args.manifest:
            shutil.copy(
                args.manifest,
                args.manifest.replace('_manifest.md5', '_manifest-md5.txt'))
            source_manifest = args.manifest.replace('_manifest.md5',
                                                    '_manifest-md5.txt')
        else:
            source_manifest = os.path.join(
                args.o,
                os.path.basename(args.i[0]) + '_manifest-md5.txt')
            ififuncs.generate_log(
                new_log_textfile,
                'EVENT = message digest calculation, status=started, eventType=messageDigestCalculation, agentName=hashlib, eventDetail=MD5 checksum of source files within ZIP'
            )
            ififuncs.hashlib_manifest(args.i[0], source_manifest,
                                      os.path.dirname(args.i[0]))
            ififuncs.generate_log(
                new_log_textfile,
                'EVENT = message digest calculation, status=finished, eventType=messageDigestCalculation, agentName=hashlib, eventDetail=MD5 checksum of source files within ZIP'
            )
        ififuncs.generate_log(
            new_log_textfile,
            'EVENT = packing, status=started, eventType=packing, agentName=makezip.py, eventDetail=Source object to be packed=%s'
            % inputs[0])
        makezip_judgement, zip_file = makezip.main([
            '-i', inputs[0], '-o',
            os.path.join(sip_path, 'objects'), '-basename', uuid + '.zip'
        ])
        ififuncs.generate_log(
            new_log_textfile,
            'EVENT = packing, status=finished, eventType=packing, agentName=makezip.py, eventDetail=Source object packed into=%s'
            % zip_file)
        if makezip_judgement is None:
            judgement = 'lossless'
        else:
            judgement = makezip_judgement
        ififuncs.generate_log(
            new_log_textfile,
            'EVENT = losslessness verification, status=finished, eventType=messageDigestCalculation, agentName=makezip.py, eventDetail=embedded crc32 checksum validation, eventOutcome=%s'
            % judgement)
        ififuncs.generate_log(
            new_log_textfile,
            'EVENT = losslessness verification, status=finished, eventType=messageDigestCalculation, agentName=makezip.py, eventDetail=embedded crc32 checksum validation, eventOutcome=%s'
            % judgement)
    else:
        log_names = move_files(inputs, sip_path, args, user)
    ififuncs.get_technical_metadata(sip_path, new_log_textfile)
    ififuncs.hashlib_manifest(metadata_dir,
                              metadata_dir + '/metadata_manifest.md5',
                              metadata_dir)
    if args.sc:
        normalise_objects_manifest(sip_path)
    new_manifest_textfile = consolidate_manifests(sip_path, 'objects',
                                                  new_log_textfile)

    if args.zip:
        ififuncs.generate_log(
            new_log_textfile,
            'EVENT = Message Digest Calculation, status=started, eventType=message digest calculation, eventDetail=%s module=hashlib'
            % zip_file)
        ififuncs.manifest_update(new_manifest_textfile, zip_file)
        ififuncs.generate_log(
            new_log_textfile,
            'EVENT = Message Digest Calculation, status=finished, eventType=message digest calculation, eventDetail=%s module=hashlib'
            % zip_file)
    consolidate_manifests(sip_path, 'metadata', new_log_textfile)
    ififuncs.hashlib_append(logs_dir, new_manifest_textfile,
                            os.path.dirname(os.path.dirname(logs_dir)))
    if args.supplement:
        os.makedirs(supplemental_dir)
        supplement_cmd = [
            '-i', args.supplement, '-user', user, '-new_folder',
            supplemental_dir,
            os.path.dirname(sip_path), '-copy'
        ]
        package_update.main(supplement_cmd)
    if args.zip:
        os.makedirs(supplemental_dir)
        supplement_cmd = [
            '-i', [inputxml, inputtracexml, dfxml, source_manifest], '-user',
            user, '-new_folder', supplemental_dir,
            os.path.dirname(sip_path), '-copy'
        ]
        package_update.main(supplement_cmd)
    if args.sc:
        print('Generating Digital Forensics XML')
        dfxml = accession.make_dfxml(args, sip_path, uuid)
        ififuncs.generate_log(
            new_log_textfile,
            'EVENT = Metadata extraction - eventDetail=File system metadata extraction using Digital Forensics XML, eventOutcome=%s, agentName=makedfxml'
            % (dfxml))
        ififuncs.manifest_update(new_manifest_textfile, dfxml)
        sha512_log = manifest.main([sip_path, '-sha512', '-s'])
        sha512_manifest = os.path.join(os.path.dirname(sip_path),
                                       uuid + '_manifest-sha512.txt')
        ififuncs.merge_logs_append(sha512_log, new_log_textfile,
                                   new_manifest_textfile)
        ififuncs.checksum_replace(sha512_manifest, new_log_textfile, 'sha512')
        os.remove(sha512_log)
    ififuncs.sort_manifest(new_manifest_textfile)
    if not args.quiet:
        if 'log_names' in locals():
            log_report(log_names)
    finish = datetime.datetime.now()
    print('\n- %s ran this script at %s and it finished at %s' %
          (user, start, finish))
    if args.d:
        process_dcp(sip_path, content_title, args, new_manifest_textfile,
                    new_log_textfile, metadata_dir, clairmeta_version)
    if args.accession:
        register = accession.make_register()
        filmographic_dict = ififuncs.extract_metadata(args.csv)[0]
        for filmographic_record in filmographic_dict:
            if filmographic_record['Reference Number'].lower(
            ) == reference_number.lower():
                if filmographic_record['Title'] == '':
                    title = filmographic_record[
                        'TitleSeries'] + '; ' + filmographic_record['EpisodeNo']
                else:
                    title = filmographic_record['Title']
        oe_register = make_oe_register()
        ififuncs.append_csv(
            oe_register,
            (object_entry.upper()[:2] + '-' + object_entry[2:], donation_date,
             '1', '', title, donor, acquisition_type[1], accession_number,
             'Representation of %s|Reproduction of %s' %
             (reference_number, parent), ''))
        accession_cmd = [
            os.path.dirname(sip_path), '-user', user, '-f', '-number',
            accession_number, '-reference', reference_number, '-register',
            register, '-csv', args.csv, '-pbcore'
        ]
        if not parent.lower() == 'n/a':
            accession_cmd.extend(['-parent', parent])
        accession_cmd.extend(['-donor', donor])
        accession_cmd.extend(['-depositor_reference', depositor_reference])
        accession_cmd.extend(['-acquisition_type', acquisition_type[2]])
        accession_cmd.extend(['-donation_date', donation_date])
        print(accession_cmd)
        accession.main(accession_cmd)
    return new_log_textfile, new_manifest_textfile
Ejemplo n.º 23
0
            total_size += os.path.getsize(os.path.join(root,files))
        output_dirname              = info[0]  
        source_textfile             = info[1]
        image_seq_without_container = info[2]
        start_number                = info[3]
        container                   = info[4]
        dpx_filename                = image_seq_without_container + "%06d." + container

        logfile = output_dirname + '/logs/%s_dpx_transcode.log' % os.path.basename(root)
        env_dict = set_environment(logfile)
        ffv12dpx = ['ffmpeg','-report','-f','image2','-framerate','24', '-start_number', start_number, '-i', os.path.abspath(dpx_filename) ,'-strict', '-2','-c:v','ffv1','-level', '3', '-pix_fmt', 'rgb48le',output_dirname +  '/video/' + os.path.basename(root) + '.mkv']
        print ffv12dpx

        subprocess.call(ffv12dpx,env=env_dict)
        parent_basename   =  os.path.basename(output_dirname)
        manifest_textfile = os.path.dirname(output_dirname) + '/' +  parent_basename + '_manifest.md5'
        ffv1_path         = output_dirname +  '/video/'  + os.path.basename(root) + '.mkv'
        ffv1_md5          = output_dirname +  '/md5/' + os.path.basename(root) + 'ffv1.framemd5'
        subprocess.call(['ffmpeg','-i', ffv1_path, '-pix_fmt', 'rgb48le','-f', 'framemd5', ffv1_md5])
        #other_textfile = other[1]
        judgement = diff_textfiles(source_textfile, ffv1_md5)
        make_manifest(output_parent_directory, os.path.basename(output_dirname), manifest_textfile)
        finish = datetime.datetime.now()
        
        print total_size
        ffv1_size = os.path.getsize(ffv1_path)
        comp_ratio =  float(total_size) / float(os.path.getsize(ffv1_path))
        append_csv(csv_report_filename, (parent_basename,judgement, start, finish,total_size, ffv1_size, comp_ratio))
        
#send_gmail(emails, csv_report_filename, 'makedpx completed', 'Hi,\n Please the attached log for details of the makedpx job, \nSincerely yours,\nIFIROBOT', config[2].rstrip(), config[3].rstrip())