Exemplo n.º 1
0
def main(args_):
    '''
    Launch all the functions for determining folder and file paths.
    '''
    args = parse_args(args_)
    source = args.input
    sip_path = ififuncs.check_for_sip([source])
    if sip_path is not None:
        package_info = analyse_package(sip_path)
    if package_info is False:
        print('Valid UUID not found in folder path')
    else:
        print package_info
    error_list, question_list = test_package(package_info)
    return package_info, error_list, question_list
Exemplo n.º 2
0
def main(args_):
    '''
    Launch all the functions for creating an IFI SIP.
    '''
    args = parse_args(args_)
    source = args.input
    sip_path = ififuncs.check_for_sip([source])
    if sip_path is not None:
        oe_path = os.path.dirname(sip_path)
        uuid = os.path.basename(sip_path)
        sip_manifest = os.path.join(oe_path, uuid) + '_manifest.md5'
    start = datetime.datetime.now()
    print args
    if args.user:
        user = args.user
    else:
        user = ififuncs.get_user()
    new_log_textfile = os.path.join(sip_path,
                                    'logs' + '/' + uuid + '_sip_log.log')
    ififuncs.generate_log(new_log_textfile, 'EVENT = rearrange.py started')
    ififuncs.generate_log(
        new_log_textfile, 'eventDetail=rearrange.py %s' %
        ififuncs.get_script_version('rearrange.py'))
    ififuncs.generate_log(new_log_textfile,
                          'Command line arguments: %s' % args)
    ififuncs.generate_log(new_log_textfile, 'EVENT = agentName=%s' % user)
    if not os.path.isdir(args.new_folder):
        os.makedirs(args.new_folder)
    for filename in args.i:
        # add test to see if it actually deleted - what if read only?
        shutil.move(filename, args.new_folder)
        print '%s has been moved into %s' % (filename, args.new_folder)
        ififuncs.generate_log(
            new_log_textfile, 'EVENT = eventType=file movement,'
            ' eventOutcomeDetailNote=%s has been moved into %s'
            ' agentName=shutil.move()' % (filename, args.new_folder))
        relative_filename = filename.replace(args.input + '/', '')
        relative_new_folder = args.new_folder.replace(args.input + '/', '')
        update_manifest(
            sip_manifest, relative_filename,
            os.path.join(relative_new_folder,
                         os.path.basename(relative_filename)),
            new_log_textfile)
    ififuncs.generate_log(new_log_textfile, 'EVENT = rearrange.py finished')
    ififuncs.checksum_replace(sip_manifest, new_log_textfile, 'md5')
    finish = datetime.datetime.now()
    print '\n', user, 'ran this script at %s and it finished at %s' % (start,
                                                                       finish)
Exemplo n.º 3
0
def main(args_):
    '''
    Launch all the functions for creating an IFI SIP.
    '''
    args = parse_args(args_)
    source = args.uuid_path
    sip_path = ififuncs.check_for_sip([source])
    if sip_path is not None:
        oe_path = os.path.dirname(sip_path)
        uuid = os.path.basename(sip_path)
        sip_manifest = os.path.join(oe_path, uuid) + '_manifest.md5'
    else:
        # this is assuming that the other workflow will be the
        # special collections workflow that has the uuid as the parent.
        # some real checks should exist for this whole if/else flow.
        sip_path = args.uuid_path
        oe_path = os.path.dirname(args.uuid_path)
        uuid = os.path.basename(sip_path)
        sip_manifest = os.path.join(oe_path, uuid + '_manifest.md5')
    start = datetime.datetime.now()
    print(args)
    if args.user:
        user = args.user
    else:
        user = ififuncs.get_user()
    new_log_textfile = os.path.join(sip_path,
                                    'logs' + '/' + uuid + '_sip_log.log')
    ififuncs.generate_log(new_log_textfile, 'EVENT = deletefiles.py started')
    ififuncs.generate_log(
        new_log_textfile, 'eventDetail=deletefiles.py %s' %
        ififuncs.get_script_version('deletefiles.py'))
    ififuncs.generate_log(new_log_textfile,
                          'Command line arguments: %s' % args)
    ififuncs.generate_log(new_log_textfile, 'EVENT = agentName=%s' % user)
    metadata_dir = os.path.join(sip_path, 'metadata')
    for filename in args.i:
        # add test to see if it actually deleted - what if read only?
        os.remove(filename)
        print(('%s has been deleted' % filename))
        ififuncs.generate_log(
            new_log_textfile, 'EVENT = eventType=deletion,'
            ' eventOutcomeDetailNote=%s has been deleted,'
            ' agentName=os.remove()' % filename)
        for metadata in os.listdir(metadata_dir):
            if os.path.basename(filename) in metadata:
                os.remove(os.path.join(metadata_dir, metadata))
                print(('%s has been deleted' %
                       os.path.join(metadata_dir, metadata)))
                ififuncs.generate_log(
                    new_log_textfile, 'EVENT = eventType=deletion,'
                    ' eventOutcomeDetailNote=%s has been deleted,'
                    ' agentName=os.remove()' %
                    os.path.join(metadata_dir, metadata))
        remove_from_manifest(sip_manifest, os.path.basename(filename),
                             new_log_textfile)
        ififuncs.sort_manifest(sip_manifest)
    ififuncs.generate_log(new_log_textfile, 'EVENT = deletefiles.py finished')
    ififuncs.checksum_replace(sip_manifest, new_log_textfile, 'md5')
    finish = datetime.datetime.now()
    print('\n- %s ran this script at %s and it finished at %s' %
          (user, start, finish))
Exemplo n.º 4
0
def main(args_):
    '''
    Launches the various functions that will accession a package
    '''
    args = parse_args(args_)
    source = args.input
    uuid_directory = ififuncs.check_for_sip([source])
    if uuid_directory is not None:
        oe_path = os.path.dirname(uuid_directory)
        oe_number = os.path.basename(oe_path)
        if args.user:
            user = args.user
        else:
            user = ififuncs.get_user()
        if args.number:
            if args.number[:3] != 'aaa':
                print 'First three characters must be \'aaa\' and last four characters must be four digits'
                accession_number = ififuncs.get_accession_number()
            elif len(args.number[3:]) != 4:
                accession_number = ififuncs.get_accession_number()
                print 'First three characters must be \'aaa\' and last four characters must be four digits'
            elif not args.number[3:].isdigit():
                accession_number = ififuncs.get_accession_number()
                print 'First three characters must be \'aaa\' and last four characters must be four digits'
            else:
                accession_number = args.number
        else:
            accession_number = ififuncs.get_accession_number()
        if args.pbcore:
            if args.reference:
                Reference_Number = args.reference.upper()
            else:
                Reference_Number = ififuncs.get_reference_number()
        if args.acquisition_type:
            acquisition_type = ififuncs.get_acquisition_type(
                args.acquisition_type)
            print acquisition_type
        accession_path = os.path.join(os.path.dirname(oe_path),
                                      accession_number)
        uuid = os.path.basename(uuid_directory)
        new_uuid_path = os.path.join(accession_path, uuid)
        logs_dir = os.path.join(new_uuid_path, 'logs')
        sipcreator_log = os.path.join(logs_dir, uuid) + '_sip_log.log'
        if args.force:
            proceed = 'Y'
        else:
            proceed = ififuncs.ask_yes_no('Do you want to rename %s with %s' %
                                          (oe_number, accession_number))
        if proceed == 'Y':
            os.rename(oe_path, accession_path)
        if args.register:
            register = args.register
        else:
            register = make_register()
        ififuncs.append_csv(register,
                            (oe_number.upper()[:2] + '-' + oe_number[2:],
                             accession_number, '', '', '', '', '', ''))
        ififuncs.generate_log(sipcreator_log, 'EVENT = accession.py started')
        ififuncs.generate_log(
            sipcreator_log, 'eventDetail=accession.py %s' %
            ififuncs.get_script_version('accession.py'))
        ififuncs.generate_log(sipcreator_log,
                              'Command line arguments: %s' % args)
        ififuncs.generate_log(sipcreator_log, 'EVENT = agentName=%s' % user)
        ififuncs.generate_log(
            sipcreator_log, 'EVENT = eventType=Identifier assignment,'
            ' eventIdentifierType=accession number, value=%s' %
            accession_number)
        ififuncs.generate_log(
            sipcreator_log, 'EVENT = eventType=accession,'
            ' eventIdentifierType=accession number, value=%s' %
            accession_number)
        sip_manifest = os.path.join(accession_path, uuid) + '_manifest.md5'
        sha512_log = manifest.main([new_uuid_path, '-sha512', '-s'])
        sha512_manifest = os.path.join(os.path.dirname(new_uuid_path),
                                       uuid + '_manifest-sha512.txt')
        ififuncs.merge_logs_append(sha512_log, sipcreator_log, sip_manifest)
        os.remove(sha512_log)
        print('Generating Digital Forensics XML')
        dfxml = make_dfxml(args, new_uuid_path, uuid)
        ififuncs.generate_log(
            sipcreator_log,
            'EVENT = Metadata extraction - eventDetail=File system metadata extraction using Digital Forensics XML, eventOutcome=%s, agentName=makedfxml'
            % (dfxml))
        # this is inefficient. The script should not have to ask for reference
        # number twice if someone wants to insert the filmographic but do not
        # want to make the pbcore csv, perhaps because the latter already exists.
        if args.csv:
            metadata_dir = os.path.join(new_uuid_path, 'metadata')
            package_filmographic = os.path.join(
                metadata_dir, Reference_Number + '_filmographic.csv')
            insert_filmographic(args.csv, Reference_Number,
                                package_filmographic)
            ififuncs.generate_log(
                sipcreator_log,
                'EVENT = Metadata extraction - eventDetail=Filmographic descriptive metadata added to metadata folder, eventOutcome=%s, agentName=accession.py'
                % (package_filmographic))
            ififuncs.manifest_update(sip_manifest, package_filmographic)
            ififuncs.sha512_update(sha512_manifest, package_filmographic)
            print('Filmographic descriptive metadata added to metadata folder')
        ififuncs.generate_log(sipcreator_log, 'EVENT = accession.py finished')
        ififuncs.checksum_replace(sip_manifest, sipcreator_log, 'md5')
        ififuncs.checksum_replace(sha512_manifest, sipcreator_log, 'sha512')
        ififuncs.manifest_update(sip_manifest, dfxml)
        ififuncs.sha512_update(sha512_manifest, dfxml)
        if args.pbcore:
            makepbcore_cmd = [
                accession_path, '-p', '-user', user, '-reference',
                Reference_Number
            ]
            if args.parent:
                makepbcore_cmd.extend(['-parent', args.parent])
            if args.acquisition_type:
                makepbcore_cmd.extend(
                    ['-acquisition_type', args.acquisition_type])
            if args.donor:
                makepbcore_cmd.extend(['-donor', args.donor])
            if args.donor:
                makepbcore_cmd.extend(
                    ['-depositor_reference', args.depositor_reference])
            if args.donation_date:
                makepbcore_cmd.extend(['-donation_date', args.donation_date])
            makepbcore.main(makepbcore_cmd)
    else:
        print 'not a valid package. The input should include a package that has been through Object Entry'
Exemplo n.º 5
0
def main(args_):
    '''
    Launch all the functions for creating an IFI SIP.
    '''
    args = parse_args(args_)
    source = args.input
    sip_path = ififuncs.check_for_sip([source])
    if sip_path is not None:
        oe_path = os.path.dirname(sip_path)
        uuid = os.path.basename(sip_path)
        sip_manifest = os.path.join(oe_path, uuid) + '_manifest.md5'
    else:
        # this is assuming that the other workflow will be the
        # special collections workflow that has the uuid as the parent.
        # some real checks should exist for this whole if/else flow.
        sip_path = args.input
        oe_path = os.path.dirname(args.input)
        uuid = os.path.basename(sip_path)
        sip_manifest = os.path.join(oe_path, uuid + '_manifest.md5')
    start = datetime.datetime.now()
    print(args)
    if args.user:
        user = args.user
    else:
        user = ififuncs.get_user()
    new_log_textfile = os.path.join(sip_path,
                                    'logs' + '/' + uuid + '_sip_log.log')
    ififuncs.generate_log(new_log_textfile,
                          'EVENT = package_update.py started')
    ififuncs.generate_log(
        new_log_textfile, 'eventDetail=package_update.py %s' %
        ififuncs.get_script_version('package_update.py'))
    ififuncs.generate_log(new_log_textfile,
                          'Command line arguments: %s' % args)
    ififuncs.generate_log(new_log_textfile, 'EVENT = agentName=%s' % user)
    if not os.path.isdir(args.new_folder):
        os.makedirs(args.new_folder)
    if isinstance(args.i[0], (list, )):
        args.i = args.i[0]
    for filenames in args.i:
        if args.copy:
            copyit.main([filenames, args.new_folder])
            ififuncs.generate_log(
                new_log_textfile, 'EVENT = eventType=file movement,'
                ' eventOutcomeDetailNote=%s has been moved into %s'
                ' agentName=copyit.py' % (filenames, args.new_folder))
            # this is hardcoded - pick this apart so that any folder can be added to.
            # this must be fixed in normalise.py as well.
            relative_new_path = args.new_folder.replace(sip_path, '')
            print((relative_new_path, 'relative'))
            if (relative_new_path[0] == '/') or relative_new_path[0] == '\\':
                relative_new_path = relative_new_path[1:].replace('\\', '/')
            sipcreator.consolidate_manifests(sip_path, relative_new_path,
                                             new_log_textfile)
            log_manifest = os.path.join(
                os.path.dirname(new_log_textfile),
                os.path.basename(filenames) + '_manifest.md5')
            ififuncs.manifest_update(sip_manifest, log_manifest)
            ififuncs.sort_manifest(sip_manifest)
        else:
            # add test to see if it actually deleted - what if read only?
            shutil.move(filenames, args.new_folder)
            ififuncs.generate_log(
                new_log_textfile, 'EVENT = eventType=file movement,'
                ' eventOutcomeDetailNote=%s has been moved into %s'
                ' agentName=shutil.move()' % (filenames, args.new_folder))
            print(('%s has been moved into %s' % (filenames, args.new_folder)))
            relative_filename = filenames.replace(
                os.path.dirname(args.input) + '/', '').replace('\\', '/')
            relative_filename = filenames.replace(
                os.path.dirname(args.input) + '\\', '').replace('\\', '/')
            relative_new_folder = args.new_folder.replace(
                os.path.dirname(args.input) + '/', '').replace('\\', '/')
            relative_new_folder = args.new_folder.replace(
                os.path.dirname(args.input) + '\\', '').replace('\\', '/')
            update_manifest(
                sip_manifest, relative_filename,
                os.path.join(relative_new_folder,
                             os.path.basename(relative_filename)).replace(
                                 '\\', '/'), new_log_textfile)
    ififuncs.generate_log(new_log_textfile,
                          'EVENT = package_update.py finished')
    ififuncs.checksum_replace(sip_manifest, new_log_textfile, 'md5')
    finish = datetime.datetime.now()
    print('\n- %s ran this script at %s and it finished at %s' %
          (user, start, finish))
Exemplo n.º 6
0
def main(args_):
    '''
    Launch all the functions for creating an IFI SIP.
    '''
    args = parse_args(args_)
    source = args.input
    sip_path = ififuncs.check_for_sip([source])
    if sip_path is not None:
        oe_path = os.path.dirname(sip_path)
        uuid = os.path.basename(sip_path)
        sip_manifest = os.path.join(
            oe_path, uuid
            ) + '_manifest.md5'
    start = datetime.datetime.now()
    print args
    if args.user:
        user = args.user
    else:
        user = ififuncs.get_user()
    new_log_textfile = os.path.join(sip_path, 'logs' + '/' + uuid + '_sip_log.log')
    ififuncs.generate_log(
        new_log_textfile,
        'EVENT = deletefiles.py started'
    )
    ififuncs.generate_log(
        new_log_textfile,
        'eventDetail=deletefiles.py %s' % ififuncs.get_script_version('deletefiles.py')
    )
    ififuncs.generate_log(
        new_log_textfile,
        'Command line arguments: %s' % args
    )
    ififuncs.generate_log(
        new_log_textfile,
        'EVENT = agentName=%s' % user
    )
    metadata_dir = os.path.join(sip_path, 'metadata')
    for filename in args.i:
        # add test to see if it actually deleted - what if read only?
        os.remove(filename)
        print '%s has been deleted' % filename
        ififuncs.generate_log(
            new_log_textfile,
            'EVENT = eventType=deletion,'
            ' eventOutcomeDetailNote=%s has been deleted,'
            ' agentName=os.remove()'
            % filename
        )
        for metadata in os.listdir(metadata_dir):
            if os.path.basename(filename) in metadata:
                os.remove(os.path.join(metadata_dir, metadata))
                print '%s has been deleted' %  os.path.join(metadata_dir, metadata)
                ififuncs.generate_log(
                    new_log_textfile,
                    'EVENT = eventType=deletion,'
                    ' eventOutcomeDetailNote=%s has been deleted,'
                    ' agentName=os.remove()'
                    % os.path.join(metadata_dir, metadata)
                )
        remove_from_manifest(sip_manifest, os.path.basename(filename), new_log_textfile)
    ififuncs.generate_log(
        new_log_textfile,
        'EVENT = deletefiles.py finished'
    )    
    ififuncs.checksum_replace(sip_manifest, new_log_textfile, 'md5')
    finish = datetime.datetime.now()
    print '\n', user, 'ran this script at %s and it finished at %s' % (start, finish)
Exemplo n.º 7
0
def main(args_):
    '''
    Launches the various functions that will accession a package
    '''
    args = parse_args(args_)
    input = args.input
    uuid_directory = ififuncs.check_for_sip([input])
    if uuid_directory is not None:
        oe_path = os.path.dirname(uuid_directory)
        oe_number = os.path.basename(oe_path)
        if args.user:
            user = args.user
        else:
            user = ififuncs.get_user()
        if args.number:
            if args.number[:3] != 'aaa':
                print 'First three characters must be \'aaa\' and last four characters must be four digits'
                accession_number = ififuncs.get_accession_number()
            elif len(args.number[3:]) != 4:
                accession_number = ififuncs.get_accession_number()
                print 'First three characters must be \'aaa\' and last four characters must be four digits'
            elif not args.number[3:].isdigit():
                accession_number = ififuncs.get_accession_number()
                print 'First three characters must be \'aaa\' and last four characters must be four digits'
            else:
                accession_number = args.number
        else:
            accession_number = ififuncs.get_accession_number()
        accession_path = os.path.join(
            os.path.dirname(oe_path), accession_number
        )
        uuid = os.path.basename(uuid_directory)
        new_uuid_path = os.path.join(accession_path, uuid)
        logs_dir = os.path.join(new_uuid_path, 'logs')
        sipcreator_log = os.path.join(logs_dir, uuid) + '_sip_log.log'
        if args.force:
            proceed = 'Y'
        else:
            proceed = ififuncs.ask_yes_no(
                'Do you want to rename %s with %s' % (oe_number, accession_number)
            )
        if proceed == 'Y':
            os.rename(oe_path, accession_path)
        register = make_register()
        ififuncs.append_csv(register, (oe_number.upper()[:2] + '-' + oe_number[2:6], accession_number, '','','','', ''))
        ififuncs.generate_log(
            sipcreator_log,
            'EVENT = accession.py started'
        )
        ififuncs.generate_log(
            sipcreator_log,
            'eventDetail=accession.py %s' % ififuncs.get_script_version('accession.py')
        )
        ififuncs.generate_log(
            sipcreator_log,
            'Command line arguments: %s' % args
        )
        ififuncs.generate_log(
            sipcreator_log,
            'EVENT = agentName=%s' % user
        )
        ififuncs.generate_log(
            sipcreator_log,
            'EVENT = eventType=Identifier assignment,'
            ' eventIdentifierType=accession number, value=%s'
            % accession_number
        )
        ififuncs.generate_log(
            sipcreator_log,
            'EVENT = eventType=accession,'
            ' eventIdentifierType=accession number, value=%s'
            % accession_number
        )
        sip_manifest = os.path.join(
            accession_path, uuid
            ) + '_manifest.md5'
        sha512_log = manifest.main([new_uuid_path, '-sha512', '-s'])
        sha512_manifest = os.path.join(
            os.path.dirname(new_uuid_path), uuid + '_manifest-sha512.txt'
        )
        ififuncs.merge_logs_append(sha512_log, sipcreator_log, sip_manifest)
        os.remove(sha512_log)
        dfxml = make_dfxml(args, new_uuid_path, uuid)
        ififuncs.generate_log(
            sipcreator_log,
            'EVENT = Metadata extraction - eventDetail=File system metadata extraction using Digital Forensics XML, eventOutcome=%s, agentName=makedfxml' % (dfxml)
        )
        ififuncs.generate_log(
            sipcreator_log,
            'EVENT = accession.py finished'
        )
        ififuncs.checksum_replace(sip_manifest, sipcreator_log, 'md5')
        ififuncs.checksum_replace(sha512_manifest, sipcreator_log, 'sha512')
        ififuncs.manifest_update(sip_manifest, dfxml)
        ififuncs.sha512_update(sha512_manifest, dfxml)
        if args.pbcore:
            makepbcore.main([accession_path, '-p', '-user', user])
    else:
        print 'not a valid package. The input should include a package that has been through Object Entry'