Пример #1
0
def start_transfer(transfer_name, transfer_type, accession, access_id, paths,
                   row_ids):
    """
    Start a new transfer.

    :param str transfer_name: Name of new transfer.
    :param str transfer_type: Type of new transfer. From TRANSFER_TYPE_DIRECTORIES.
    :param str accession: Accession number of new transfer.
    :param str access_id: Access system identifier for the new transfer.
    :param list paths: List of <location_uuid>:<relative_path> to be copied into the new transfer. Location UUIDs should be associated with this pipeline, and relative path should be relative to the location.
    :param list row_ids: ID of the associated TransferMetadataSet for disk image ingest.
    :returns: Dict with {'message': <message>, ['error': True, 'path': <path>]}.  Error is a boolean, present and True if there is an error.  Message describes the success or failure. Path is populated if there is no error.
    """
    if not transfer_name:
        raise ValueError('No transfer name provided.')
    if not paths:
        raise ValueError('No path provided.')

    # Create temp directory that everything will be copied into
    temp_base_dir = os.path.join(SHARED_DIRECTORY_ROOT, 'tmp')
    temp_dir = tempfile.mkdtemp(dir=temp_base_dir)
    os.chmod(temp_dir, 0o770)  # Needs to be writeable by the SS

    for i, path in enumerate(paths):
        index = i + 1  # so transfers start from 1, not 0
        # Don't suffix the first transfer component, only subsequent ones
        if index > 1:
            target = transfer_name + '_' + str(index)
        else:
            target = transfer_name
        row_id = row_ids[i]

        if helpers.file_is_an_archive(path):
            transfer_dir = temp_dir
            p = path.split(':', 1)[1]
            logger.debug('found a zip file, splitting path ' + p)
            filepath = os.path.join(temp_dir, os.path.basename(p))
        else:
            path = os.path.join(path, '.')  # Copy contents of dir but not dir
            transfer_dir = os.path.join(temp_dir, target)
            filepath = os.path.join(temp_dir, target)

        transfer_relative = transfer_dir.replace(SHARED_DIRECTORY_ROOT, '', 1)
        copy_from_transfer_sources([path], transfer_relative)
        filepath = archivematicaFunctions.unicodeToStr(filepath)
        try:
            destination = copy_to_start_transfer(
                filepath=filepath,
                type=transfer_type,
                accession=accession,
                access_id=access_id,
                transfer_metadata_set_row_uuid=row_id)
        except Exception as e:
            logger.exception('Error starting transfer {}: {}'.format(
                filepath, e))
            raise Exception('Error starting transfer {}: {}'.format(
                filepath, e))

    shutil.rmtree(temp_dir)
    return {'message': _('Copy successful.'), 'path': destination}
Пример #2
0
def unapproved_transfers(request):
    # Example: http://127.0.0.1/api/transfer/unapproved?username=mike&api_key=<API key>
    response = {}
    unapproved = []

    jobs = models.Job.objects.filter(
        (Q(jobtype="Approve standard transfer")
         | Q(jobtype="Approve zipped transfer")
         | Q(jobtype="Approve DSpace transfer")
         | Q(jobtype="Approve bagit transfer")
         | Q(jobtype="Approve zipped bagit transfer"))
        & Q(currentstep=models.Job.STATUS_AWAITING_DECISION))

    for job in jobs:
        # remove standard transfer path from directory (and last character)
        type_and_directory = job.directory.replace(
            get_modified_standard_transfer_path() + "/", "", 1)

        # remove trailing slash if not a zipped bag file
        if not helpers.file_is_an_archive(job.directory):
            type_and_directory = type_and_directory[:-1]

        transfer_watch_directory = type_and_directory.split("/")[0]
        # Get transfer type from transfer directory
        transfer_type_directories_reversed = {
            v: k
            for k, v in
            filesystem_ajax_views.TRANSFER_TYPE_DIRECTORIES.items()
        }
        transfer_type = transfer_type_directories_reversed[
            transfer_watch_directory]

        job_directory = type_and_directory.replace(
            transfer_watch_directory + "/", "", 1)

        unapproved.append({
            "type": transfer_type,
            "directory": job_directory,
            "uuid": job.sipuuid
        })

    # get list of unapproved transfers
    # return list as JSON
    response["results"] = unapproved
    response["message"] = "Fetched unapproved transfers successfully."
    return helpers.json_response(response)
Пример #3
0
def unapproved_transfers(request):
    # Example: http://127.0.0.1/api/transfer/unapproved?username=mike&api_key=<API key>
    if request.method == 'GET':
        auth_error = authenticate_request(request)

        response = {}

        if auth_error is None:
            error = None
            unapproved = []

            jobs = models.Job.objects.filter(
                (Q(jobtype="Approve standard transfer")
                 | Q(jobtype="Approve DSpace transfer")
                 | Q(jobtype="Approve bagit transfer")
                 | Q(jobtype="Approve zipped bagit transfer"))
                & Q(currentstep='Awaiting decision'))

            for job in jobs:
                # remove standard transfer path from directory (and last character)
                type_and_directory = job.directory.replace(
                    get_modified_standard_transfer_path() + '/', '', 1)

                # remove trailing slash if not a zipped bag file
                if not helpers.file_is_an_archive(job.directory):
                    type_and_directory = type_and_directory[:-1]

                transfer_watch_directory = type_and_directory.split('/')[0]
                # Get transfer type from transfer directory
                transfer_type_directories_reversed = {
                    v: k
                    for k, v in
                    filesystem_ajax_views.TRANSFER_TYPE_DIRECTORIES.items()
                }
                transfer_type = transfer_type_directories_reversed[
                    transfer_watch_directory]

                job_directory = type_and_directory.replace(
                    transfer_watch_directory + '/', '', 1)

                unapproved.append({
                    'type': transfer_type,
                    'directory': job_directory,
                    'uuid': job.sipuuid,
                })

            # get list of unapproved transfers
            # return list as JSON
            response['results'] = unapproved

            if error is not None:
                response['message'] = error
                response['error'] = True
            else:
                response[
                    'message'] = 'Fetched unapproved transfers successfully.'

                if error is not None:
                    return helpers.json_response(response, status_code=500)
                else:
                    return helpers.json_response(response)
        else:
            response['message'] = auth_error
            response['error'] = True
            return helpers.json_response(response, status_code=403)
    else:
        return django.http.HttpResponseNotAllowed(permitted_methods=['GET'])
Пример #4
0
def unapproved_transfers(request):
    if request.method == 'GET':
        auth_error = authenticate_request(request)

        response = {}

        if auth_error == None:
            message    = ''
            error      = None
            unapproved = []

            jobs = models.Job.objects.filter(
                 (
                     Q(jobtype="Approve standard transfer")
                     | Q(jobtype="Approve DSpace transfer")
                     | Q(jobtype="Approve bagit transfer")
                     | Q(jobtype="Approve zipped bagit transfer")
                 ) & Q(currentstep='Awaiting decision')
            )

            for job in jobs:
                # remove standard transfer path from directory (and last character)
                type_and_directory = job.directory.replace(
                    get_modified_standard_transfer_path() + '/',
                    '',
                    1
                )

                # remove trailing slash if not a zipped bag file
                if not helpers.file_is_an_archive(job.directory):
                    type_and_directory = type_and_directory[:-1]

                transfer_watch_directory = type_and_directory.split('/')[0]
                transfer_type = helpers.transfer_type_by_directory(transfer_watch_directory)

                job_directory = type_and_directory.replace(transfer_watch_directory + '/', '', 1)

                unapproved.append({
                    'type':      transfer_type,
                    'directory': job_directory
                })

            # get list of unapproved transfers
            # return list as JSON
            response['results'] = unapproved

            if error != None:
                response['message'] = error
                response['error']   = True
            else:
                response['message'] = 'Fetched unapproved transfers successfully.'

                if error != None:
                    return HttpResponseServerError(
                        simplejson.JSONEncoder().encode(response),
                        mimetype='application/json'
                    )
                else:
                    return HttpResponse(
                        simplejson.JSONEncoder().encode(response),
                        mimetype='application/json'
                    )
        else:
            response['message'] = auth_error
            response['error']   = True 
            return HttpResponseForbidden(
                simplejson.JSONEncoder().encode(response),
                mimetype='application/json'
            )
    else:
        return Http404
Пример #5
0
def copy_to_start_transfer(request):
    filepath  = archivematicaFunctions.unicodeToStr(request.POST.get('filepath', ''))
    type      = request.POST.get('type', '')
    accession = request.POST.get('accession', '')

    error = check_filepath_exists('/' + filepath)

    if error == None:
        # confine destination to subdir of originals
        filepath = os.path.join('/', filepath)
        basename = os.path.basename(filepath)

        # default to standard transfer
        type_paths = {
          'standard':     'standardTransfer',
          'unzipped bag': 'baggitDirectory',
          'zipped bag':   'baggitZippedDirectory',
          'dspace':       'Dspace',
          'maildir':      'maildir',
          'TRIM':         'TRIM'
        }

        try:
          type_subdir = type_paths[type]
          destination = os.path.join(ACTIVE_TRANSFER_DIR, type_subdir)
        except KeyError:
          destination = os.path.join(STANDARD_TRANSFER_DIR)

        # if transfer compontent path leads to a ZIP file, treat as zipped
        # bag
        if not helpers.file_is_an_archive(filepath):
            destination = os.path.join(destination, basename)
            destination = pad_destination_filepath_if_it_already_exists(destination)

        # relay accession via DB row that MCPClient scripts will use to get
        # supplementary info from
        if accession != '':
            temp_uuid = uuid.uuid4().__str__()
            mcp_destination = destination.replace(SHARED_DIRECTORY_ROOT + '/', '%sharedPath%') + '/'
            transfer = models.Transfer.objects.create(
                uuid=temp_uuid,
                accessionid=accession,
                currentlocation=mcp_destination
            )
            transfer.save()

        try:
            shutil.move(filepath, destination)
        except:
            error = 'Error copying from ' + filepath + ' to ' + destination + '. (' + str(sys.exc_info()[0]) + ')'

    response = {}

    if error != None:
        response['message'] = error
        response['error']   = True
    else:
        response['message'] = 'Copy successful.'

    return HttpResponse(
        simplejson.JSONEncoder().encode(response),
        mimetype='application/json'
    )
Пример #6
0
def copy_transfer_component(request):
    transfer_name = archivematicaFunctions.unicodeToStr(request.POST.get('name', ''))
    path = archivematicaFunctions.unicodeToStr(request.POST.get('path', ''))
    destination = archivematicaFunctions.unicodeToStr(request.POST.get('destination', ''))

    error = None

    if transfer_name == '':
        error = 'No transfer name provided.'
    else:
        if path == '':
            error = 'No path provided.'
        else:
            # if transfer compontent path leads to an archive, treat as zipped
            # bag
            if helpers.file_is_an_archive(path):
                rsync_copy(path, destination)
                paths_copied = 1
            else:
                transfer_dir = os.path.join(destination, transfer_name)

                # Create directory before it is used, otherwise shutil.copy()
                # would that location to store a file
                if not os.path.isdir(transfer_dir):
                    os.mkdir(transfer_dir)

                paths_copied = 0

                # cycle through each path copying files/dirs inside it to transfer dir
                for entry in sorted_directory_list(path):
                    entry_path = os.path.join(path, entry)
                    if os.path.isdir(entry_path):
                        rsync_copy(entry_path, transfer_dir)
                        """
                        destination_dir = os.path.join(transfer_dir, entry)
                        try:
                            shutil.copytree(
                                entry_path,
                                destination_dir
                            )
                        except:
                            error = 'Error copying from ' + entry_path + ' to ' + destination_dir + '. (' + str(sys.exc_info()[0]) + ')'
                        """
                    else:
                        rsync_copy(entry_path, transfer_dir)
                        #shutil.copy(entry_path, transfer_dir)

                    paths_copied = paths_copied + 1

    response = {}

    if error != None:
      response['message'] = error
      response['error']   = True
    else:
      response['message'] = 'Copied ' + str(paths_copied) + ' entries.'

    return HttpResponse(
        simplejson.JSONEncoder().encode(response),
        mimetype='application/json'
    )
Пример #7
0
def copy_to_start_transfer(request):
    filepath = archivematicaFunctions.unicodeToStr(
        request.POST.get('filepath', ''))
    type = request.POST.get('type', '')
    accession = request.POST.get('accession', '')

    error = check_filepath_exists('/' + filepath)

    if error == None:
        # confine destination to subdir of originals
        filepath = os.path.join('/', filepath)
        basename = os.path.basename(filepath)

        # default to standard transfer
        type_paths = {
            'standard': 'standardTransfer',
            'unzipped bag': 'baggitDirectory',
            'zipped bag': 'baggitZippedDirectory',
            'dspace': 'Dspace',
            'maildir': 'maildir',
            'TRIM': 'TRIM'
        }

        try:
            type_subdir = type_paths[type]
            destination = os.path.join(ACTIVE_TRANSFER_DIR, type_subdir)
        except KeyError:
            destination = os.path.join(STANDARD_TRANSFER_DIR)

        # if transfer compontent path leads to a ZIP file, treat as zipped
        # bag
        if not helpers.file_is_an_archive(filepath):
            destination = os.path.join(destination, basename)
            destination = pad_destination_filepath_if_it_already_exists(
                destination)

        # relay accession via DB row that MCPClient scripts will use to get
        # supplementary info from
        if accession != '':
            temp_uuid = uuid.uuid4().__str__()
            mcp_destination = destination.replace(SHARED_DIRECTORY_ROOT + '/',
                                                  '%sharedPath%') + '/'
            transfer = models.Transfer.objects.create(
                uuid=temp_uuid,
                accessionid=accession,
                currentlocation=mcp_destination)
            transfer.save()

        try:
            shutil.move(filepath, destination)
        except:
            error = 'Error copying from ' + filepath + ' to ' + destination + '. (' + str(
                sys.exc_info()[0]) + ')'

    response = {}

    if error != None:
        response['message'] = error
        response['error'] = True
    else:
        response['message'] = 'Copy successful.'

    return HttpResponse(simplejson.JSONEncoder().encode(response),
                        mimetype='application/json')
Пример #8
0
def copy_transfer_component(request):
    transfer_name = archivematicaFunctions.unicodeToStr(
        request.POST.get('name', ''))
    path = archivematicaFunctions.unicodeToStr(request.POST.get('path', ''))
    destination = archivematicaFunctions.unicodeToStr(
        request.POST.get('destination', ''))

    error = None

    if transfer_name == '':
        error = 'No transfer name provided.'
    else:
        if path == '':
            error = 'No path provided.'
        else:
            # if transfer compontent path leads to an archive, treat as zipped
            # bag
            if helpers.file_is_an_archive(path):
                rsync_copy(path, destination)
                paths_copied = 1
            else:
                transfer_dir = os.path.join(destination, transfer_name)

                # Create directory before it is used, otherwise shutil.copy()
                # would that location to store a file
                if not os.path.isdir(transfer_dir):
                    os.mkdir(transfer_dir)

                paths_copied = 0

                # cycle through each path copying files/dirs inside it to transfer dir
                for entry in sorted_directory_list(path):
                    entry_path = os.path.join(path, entry)
                    if os.path.isdir(entry_path):
                        rsync_copy(entry_path, transfer_dir)
                        """
                        destination_dir = os.path.join(transfer_dir, entry)
                        try:
                            shutil.copytree(
                                entry_path,
                                destination_dir
                            )
                        except:
                            error = 'Error copying from ' + entry_path + ' to ' + destination_dir + '. (' + str(sys.exc_info()[0]) + ')'
                        """
                    else:
                        rsync_copy(entry_path, transfer_dir)
                        #shutil.copy(entry_path, transfer_dir)

                    paths_copied = paths_copied + 1

    response = {}

    if error != None:
        response['message'] = error
        response['error'] = True
    else:
        response['message'] = 'Copied ' + str(paths_copied) + ' entries.'

    return HttpResponse(simplejson.JSONEncoder().encode(response),
                        mimetype='application/json')