Ejemplo n.º 1
0
    def is_there_rule_match(self,
                            file_handler,
                            workspaces,
                            no_match_status=None):
        """Applies rules to an ingest record, determining if there is a match and updating as indicated in rule match

        :param file_handler: Rules to be matched against the ingest record
        :type: :class: `ingest.handlers.file_handler.FileHandler`
        :param workspaces: mimetype to workspace mapping
        :type: dict
        :param no_match_status: Optional status to apply when rules aren't matched
        :type: string
        :return: The ingest record if matched otherwise None
        :rtype: :class:`ingest.models.Ingest`
        """

        matched = True
        logger.info('Applying rules to %s (%s, %s)', self.file_name,
                    self.media_type, file_size_to_string(self.file_size))
        matched_rule = file_handler.match_file_name(self.file_name)
        if matched_rule:
            for data_type_tag in matched_rule.data_types:
                self.add_data_type_tag(data_type_tag)
            file_path = self.file_path
            if matched_rule.new_file_path:
                today = now()
                year_dir = str(today.year)
                month_dir = '%02d' % today.month
                day_dir = '%02d' % today.day
                self.new_file_path = os.path.join(matched_rule.new_file_path,
                                                  year_dir, month_dir, day_dir,
                                                  self.file_name)
                file_path = self.new_file_path
            workspace_name = self.workspace.name
            if matched_rule.new_workspace:
                self.new_workspace = workspaces[matched_rule.new_workspace]
                workspace_name = self.new_workspace.name
            if self.new_file_path or self.new_workspace:
                logger.info(
                    'Rule match, %s will be moved to %s on workspace %s',
                    self.file_name, file_path, workspace_name)
            else:
                logger.info(
                    'Rule match, %s will be registered as %s on workspace %s',
                    self.file_name, file_path, workspace_name)
        else:
            matched = False
            if no_match_status:
                logger.info('No rule match for %s, file is being set to %s',
                            self.file_name, no_match_status)
                self.status = no_match_status
            else:
                logger.info('No rule match for %s, file is being skipped',
                            self.file_name)

        return matched
Ejemplo n.º 2
0
    def _process_ingest(self, ingest, file_path, file_size):
        """Processes the ingest file by applying the Strike configuration rules. This method will update the ingest
        model in the database and create an ingest task (if applicable) in an atomic transaction. This method should
        either be called immediately after _create_ingest() or after _complete_transfer().

        :param ingest: The ingest model
        :type ingest: :class:`ingest.models.Ingest`
        :param file_path: The relative location of the ingest file within the workspace
        :type file_path: string
        :param file_size: The size of the file in bytes
        :type file_size: long
        """

        if ingest.status not in ['TRANSFERRING', 'TRANSFERRED']:
            raise Exception('Invalid ingest status: %s' % ingest.status)

        file_name = ingest.file_name
        logger.info('Applying rules to %s (%s, %s)', file_name, ingest.media_type, file_size_to_string(file_size))
        ingest.file_path = file_path
        ingest.file_size = file_size

        matched_rule = self._file_handler.match_file_name(file_name)
        if matched_rule:
            for data_type_tag in matched_rule.data_types:
                ingest.add_data_type_tag(data_type_tag)
            file_path = ingest.file_path
            if matched_rule.new_file_path:
                today = now()
                year_dir = str(today.year)
                month_dir = '%02d' % today.month
                day_dir = '%02d' % today.day
                ingest.new_file_path = os.path.join(matched_rule.new_file_path, year_dir, month_dir, day_dir, file_name)
                file_path = ingest.new_file_path
            workspace_name = ingest.workspace.name
            if matched_rule.new_workspace:
                ingest.new_workspace = self._workspaces[matched_rule.new_workspace]
                workspace_name = ingest.new_workspace.name
            if ingest.new_file_path or ingest.new_workspace:
                logger.info('Rule match, %s will be moved to %s on workspace %s', file_name, file_path, workspace_name)
            else:
                logger.info('Rule match, %s will be registered as %s on workspace %s', file_name, file_path,
                            workspace_name)
            if not ingest.id:
                ingest.save()
            self._start_ingest_task(ingest)
        else:
            logger.info('No rule match for %s, file is being deferred', file_name)
            ingest.status = 'DEFERRED'
            ingest.save()
Ejemplo n.º 3
0
    def _update_transfer(self, ingest, bytes_transferred):
        """Updates how many bytes have currently been transferred for the given ingest. The database save is the
        caller's responsibility. This method should only be used between calls to _start_transfer() and
        _complete_transfer().

        :param ingest: The ingest model
        :type ingest: :class:`ingest.models.Ingest`
        :param bytes_transferred: How many bytes have currently been transferred
        :type bytes_transferred: long
        """

        if ingest.status != 'TRANSFERRING':
            raise Exception('Invalid ingest status: %s' % ingest.status)
        ingest.bytes_transferred = bytes_transferred

        logger.info('%s of %s copied', file_size_to_string(bytes_transferred), ingest.file_name)
Ejemplo n.º 4
0
    def _update_transfer(self, ingest, bytes_transferred):
        """Updates how many bytes have currently been transferred for the given ingest. The database save is the
        caller's responsibility. This method should only be used between calls to _start_transfer() and
        _complete_transfer().

        :param ingest: The ingest model
        :type ingest: :class:`ingest.models.Ingest`
        :param bytes_transferred: How many bytes have currently been transferred
        :type bytes_transferred: long
        """

        if ingest.status != 'TRANSFERRING':
            raise Exception('Invalid ingest status: %s' % ingest.status)
        ingest.bytes_transferred = bytes_transferred

        logger.info('%s of %s copied', file_size_to_string(bytes_transferred),
                    ingest.file_name)
Ejemplo n.º 5
0
    def _complete_transfer(self, ingest, when, bytes_transferred):
        """Completes the transfer of the given ingest into a workspace. The database save is the caller's
        responsibility. This method should only be used if _start_transfer() was called first.

        :param ingest: The ingest model
        :type ingest: :class:`ingest.models.Ingest`
        :param when: When the transfer of the file completed
        :type when: :class:`datetime.datetime`
        :param bytes_transferred: How many total bytes were transferred
        :type bytes_transferred: long
        """

        if ingest.status != 'TRANSFERRING':
            raise Exception('Invalid ingest status: %s' % ingest.status)
        ingest.status = 'TRANSFERRED'
        ingest.bytes_transferred = bytes_transferred
        ingest.transfer_ended = when
        ingest.file_size = bytes_transferred

        logger.info('%s has finished transferring to %s, total of %s copied', ingest.file_name, ingest.workspace.name,
                    file_size_to_string(bytes_transferred))
Ejemplo n.º 6
0
    def _complete_transfer(self, ingest, when, bytes_transferred):
        """Completes the transfer of the given ingest into a workspace. The database save is the caller's
        responsibility. This method should only be used if _start_transfer() was called first.

        :param ingest: The ingest model
        :type ingest: :class:`ingest.models.Ingest`
        :param when: When the transfer of the file completed
        :type when: :class:`datetime.datetime`
        :param bytes_transferred: How many total bytes were transferred
        :type bytes_transferred: long
        """

        if ingest.status != 'TRANSFERRING':
            raise Exception('Invalid ingest status: %s' % ingest.status)
        ingest.status = 'TRANSFERRED'
        ingest.bytes_transferred = bytes_transferred
        ingest.transfer_ended = when
        ingest.file_size = bytes_transferred

        logger.info('%s has finished transferring to %s, total of %s copied',
                    ingest.file_name, ingest.workspace.name,
                    file_size_to_string(bytes_transferred))