def is_there_rule_match(self, file_handler, workspaces, no_match_status=None): """Applies rules to an ingest record, determining if there is a match and updating as indicated in rule match :param file_handler: Rules to be matched against the ingest record :type: :class: `ingest.handlers.file_handler.FileHandler` :param workspaces: mimetype to workspace mapping :type: dict :param no_match_status: Optional status to apply when rules aren't matched :type: string :return: The ingest record if matched otherwise None :rtype: :class:`ingest.models.Ingest` """ matched = True logger.info('Applying rules to %s (%s, %s)', self.file_name, self.media_type, file_size_to_string(self.file_size)) matched_rule = file_handler.match_file_name(self.file_name) if matched_rule: for data_type_tag in matched_rule.data_types: self.add_data_type_tag(data_type_tag) file_path = self.file_path if matched_rule.new_file_path: today = now() year_dir = str(today.year) month_dir = '%02d' % today.month day_dir = '%02d' % today.day self.new_file_path = os.path.join(matched_rule.new_file_path, year_dir, month_dir, day_dir, self.file_name) file_path = self.new_file_path workspace_name = self.workspace.name if matched_rule.new_workspace: self.new_workspace = workspaces[matched_rule.new_workspace] workspace_name = self.new_workspace.name if self.new_file_path or self.new_workspace: logger.info( 'Rule match, %s will be moved to %s on workspace %s', self.file_name, file_path, workspace_name) else: logger.info( 'Rule match, %s will be registered as %s on workspace %s', self.file_name, file_path, workspace_name) else: matched = False if no_match_status: logger.info('No rule match for %s, file is being set to %s', self.file_name, no_match_status) self.status = no_match_status else: logger.info('No rule match for %s, file is being skipped', self.file_name) return matched
def _process_ingest(self, ingest, file_path, file_size): """Processes the ingest file by applying the Strike configuration rules. This method will update the ingest model in the database and create an ingest task (if applicable) in an atomic transaction. This method should either be called immediately after _create_ingest() or after _complete_transfer(). :param ingest: The ingest model :type ingest: :class:`ingest.models.Ingest` :param file_path: The relative location of the ingest file within the workspace :type file_path: string :param file_size: The size of the file in bytes :type file_size: long """ if ingest.status not in ['TRANSFERRING', 'TRANSFERRED']: raise Exception('Invalid ingest status: %s' % ingest.status) file_name = ingest.file_name logger.info('Applying rules to %s (%s, %s)', file_name, ingest.media_type, file_size_to_string(file_size)) ingest.file_path = file_path ingest.file_size = file_size matched_rule = self._file_handler.match_file_name(file_name) if matched_rule: for data_type_tag in matched_rule.data_types: ingest.add_data_type_tag(data_type_tag) file_path = ingest.file_path if matched_rule.new_file_path: today = now() year_dir = str(today.year) month_dir = '%02d' % today.month day_dir = '%02d' % today.day ingest.new_file_path = os.path.join(matched_rule.new_file_path, year_dir, month_dir, day_dir, file_name) file_path = ingest.new_file_path workspace_name = ingest.workspace.name if matched_rule.new_workspace: ingest.new_workspace = self._workspaces[matched_rule.new_workspace] workspace_name = ingest.new_workspace.name if ingest.new_file_path or ingest.new_workspace: logger.info('Rule match, %s will be moved to %s on workspace %s', file_name, file_path, workspace_name) else: logger.info('Rule match, %s will be registered as %s on workspace %s', file_name, file_path, workspace_name) if not ingest.id: ingest.save() self._start_ingest_task(ingest) else: logger.info('No rule match for %s, file is being deferred', file_name) ingest.status = 'DEFERRED' ingest.save()
def _update_transfer(self, ingest, bytes_transferred): """Updates how many bytes have currently been transferred for the given ingest. The database save is the caller's responsibility. This method should only be used between calls to _start_transfer() and _complete_transfer(). :param ingest: The ingest model :type ingest: :class:`ingest.models.Ingest` :param bytes_transferred: How many bytes have currently been transferred :type bytes_transferred: long """ if ingest.status != 'TRANSFERRING': raise Exception('Invalid ingest status: %s' % ingest.status) ingest.bytes_transferred = bytes_transferred logger.info('%s of %s copied', file_size_to_string(bytes_transferred), ingest.file_name)
def _complete_transfer(self, ingest, when, bytes_transferred): """Completes the transfer of the given ingest into a workspace. The database save is the caller's responsibility. This method should only be used if _start_transfer() was called first. :param ingest: The ingest model :type ingest: :class:`ingest.models.Ingest` :param when: When the transfer of the file completed :type when: :class:`datetime.datetime` :param bytes_transferred: How many total bytes were transferred :type bytes_transferred: long """ if ingest.status != 'TRANSFERRING': raise Exception('Invalid ingest status: %s' % ingest.status) ingest.status = 'TRANSFERRED' ingest.bytes_transferred = bytes_transferred ingest.transfer_ended = when ingest.file_size = bytes_transferred logger.info('%s has finished transferring to %s, total of %s copied', ingest.file_name, ingest.workspace.name, file_size_to_string(bytes_transferred))