Example #1
0
    def process_parse(self, source_file):
        '''Processes the given source file parse by creating the appropriate jobs if the rule is triggered. All
        database changes are made in an atomic transaction.

        :param source_file_id: The source file that was parsed
        :type source_file_id: :class:`source.models.SourceFile`
        '''

        # If this parse file has the correct media type or the correct data types, the rule is triggered
        media_type_match = not self._media_type or self._media_type == source_file.media_type
        data_types_match = not self._data_types or self._data_types <= source_file.get_data_type_tags()

        if not media_type_match or not data_types_match:
            return

        msg = 'Parse rule for '
        if not self._media_type:
            msg += 'all media types '
        else:
            msg += 'media type %s ' % self._media_type
        if self._data_types:
            msg += 'and data types %s ' % ','.join(self._data_types)
        msg += 'was triggered'
        logger.info(msg)

        event = ParseTriggerEvent(self._model, source_file).save_to_db()

        # Create triggered jobs
        for job in self._jobs_to_create:
            job_type = self._job_type_map[(job['job_type']['name'], job['job_type']['version'])]
            file_input_name = job['file_input_name']
            job_data = JobData({})
            job_data.add_file_input(file_input_name, source_file.id)

            # If workspace name has been provided, add that to the job data for each output file
            if 'workspace_name' in job:
                workspace = self._workspace_map[job['workspace_name']]
                job_type.get_job_interface().add_workspace_to_data(job_data, workspace.id)
            logger.info('Queuing new job of type %s %s', job_type.name, job_type.version)
            Queue.objects.queue_new_job(job_type, job_data.get_dict(), event)

        # Create triggered recipes
        for recipe in self._recipes_to_create:
            recipe_type = self._recipe_type_map[(recipe['recipe_type']['name'], recipe['recipe_type']['version'])]
            file_input_name = recipe['file_input_name']
            recipe_data = RecipeData({})
            recipe_data.add_file_input(file_input_name, source_file.id)

            # If workspace name has been provided, add that to the recipe data for each output file
            if 'workspace_name' in recipe:
                workspace = self._workspace_map[recipe['workspace_name']]
                recipe_data.set_workspace_id(workspace.id)
            logger.info('Queuing new recipe of type %s %s', recipe_type.name, recipe_type.version)
            Queue.objects.queue_new_recipe(recipe_type, recipe_data.get_dict(), event)
Example #2
0
    def process_ingested_source_file(self, source_file, when):
        """Processes the given ingested source file by checking it against all ingest trigger rules and creating the
        corresponding jobs and recipes for any triggered rules. All database changes are made in an atomic transaction.

        :param source_file: The source file that was ingested
        :type source_file: :class:`source.models.SourceFile`
        :param when: When the source file was ingested
        :type when: :class:`datetime.datetime`
        """

        msg = 'Processing trigger rules for ingested source file with media type %s and data types %s'
        logger.info(msg, source_file.media_type,
                    str(list(source_file.get_data_type_tags())))

        any_rules = False
        for entry in RecipeType.objects.get_active_trigger_rules(INGEST_TYPE):
            rule = entry[0]
            thing_to_create = entry[1]
            rule_config = rule.get_configuration()
            condition = rule_config.get_condition()

            if condition.is_condition_met(source_file):
                logger.info(condition.get_triggered_message())
                any_rules = True

                event = self._create_ingest_trigger_event(
                    source_file, rule, when)
                workspace = Workspace.objects.get(
                    name=rule_config.get_workspace_name())

                if isinstance(thing_to_create, JobType):
                    job_type = thing_to_create
                    job_data = JobData({})
                    job_data.add_file_input(rule_config.get_input_data_name(),
                                            source_file.id)
                    job_type.get_job_interface().add_workspace_to_data(
                        job_data, workspace.id)
                    logger.info('Queuing new job of type %s %s', job_type.name,
                                job_type.version)
                    Queue.objects.queue_new_job(job_type, job_data, event)
                elif isinstance(thing_to_create, RecipeType):
                    recipe_type = thing_to_create
                    recipe_data = RecipeData({})
                    recipe_data.add_file_input(
                        rule_config.get_input_data_name(), source_file.id)
                    recipe_data.set_workspace_id(workspace.id)
                    logger.info('Queuing new recipe of type %s %s',
                                recipe_type.name, recipe_type.version)
                    Queue.objects.queue_new_recipe(recipe_type, recipe_data,
                                                   event)

        if not any_rules:
            logger.info('No rules triggered')
Example #3
0
    def process_ingest(self, ingest, source_file_id):
        """Processes the given source file ingest by creating the appropriate jobs if the rule is triggered. All
        database changes are made in an atomic transaction.

        :param ingest: The ingest to process
        :type ingest: :class:`ingest.models.Ingest`
        :param source_file_id: The ID of the source file that was ingested
        :type source_file_id: long
        """

        # Only trigger when this ingest file has the correct media type and ingest types
        if self._media_type and self._media_type != ingest.media_type:
            return
        if not self._data_types.issubset(ingest.get_data_type_tags()):
            return

        if not self._media_type:
            logger.info("Ingest rule for all media types was triggered")
        else:
            logger.info("Ingest rule for media type %s was triggered", self._media_type)
        event = IngestTriggerEvent(self._model, ingest).save_to_db()

        # Create triggered jobs
        for job in self._jobs_to_create:
            job_type = self._job_type_map[(job["job_type"]["name"], job["job_type"]["version"])]
            file_input_name = job["file_input_name"]
            job_data = JobData({})
            job_data.add_file_input(file_input_name, source_file_id)

            # If workspace name has been provided, add that to the job data for each output file
            if "workspace_name" in job:
                workspace = self._workspace_map[job["workspace_name"]]
                job_type.get_job_interface().add_workspace_to_data(job_data, workspace.id)
            logger.info("Queuing new job of type %s %s", job_type.name, job_type.version)
            Queue.objects.queue_new_job(job_type, job_data.get_dict(), event)

        # Create triggered recipes
        for recipe in self._recipes_to_create:
            recipe_type = self._recipe_type_map[(recipe["recipe_type"]["name"], recipe["recipe_type"]["version"])]
            file_input_name = recipe["file_input_name"]
            recipe_data = RecipeData({})
            recipe_data.add_file_input(file_input_name, source_file_id)

            # If workspace name has been provided, add that to the recipe data for each output file
            if "workspace_name" in recipe:
                workspace = self._workspace_map[recipe["workspace_name"]]
                recipe_data.set_workspace_id(workspace.id)
            logger.info("Queuing new recipe of type %s %s", recipe_type.name, recipe_type.version)
            Queue.objects.queue_new_recipe(recipe_type, recipe_data.get_dict(), event)
    def process_ingested_source_file(self, source_file, when):
        """Processes the given ingested source file by checking it against all ingest trigger rules and creating the
        corresponding jobs and recipes for any triggered rules. All database changes are made in an atomic transaction.

        :param source_file: The source file that was ingested
        :type source_file: :class:`source.models.SourceFile`
        :param when: When the source file was ingested
        :type when: :class:`datetime.datetime`
        """

        msg = 'Processing trigger rules for ingested source file with media type %s and data types %s'
        logger.info(msg, source_file.media_type, str(list(source_file.get_data_type_tags())))

        any_rules = False
        for entry in RecipeType.objects.get_active_trigger_rules(INGEST_TYPE):
            rule = entry[0]
            thing_to_create = entry[1]
            rule_config = rule.get_configuration()
            condition = rule_config.get_condition()

            if condition.is_condition_met(source_file):
                logger.info(condition.get_triggered_message())
                any_rules = True

                event = self._create_ingest_trigger_event(source_file, rule, when)
                workspace = Workspace.objects.get(name=rule_config.get_workspace_name())

                if isinstance(thing_to_create, JobType):
                    job_type = thing_to_create
                    job_data = JobData({})
                    job_data.add_file_input(rule_config.get_input_data_name(), source_file.id)
                    job_type.get_job_interface().add_workspace_to_data(job_data, workspace.id)
                    logger.info('Queuing new job of type %s %s', job_type.name, job_type.version)
                    Queue.objects.queue_new_job(job_type, job_data, event)
                elif isinstance(thing_to_create, RecipeType):
                    recipe_type = thing_to_create
                    recipe_data = RecipeData({})
                    recipe_data.add_file_input(rule_config.get_input_data_name(), source_file.id)
                    recipe_data.set_workspace_id(workspace.id)
                    logger.info('Queuing new recipe of type %s %s', recipe_type.name, recipe_type.version)
                    Queue.objects.queue_new_recipe(recipe_type, recipe_data, event)

        if not any_rules:
            logger.info('No rules triggered')
Example #5
0
    def _process_trigger(self, batch, trigger_config, input_file):
        """Processes the given input file within the context of a particular batch request.

        Each batch recipe and its batch jobs are created in an atomic transaction to support resuming the batch command
        when it is interrupted prematurely.

        :param batch: The batch that defines the recipes to schedule
        :type batch: :class:`batch.models.Batch`
        :param trigger_config: The trigger rule configuration to use when evaluating source files.
        :type trigger_config: :class:`batch.configuration.definition.batch_definition.BatchTriggerConfiguration`
        :param input_file: The input file that should trigger a new batch recipe
        :type input_file: :class:`storage.models.ScaleFile`
        """

        # Check whether the source file matches the trigger condition
        if hasattr(trigger_config, 'get_condition'):
            condition = trigger_config.get_condition()
            if not condition.is_condition_met(input_file):
                return

        # Build recipe data to pass input file parameters to new recipes
        recipe_data = RecipeData({})
        if hasattr(trigger_config, 'get_input_data_name'):
            recipe_data.add_file_input(trigger_config.get_input_data_name(),
                                       input_file.id)
        if hasattr(trigger_config, 'get_workspace_name'):
            workspace = Workspace.objects.get(
                name=trigger_config.get_workspace_name())
            recipe_data.set_workspace_id(workspace.id)

        description = {
            'version': '1.0',
            'file_id': input_file.id,
            'file_name': input_file.file_name,
        }
        event = TriggerEvent.objects.create_trigger_event(
            'BATCH', None, description, timezone.now())
        handler = Queue.objects.queue_new_recipe(batch.recipe_type,
                                                 recipe_data, event)

        # Create all the batch models for the new recipe and jobs
        self._create_batch_models(batch, handler)