def process_parse(self, source_file):
'''Processes the given source file parse by creating the appropriate jobs if the rule is triggered. All
database changes are made in an atomic transaction.
:param source_file_id: The source file that was parsed
:type source_file_id: :class:`source.models.SourceFile`
'''
# If this parse file has the correct media type or the correct data types, the rule is triggered
media_type_match = not self._media_type or self._media_type == source_file.media_type
data_types_match = not self._data_types or self._data_types <= source_file.get_data_type_tags()
if not media_type_match or not data_types_match:
return
msg = 'Parse rule for '
if not self._media_type:
msg += 'all media types '
else:
msg += 'media type %s ' % self._media_type
if self._data_types:
msg += 'and data types %s ' % ','.join(self._data_types)
msg += 'was triggered'
logger.info(msg)
event = ParseTriggerEvent(self._model, source_file).save_to_db()
# Create triggered jobs
for job in self._jobs_to_create:
job_type = self._job_type_map[(job['job_type']['name'], job['job_type']['version'])]
file_input_name = job['file_input_name']
job_data = JobData({})
job_data.add_file_input(file_input_name, source_file.id)
# If workspace name has been provided, add that to the job data for each output file
if 'workspace_name' in job:
workspace = self._workspace_map[job['workspace_name']]
job_type.get_job_interface().add_workspace_to_data(job_data, workspace.id)
logger.info('Queuing new job of type %s %s', job_type.name, job_type.version)
Queue.objects.queue_new_job(job_type, job_data.get_dict(), event)
# Create triggered recipes
for recipe in self._recipes_to_create:
recipe_type = self._recipe_type_map[(recipe['recipe_type']['name'], recipe['recipe_type']['version'])]
file_input_name = recipe['file_input_name']
recipe_data = RecipeData({})
recipe_data.add_file_input(file_input_name, source_file.id)
# If workspace name has been provided, add that to the recipe data for each output file
if 'workspace_name' in recipe:
workspace = self._workspace_map[recipe['workspace_name']]
recipe_data.set_workspace_id(workspace.id)
logger.info('Queuing new recipe of type %s %s', recipe_type.name, recipe_type.version)
Queue.objects.queue_new_recipe(recipe_type, recipe_data.get_dict(), event)
def process_ingested_source_file(self, source_file, when):
"""Processes the given ingested source file by checking it against all ingest trigger rules and creating the
corresponding jobs and recipes for any triggered rules. All database changes are made in an atomic transaction.
:param source_file: The source file that was ingested
:type source_file: :class:`source.models.SourceFile`
:param when: When the source file was ingested
:type when: :class:`datetime.datetime`
"""
msg = 'Processing trigger rules for ingested source file with media type %s and data types %s'
logger.info(msg, source_file.media_type,
str(list(source_file.get_data_type_tags())))
any_rules = False
for entry in RecipeType.objects.get_active_trigger_rules(INGEST_TYPE):
rule = entry[0]
thing_to_create = entry[1]
rule_config = rule.get_configuration()
condition = rule_config.get_condition()
if condition.is_condition_met(source_file):
logger.info(condition.get_triggered_message())
any_rules = True
event = self._create_ingest_trigger_event(
source_file, rule, when)
workspace = Workspace.objects.get(
name=rule_config.get_workspace_name())
if isinstance(thing_to_create, JobType):
job_type = thing_to_create
job_data = JobData({})
job_data.add_file_input(rule_config.get_input_data_name(),
source_file.id)
job_type.get_job_interface().add_workspace_to_data(
job_data, workspace.id)
logger.info('Queuing new job of type %s %s', job_type.name,
job_type.version)
Queue.objects.queue_new_job(job_type, job_data, event)
elif isinstance(thing_to_create, RecipeType):
recipe_type = thing_to_create
recipe_data = RecipeData({})
recipe_data.add_file_input(
rule_config.get_input_data_name(), source_file.id)
recipe_data.set_workspace_id(workspace.id)
logger.info('Queuing new recipe of type %s %s',
recipe_type.name, recipe_type.version)
Queue.objects.queue_new_recipe(recipe_type, recipe_data,
event)
if not any_rules:
logger.info('No rules triggered')
def process_ingest(self, ingest, source_file_id):
"""Processes the given source file ingest by creating the appropriate jobs if the rule is triggered. All
database changes are made in an atomic transaction.
:param ingest: The ingest to process
:type ingest: :class:`ingest.models.Ingest`
:param source_file_id: The ID of the source file that was ingested
:type source_file_id: long
"""
# Only trigger when this ingest file has the correct media type and ingest types
if self._media_type and self._media_type != ingest.media_type:
return
if not self._data_types.issubset(ingest.get_data_type_tags()):
return
if not self._media_type:
logger.info("Ingest rule for all media types was triggered")
else:
logger.info("Ingest rule for media type %s was triggered", self._media_type)
event = IngestTriggerEvent(self._model, ingest).save_to_db()
# Create triggered jobs
for job in self._jobs_to_create:
job_type = self._job_type_map[(job["job_type"]["name"], job["job_type"]["version"])]
file_input_name = job["file_input_name"]
job_data = JobData({})
job_data.add_file_input(file_input_name, source_file_id)
# If workspace name has been provided, add that to the job data for each output file
if "workspace_name" in job:
workspace = self._workspace_map[job["workspace_name"]]
job_type.get_job_interface().add_workspace_to_data(job_data, workspace.id)
logger.info("Queuing new job of type %s %s", job_type.name, job_type.version)
Queue.objects.queue_new_job(job_type, job_data.get_dict(), event)
# Create triggered recipes
for recipe in self._recipes_to_create:
recipe_type = self._recipe_type_map[(recipe["recipe_type"]["name"], recipe["recipe_type"]["version"])]
file_input_name = recipe["file_input_name"]
recipe_data = RecipeData({})
recipe_data.add_file_input(file_input_name, source_file_id)
# If workspace name has been provided, add that to the recipe data for each output file
if "workspace_name" in recipe:
workspace = self._workspace_map[recipe["workspace_name"]]
recipe_data.set_workspace_id(workspace.id)
logger.info("Queuing new recipe of type %s %s", recipe_type.name, recipe_type.version)
Queue.objects.queue_new_recipe(recipe_type, recipe_data.get_dict(), event)
def process_ingested_source_file(self, source_file, when):
"""Processes the given ingested source file by checking it against all ingest trigger rules and creating the
corresponding jobs and recipes for any triggered rules. All database changes are made in an atomic transaction.
:param source_file: The source file that was ingested
:type source_file: :class:`source.models.SourceFile`
:param when: When the source file was ingested
:type when: :class:`datetime.datetime`
"""
msg = 'Processing trigger rules for ingested source file with media type %s and data types %s'
logger.info(msg, source_file.media_type, str(list(source_file.get_data_type_tags())))
any_rules = False
for entry in RecipeType.objects.get_active_trigger_rules(INGEST_TYPE):
rule = entry[0]
thing_to_create = entry[1]
rule_config = rule.get_configuration()
condition = rule_config.get_condition()
if condition.is_condition_met(source_file):
logger.info(condition.get_triggered_message())
any_rules = True
event = self._create_ingest_trigger_event(source_file, rule, when)
workspace = Workspace.objects.get(name=rule_config.get_workspace_name())
if isinstance(thing_to_create, JobType):
job_type = thing_to_create
job_data = JobData({})
job_data.add_file_input(rule_config.get_input_data_name(), source_file.id)
job_type.get_job_interface().add_workspace_to_data(job_data, workspace.id)
logger.info('Queuing new job of type %s %s', job_type.name, job_type.version)
Queue.objects.queue_new_job(job_type, job_data, event)
elif isinstance(thing_to_create, RecipeType):
recipe_type = thing_to_create
recipe_data = RecipeData({})
recipe_data.add_file_input(rule_config.get_input_data_name(), source_file.id)
recipe_data.set_workspace_id(workspace.id)
logger.info('Queuing new recipe of type %s %s', recipe_type.name, recipe_type.version)
Queue.objects.queue_new_recipe(recipe_type, recipe_data, event)
if not any_rules:
logger.info('No rules triggered')
def _process_trigger(self, batch, trigger_config, input_file):
"""Processes the given input file within the context of a particular batch request.
Each batch recipe and its batch jobs are created in an atomic transaction to support resuming the batch command
when it is interrupted prematurely.
:param batch: The batch that defines the recipes to schedule
:type batch: :class:`batch.models.Batch`
:param trigger_config: The trigger rule configuration to use when evaluating source files.
:type trigger_config: :class:`batch.configuration.definition.batch_definition.BatchTriggerConfiguration`
:param input_file: The input file that should trigger a new batch recipe
:type input_file: :class:`storage.models.ScaleFile`
"""
# Check whether the source file matches the trigger condition
if hasattr(trigger_config, 'get_condition'):
condition = trigger_config.get_condition()
if not condition.is_condition_met(input_file):
return
# Build recipe data to pass input file parameters to new recipes
recipe_data = RecipeData({})
if hasattr(trigger_config, 'get_input_data_name'):
recipe_data.add_file_input(trigger_config.get_input_data_name(),
input_file.id)
if hasattr(trigger_config, 'get_workspace_name'):
workspace = Workspace.objects.get(
name=trigger_config.get_workspace_name())
recipe_data.set_workspace_id(workspace.id)
description = {
'version': '1.0',
'file_id': input_file.id,
'file_name': input_file.file_name,
}
event = TriggerEvent.objects.create_trigger_event(
'BATCH', None, description, timezone.now())
handler = Queue.objects.queue_new_recipe(batch.recipe_type,
recipe_data, event)
# Create all the batch models for the new recipe and jobs
self._create_batch_models(batch, handler)