def process_parse(self, source_file):
'''Processes the given source file parse by creating the appropriate jobs if the rule is triggered. All
database changes are made in an atomic transaction.
:param source_file_id: The source file that was parsed
:type source_file_id: :class:`source.models.SourceFile`
'''
# If this parse file has the correct media type or the correct data types, the rule is triggered
media_type_match = not self._media_type or self._media_type == source_file.media_type
data_types_match = not self._data_types or self._data_types <= source_file.get_data_type_tags()
if not media_type_match or not data_types_match:
return
msg = 'Parse rule for '
if not self._media_type:
msg += 'all media types '
else:
msg += 'media type %s ' % self._media_type
if self._data_types:
msg += 'and data types %s ' % ','.join(self._data_types)
msg += 'was triggered'
logger.info(msg)
event = ParseTriggerEvent(self._model, source_file).save_to_db()
# Create triggered jobs
for job in self._jobs_to_create:
job_type = self._job_type_map[(job['job_type']['name'], job['job_type']['version'])]
file_input_name = job['file_input_name']
job_data = JobData({})
job_data.add_file_input(file_input_name, source_file.id)
# If workspace name has been provided, add that to the job data for each output file
if 'workspace_name' in job:
workspace = self._workspace_map[job['workspace_name']]
job_type.get_job_interface().add_workspace_to_data(job_data, workspace.id)
logger.info('Queuing new job of type %s %s', job_type.name, job_type.version)
Queue.objects.queue_new_job(job_type, job_data.get_dict(), event)
# Create triggered recipes
for recipe in self._recipes_to_create:
recipe_type = self._recipe_type_map[(recipe['recipe_type']['name'], recipe['recipe_type']['version'])]
file_input_name = recipe['file_input_name']
recipe_data = RecipeData({})
recipe_data.add_file_input(file_input_name, source_file.id)
# If workspace name has been provided, add that to the recipe data for each output file
if 'workspace_name' in recipe:
workspace = self._workspace_map[recipe['workspace_name']]
recipe_data.set_workspace_id(workspace.id)
logger.info('Queuing new recipe of type %s %s', recipe_type.name, recipe_type.version)
Queue.objects.queue_new_recipe(recipe_type, recipe_data.get_dict(), event)
def process_ingested_source_file(self, source_file, when):
"""Processes the given ingested source file by checking it against all ingest trigger rules and creating the
corresponding jobs and recipes for any triggered rules. All database changes are made in an atomic transaction.
:param source_file: The source file that was ingested
:type source_file: :class:`source.models.SourceFile`
:param when: When the source file was ingested
:type when: :class:`datetime.datetime`
"""
msg = 'Processing trigger rules for ingested source file with media type %s and data types %s'
logger.info(msg, source_file.media_type,
str(list(source_file.get_data_type_tags())))
any_rules = False
for entry in RecipeType.objects.get_active_trigger_rules(INGEST_TYPE):
rule = entry[0]
thing_to_create = entry[1]
rule_config = rule.get_configuration()
condition = rule_config.get_condition()
if condition.is_condition_met(source_file):
logger.info(condition.get_triggered_message())
any_rules = True
event = self._create_ingest_trigger_event(
source_file, rule, when)
workspace = Workspace.objects.get(
name=rule_config.get_workspace_name())
if isinstance(thing_to_create, JobType):
job_type = thing_to_create
job_data = JobData({})
job_data.add_file_input(rule_config.get_input_data_name(),
source_file.id)
job_type.get_job_interface().add_workspace_to_data(
job_data, workspace.id)
logger.info('Queuing new job of type %s %s', job_type.name,
job_type.version)
Queue.objects.queue_new_job(job_type, job_data, event)
elif isinstance(thing_to_create, RecipeType):
recipe_type = thing_to_create
recipe_data = RecipeData({})
recipe_data.add_file_input(
rule_config.get_input_data_name(), source_file.id)
recipe_data.set_workspace_id(workspace.id)
logger.info('Queuing new recipe of type %s %s',
recipe_type.name, recipe_type.version)
Queue.objects.queue_new_recipe(recipe_type, recipe_data,
event)
if not any_rules:
logger.info('No rules triggered')
def process_ingest(self, ingest, source_file_id):
"""Processes the given source file ingest by creating the appropriate jobs if the rule is triggered. All
database changes are made in an atomic transaction.
:param ingest: The ingest to process
:type ingest: :class:`ingest.models.Ingest`
:param source_file_id: The ID of the source file that was ingested
:type source_file_id: long
"""
# Only trigger when this ingest file has the correct media type and ingest types
if self._media_type and self._media_type != ingest.media_type:
return
if not self._data_types.issubset(ingest.get_data_type_tags()):
return
if not self._media_type:
logger.info("Ingest rule for all media types was triggered")
else:
logger.info("Ingest rule for media type %s was triggered", self._media_type)
event = IngestTriggerEvent(self._model, ingest).save_to_db()
# Create triggered jobs
for job in self._jobs_to_create:
job_type = self._job_type_map[(job["job_type"]["name"], job["job_type"]["version"])]
file_input_name = job["file_input_name"]
job_data = JobData({})
job_data.add_file_input(file_input_name, source_file_id)
# If workspace name has been provided, add that to the job data for each output file
if "workspace_name" in job:
workspace = self._workspace_map[job["workspace_name"]]
job_type.get_job_interface().add_workspace_to_data(job_data, workspace.id)
logger.info("Queuing new job of type %s %s", job_type.name, job_type.version)
Queue.objects.queue_new_job(job_type, job_data.get_dict(), event)
# Create triggered recipes
for recipe in self._recipes_to_create:
recipe_type = self._recipe_type_map[(recipe["recipe_type"]["name"], recipe["recipe_type"]["version"])]
file_input_name = recipe["file_input_name"]
recipe_data = RecipeData({})
recipe_data.add_file_input(file_input_name, source_file_id)
# If workspace name has been provided, add that to the recipe data for each output file
if "workspace_name" in recipe:
workspace = self._workspace_map[recipe["workspace_name"]]
recipe_data.set_workspace_id(workspace.id)
logger.info("Queuing new recipe of type %s %s", recipe_type.name, recipe_type.version)
Queue.objects.queue_new_recipe(recipe_type, recipe_data.get_dict(), event)
def test_successful(self):
"""Tests calling JobData.add_file_input() successfully."""
data = {'input_data': []}
job_data = JobData(data)
# Method to test, we will test it by calling validate below
job_data.add_file_input('File1', self.file_1.id)
file_desc_1 = ScaleFileDescription()
file_desc_1.add_allowed_media_type('application/json')
files = {'File1': (True, False, file_desc_1)}
# No exception is success
warnings = JobData(data).validate_input_files(files)
self.assertFalse(warnings)
def test_successful(self):
"""Tests calling JobData.add_file_input() successfully."""
data = {'input_data': []}
job_data = JobData(data)
# Method to test, we will test it by calling validate below
job_data.add_file_input('File1', self.file_1.id)
file_desc_1 = ScaleFileDescription()
file_desc_1.add_allowed_media_type('application/json')
files = {'File1': (True, False, file_desc_1)}
# No exception is success
warnings = JobData(data).validate_input_files(files)
self.assertFalse(warnings)
def process_ingested_source_file(self, source_file, when):
"""Processes the given ingested source file by checking it against all ingest trigger rules and creating the
corresponding jobs and recipes for any triggered rules. All database changes are made in an atomic transaction.
:param source_file: The source file that was ingested
:type source_file: :class:`source.models.SourceFile`
:param when: When the source file was ingested
:type when: :class:`datetime.datetime`
"""
msg = 'Processing trigger rules for ingested source file with media type %s and data types %s'
logger.info(msg, source_file.media_type, str(list(source_file.get_data_type_tags())))
any_rules = False
for entry in RecipeType.objects.get_active_trigger_rules(INGEST_TYPE):
rule = entry[0]
thing_to_create = entry[1]
rule_config = rule.get_configuration()
condition = rule_config.get_condition()
if condition.is_condition_met(source_file):
logger.info(condition.get_triggered_message())
any_rules = True
event = self._create_ingest_trigger_event(source_file, rule, when)
workspace = Workspace.objects.get(name=rule_config.get_workspace_name())
if isinstance(thing_to_create, JobType):
job_type = thing_to_create
job_data = JobData({})
job_data.add_file_input(rule_config.get_input_data_name(), source_file.id)
job_type.get_job_interface().add_workspace_to_data(job_data, workspace.id)
logger.info('Queuing new job of type %s %s', job_type.name, job_type.version)
Queue.objects.queue_new_job(job_type, job_data, event)
elif isinstance(thing_to_create, RecipeType):
recipe_type = thing_to_create
recipe_data = RecipeData({})
recipe_data.add_file_input(rule_config.get_input_data_name(), source_file.id)
recipe_data.set_workspace_id(workspace.id)
logger.info('Queuing new recipe of type %s %s', recipe_type.name, recipe_type.version)
Queue.objects.queue_new_recipe(recipe_type, recipe_data, event)
if not any_rules:
logger.info('No rules triggered')
