def test_successful_recipe_creation(self):
"""Tests successfully processing a parse that triggers recipe creation."""
# Set up data
configuration = {
'version': '1.0',
'condition': {
'media_type': 'text/plain',
},
'data': {
'input_data_name': self.input_name,
'workspace_name': self.workspace.name
},
}
rule_model = trigger_test_utils.create_trigger_rule(
trigger_type='PARSE', configuration=configuration)
self.recipe_type_1.trigger_rule = rule_model
self.recipe_type_1.save()
# Call method to test
ParseTriggerHandler().process_parsed_source_file(self.source_file)
# Check results...ensure first job is queued
queue_1 = Queue.objects.get(job_type=self.job_type_2.id)
job_exe_1 = JobExecution.objects.select_related().get(
pk=queue_1.job_exe_id)
job_1 = job_exe_1.job
self.assertEqual(job_1.data['input_data'][0]['name'], self.input_name)
self.assertEqual(job_1.data['input_data'][0]['file_id'],
self.source_file.id)
self.assertEqual(job_1.data['output_data'][0]['name'],
self.output_name)
self.assertEqual(job_1.data['output_data'][0]['workspace_id'],
self.workspace.id)
def test_successful_job_creation(self):
"""Tests successfully processing a parse that triggers job creation."""
# Set up data
configuration = {
'version': '1.0',
'condition': {
'media_type': 'text/plain',
'data_types': ['type1', 'type2'],
},
'data': {
'input_data_name': self.input_name,
'workspace_name': self.workspace.name
},
}
rule_model = trigger_test_utils.create_trigger_rule(
trigger_type='PARSE', configuration=configuration)
self.job_type_1.trigger_rule = rule_model
self.job_type_1.save()
# Call method to test
ParseTriggerHandler().process_parsed_source_file(self.source_file)
# Check results
queue_1 = Queue.objects.get(job_type=self.job_type_1.id)
job_1 = Job.objects.get(pk=queue_1.job_id)
self.assertEqual(job_1.data['input_data'][0]['name'], self.input_name)
self.assertEqual(job_1.data['input_data'][0]['file_id'],
self.source_file.id)
def ready(self):
"""
Override this method in subclasses to run code when Django starts.
"""
from job.configuration.data.data_file import DATA_FILE_PARSE_SAVER
from source.configuration.source_data_file import SourceDataFileParseSaver
from source.triggers.parse_trigger_handler import ParseTriggerHandler
from trigger.handler import register_trigger_rule_handler
# Register source file parse saver
DATA_FILE_PARSE_SAVER[u'DATA_FILE_PARSE_SAVER'] = SourceDataFileParseSaver()
# Register parse trigger rule handler
register_trigger_rule_handler(ParseTriggerHandler())
def ready(self):
"""
Override this method in subclasses to run code when Django starts.
"""
from job.configuration.data.data_file import DATA_FILE_PARSE_SAVER
from source.configuration.source_data_file import SourceDataFileParseSaver
from source.triggers.parse_trigger_handler import ParseTriggerHandler
from trigger.handler import register_trigger_rule_handler
# Register source file parse saver
DATA_FILE_PARSE_SAVER[
'DATA_FILE_PARSE_SAVER'] = SourceDataFileParseSaver()
# Register parse trigger rule handler
register_trigger_rule_handler(ParseTriggerHandler())
# Register source message types
from messaging.messages.factory import add_message_type
from source.messages.purge_source_file import PurgeSourceFile
add_message_type(PurgeSourceFile)
def save_parse_results(self, src_file_id, geo_json, data_started,
data_ended, data_types, new_workspace_path,
work_dir):
'''Saves the given parse results to the source file for the given ID. All database changes occur in an atomic
transaction.
:param src_file_id: The ID of the source file
:type src_file_id: int
:param geo_json: The associated geojson data, possibly None
:type geo_json: dict
:param data_started: The start time of the data contained in the source file, possibly None
:type data_started: :class:`datetime.datetime` or None
:param data_ended: The end time of the data contained in the source file, possibly None
:type data_ended: :class:`datetime.datetime` or None
:param data_types: List of strings containing the data types tags for this source file.
:type data_types: list
:param new_workspace_path: New workspace path to move the source file to now that parse data is available. If
None, the source file should not be moved.
:type new_workspace_path: str
:param work_dir: Absolute path to a local work directory available to assist in moving the source file. Only
needed if new_workspace_path is not None.
:type work_dir: str
'''
geom = None
props = None
if geo_json:
geom, props = geo_utils.parse_geo_json(geo_json)
# Acquire model lock
src_file = SourceFile.objects.select_for_update().get(pk=src_file_id)
src_file.is_parsed = True
src_file.parsed = now()
src_file.data_started = data_started
src_file.data_ended = data_ended
target_date = src_file.data_started
if target_date is None:
target_date = src_file.data_ended
if target_date is None:
target_date = src_file.created
for tag in data_types:
src_file.add_data_type_tag(tag)
if geom:
src_file.geometry = geom
src_file.center_point = geo_utils.get_center_point(geom)
src_file.meta_data = props
# src_file already exists so we don't need to save/set_countries/save, just a single save is fine
src_file.set_countries()
src_file.save()
# Move the source file if a new workspace path is provided and the workspace allows it
if new_workspace_path and src_file.workspace.is_move_enabled:
old_workspace_path = src_file.file_path
ScaleFile.objects.move_files(work_dir,
[(src_file, new_workspace_path)])
try:
# Check trigger rules for parsed source files
ParseTriggerHandler().process_parsed_source_file(src_file)
except Exception:
# Move file back if there was an error
if new_workspace_path and src_file.workspace.is_move_enabled:
ScaleFile.objects.move_files(work_dir,
[(src_file, old_workspace_path)])
raise
def save_parse_results(self, src_file_id, geo_json, data_started,
data_ended, data_types, new_workspace_path):
"""Saves the given parse results to the source file for the given ID. All database changes occur in an atomic
transaction.
:param src_file_id: The ID of the source file
:type src_file_id: int
:param geo_json: The associated geojson data, possibly None
:type geo_json: dict
:param data_started: The start time of the data contained in the source file, possibly None
:type data_started: :class:`datetime.datetime` or None
:param data_ended: The end time of the data contained in the source file, possibly None
:type data_ended: :class:`datetime.datetime` or None
:param data_types: List of strings containing the data types tags for this source file.
:type data_types: [string]
:param new_workspace_path: New workspace path to move the source file to now that parse data is available. If
None, the source file should not be moved.
:type new_workspace_path: str
"""
geom = None
props = None
if geo_json:
geom, props = geo_utils.parse_geo_json(geo_json)
# Acquire model lock
src_file = ScaleFile.objects.select_for_update().get(
pk=src_file_id, file_type='SOURCE')
src_file.is_parsed = True
src_file.parsed = now()
if data_started and not data_ended:
src_file.data_started = data_started
src_file.data_ended = data_started
elif not data_started and data_ended:
src_file.data_started = data_ended
src_file.data_ended = data_ended
elif not data_ended and not data_started:
src_file.data_started = None
src_file.data_ended = None
else:
src_file.data_started = data_started
src_file.data_ended = data_ended
src_file.source_started = src_file.data_started
src_file.source_ended = src_file.data_ended
for tag in data_types:
src_file.add_data_type_tag(tag)
if geom:
src_file.geometry = geom
src_file.center_point = geo_utils.get_center_point(geom)
if props:
src_file.meta_data = props
# src_file already exists so we don't need to save/set_countries/save, just a single save is fine
src_file.set_countries()
src_file.save()
try:
# Try to update corresponding ingest models with this file's data time
from ingest.models import Ingest
Ingest.objects.filter(source_file_id=src_file_id).update(
data_started=data_started, data_ended=data_ended)
except ImportError:
pass
# Move the source file if a new workspace path is provided and the workspace allows it
old_workspace_path = src_file.file_path
if new_workspace_path and src_file.workspace.is_move_enabled:
ScaleFile.objects.move_files(
[FileMove(src_file, new_workspace_path)])
try:
# Check trigger rules for parsed source files
ParseTriggerHandler().process_parsed_source_file(src_file)
except Exception:
# Move file back if there was an error
if new_workspace_path and src_file.workspace.is_move_enabled:
ScaleFile.objects.move_files(
[FileMove(src_file, old_workspace_path)])
raise