def _perform_post_steps(self, job_exe):
"""Populates the full set of command arguments for the job execution
:param job_exe: The job execution
:type job_exe: :class:`job.models.JobExecution`
"""
job_interface = job_exe.job_type.get_job_interface()
job_data = job_exe.job.get_job_data()
stdout_and_stderr = None
try:
stdout_and_stderr, _last_modified = job_exe.get_log_text()
except:
logger.exception('Failed to retrieve job execution logs')
if stdout_and_stderr is None:
stdout_and_stderr = ''
with transaction.atomic():
if JobInterfaceSunset.is_seed_dict(job_interface.definition):
job_results = JobResults()
job_results.perform_post_steps(job_interface, job_data,
job_exe)
else:
job_results, results_manifest = job_interface.perform_post_steps(
job_exe, job_data, stdout_and_stderr)
job_exe_output = JobExecutionOutput()
job_exe_output.job_exe_id = job_exe.id
job_exe_output.job_id = job_exe.job_id
job_exe_output.job_type_id = job_exe.job_type_id
job_exe_output.exe_num = job_exe.exe_num
job_exe_output.output = job_results.get_dict()
job_exe_output.save()
def test_capture_output_json(self):
results = JobResults()
with patch("__builtin__.open",
mock_open(read_data=json.dumps(self.outputs_json_dict))):
results._capture_output_json(self.seed_outputs_json)
self.assertDictEqual(results.get_dict(), {
'files': {},
'json': {
'INPUT_SIZE': 50
},
'version': '7'
})
def test_capture_output_files_missing(self, get_files):
output_files = [SeedOutputFiles(self.test_output_snippet)]
get_files.side_effect = OutputCaptureError('message')
with self.assertRaises(OutputCaptureError) as exc:
JobResults()._capture_output_files(output_files)
def test_capture_source_metadata_files(self, join):
input_files = [SeedInputFiles(x) for x in self.test_input_snippets]
name_to_id = {"INPUT_FILE": [1]}
metadata_name = 'INPUT_FILE.metadata.json'
join.return_value = metadata_name
metadata = {
'type': 'Feature',
'geometry': {
'type': 'Point',
'coordinates': [0, 1]
},
'properties': {
'dataStarted': '2018-06-01T00:00:00Z',
'dataEnded': '2018-06-01T01:00:00Z',
'dataTypes': ['one', 'two', 'three']
}
}
with open(metadata_name, 'w') as metadata_file:
json.dump(metadata, metadata_file)
outputs = JobResults()._capture_source_metadata_files(
input_files, name_to_id)
os.remove(metadata_name)
self.assertEqual(len(outputs), 1)
self.assertDictEqual(outputs[1].data, metadata)
def test_capture_output_files_multiple(self, get_files):
output_files = [SeedOutputFiles(self.test_output_snippet)]
name = 'OUTPUT_TIFFS'
get_files.return_value = ['outfile0.tif', 'outfile1.tif']
outputs = JobResults()._capture_output_files(output_files)
self.assertIn(name, outputs)
files = outputs[name]
self.assertEqual(len(files), 2)
self.assertEqual(files[0].__dict__, ProductFileMetadata(name, 'outfile0.tif', media_type='image/tiff').__dict__)
self.assertEqual(files[1].__dict__, ProductFileMetadata(name, 'outfile1.tif', media_type='image/tiff').__dict__)
def test_store_output_files(self, dummy_store, isfile):
workspace = storage_test_utils.create_workspace()
files = {'OUTPUT_TIFFS': [ProductFileMetadata('OUTPUT_TIFFS', 'outfile0.tif', media_type='image/tiff')]}
job_data = JobData({})
job_config = JobConfiguration()
job_config.add_output_workspace('OUTPUT_TIFFS', workspace.name)
job_exe = Mock()
job_exe.job_type.get_job_configuration.return_value = job_config
results = JobResults()._store_output_data_files(files, job_data, job_exe)
self.assertEqual({'OUTPUT_TIFFS': [1]}, results.files)
def test_capture_output_files_metadata(self, get_files):
output_files = [SeedOutputFiles(self.test_output_snippet)]
name = 'OUTPUT_TIFFS'
get_files.return_value = ['outfile0.tif']
metadata = {
'type': 'Feature',
'geometry': {
'type': 'Point',
'coordinates': [0, 1]
},
'properties': {
'dataStarted': '2018-06-01T00:00:00Z',
'dataEnded': '2018-06-01T01:00:00Z',
'sourceStarted': '2018-06-01T00:00:00Z',
'sourceEnded': '2018-06-01T06:00:00Z',
'sourceSensorClass': 'Platform',
'sourceSensor': 'X1',
'sourceCollection': '12345A',
'sourceTask': 'Calibration'
}
}
metadata_name = 'outfile0.tif.metadata.json'
with open(metadata_name, 'w') as metadata_file:
json.dump(metadata, metadata_file)
outputs = JobResults()._capture_output_files(output_files)
os.remove(metadata_name)
self.assertIn(name, outputs)
files = outputs[name]
self.assertEqual(len(files), 1)
self.assertDictEqual(
files[0].__dict__,
ProductFileMetadata(output_name=name,
local_path='outfile0.tif',
media_type='image/tiff',
data_start='2018-06-01T00:00:00Z',
data_end='2018-06-01T01:00:00Z',
geojson=metadata,
source_started='2018-06-01T00:00:00Z',
source_ended='2018-06-01T06:00:00Z',
source_sensor_class='Platform',
source_sensor='X1',
source_collection='12345A',
source_task='Calibration').__dict__)
from django.db.utils import DatabaseError, OperationalError
from django.utils.timezone import now
from django.test import TransactionTestCase
from mock import patch
from error.exceptions import ScaleDatabaseError, ScaleIOError, ScaleOperationalError
from job.configuration.results.exceptions import InvalidResultsManifest, MissingRequiredOutput
from job.seed.results.job_results import JobResults
from job.configuration.results.results_manifest.results_manifest import ResultsManifest
from job.management.commands.scale_post_steps import Command as PostCommand
from job.models import JobExecutionOutput
from job.test import utils as job_utils
from recipe.test import utils as recipe_utils
from trigger.models import TriggerEvent
JOB_RESULTS = JobResults()
RESULTS_MANIFEST = ResultsManifest()
RESULTS = (JOB_RESULTS, RESULTS_MANIFEST)
class TestPostJobSteps(TransactionTestCase):
def setUp(self):
django.setup()
self.recipe_type = recipe_utils.create_recipe_type_v6()
self.recipe = recipe_utils.create_recipe(recipe_type=self.recipe_type)
cmd = 'command'
cmd_args = 'args'
outputs = [{
'name': 'arg1',
