def test_simple_validation_1_1(self):
json_manifest = {
"version":
"1.1",
"output_data": [{
"name": "output_file",
"file": {
"path": "/tmp/job_exe_231/outputs/output.csv",
"geo_metadata": {
"data_started": "2015-05-15T10:34:12Z",
"data_ended": "2015-05-15T10:36:12Z",
"geo_json": {
"type":
"Polygon",
"coordinates": [[[1.0, 10.0], [2.0, 10.0],
[2.0, 20.0], [1.0, 20.0],
[1.0, 10.0]]]
}
}
}
}]
}
input_files = {"input_file": False}
output_files = {"output_file": (False, True)}
manifest = ResultsManifest(json_manifest)
manifest.validate(output_files)
def test_simple_validation_1_1(self):
json_manifest = {
"version": "1.1",
"output_data": [
{
"name" : "output_file",
"file": {
"path" : "/tmp/job_exe_231/outputs/output.csv",
"geo_metadata": {
"data_started": "2015-05-15T10:34:12Z",
"data_ended" : "2015-05-15T10:36:12Z",
"geo_json": {
"type": "Polygon",
"coordinates": [ [ [ 1.0, 10.0 ], [ 2.0, 10.0 ], [ 2.0, 20.0 ],[ 1.0, 20.0 ], [ 1.0, 10.0 ] ] ]
}
}
}
}
]
}
input_files = {
"input_file": False
}
output_files = {
"output_file": (False, True)
}
manifest = ResultsManifest(json_manifest)
manifest.validate(output_files)
def test_successful_with_full_recipe(self):
"""Tests calling QueueManager.handle_job_completion() successfully with all jobs in a recipe."""
# Queue the recipe
recipe_id = Queue.objects.queue_new_recipe(self.recipe_type, self.data,
self.event)
# Fake out completing Job 1
job_1 = RecipeJob.objects.select_related('job').get(
recipe_id=recipe_id, job_name='Job 1').job
job_exe_1 = JobExecution.objects.get(job_id=job_1.id)
output_file_1 = product_test_utils.create_product(
job_exe=job_exe_1, workspace=self.workspace)
output_file_2 = product_test_utils.create_product(
job_exe=job_exe_1, workspace=self.workspace)
results = JobResults()
results.add_file_list_parameter('Test Output 1',
[output_file_1.id, output_file_2.id])
JobExecution.objects.post_steps_results(job_exe_1.id, results,
ResultsManifest())
Job.objects.filter(pk=job_1.id).update(status='RUNNING')
JobExecution.objects.filter(pk=job_exe_1.id).update(status='RUNNING')
Queue.objects.handle_job_completion(job_exe_1.id, now())
# Fake out completing Job 2
job_2 = RecipeJob.objects.select_related('job').get(
recipe_id=recipe_id, job_name='Job 2').job
job_exe_2 = JobExecution.objects.get(job_id=job_2.id)
output_file_1 = product_test_utils.create_product(
job_exe=job_exe_2, workspace=self.workspace)
output_file_2 = product_test_utils.create_product(
job_exe=job_exe_2, workspace=self.workspace)
results = JobResults()
results.add_file_list_parameter('Test Output 2',
[output_file_1.id, output_file_2.id])
JobExecution.objects.post_steps_results(job_exe_2.id, results,
ResultsManifest())
Job.objects.filter(pk=job_2.id).update(status='RUNNING')
JobExecution.objects.filter(pk=job_exe_2.id).update(status='RUNNING')
# Call method to test
Queue.objects.handle_job_completion(job_exe_2.id, now())
# Make sure processor was called
self.assertEqual(self.mock_processor.process_completed.call_count, 2)
# Make sure final recipe attributes are updated
recipe = Recipe.objects.get(pk=recipe_id)
self.assertIsNotNone(recipe.completed)
def test_convert_1_0_to_1_1(self):
json_manifest = {
"version": "1.0",
"files": [
{"name" : "output_file", "path" : "/tmp/job_exe_231/outputs/output.csv"},
{"name": "output_files", "paths": ["/tmp/job_exe_231/outputs/output.csv", "/tmp/job_exe_231/outputs/output2.csv"]}
],
"parse_results": [
{
"filename" : "myfile.h5",
"data_started" : "2015-05-15T10:34:12Z",
"data_ended" : "2015-05-15T10:36:12Z",
"data_types" : ["H5", "VEG"]
}
],
"errors": []
}
new_format = {
"version": "1.1",
"output_data": [
{
"name" : "output_file",
"file": {
"path" : "/tmp/job_exe_231/outputs/output.csv"
}
},
{
"name" : "output_files",
"files": [
{
"path" : "/tmp/job_exe_231/outputs/output.csv"
},
{
"path" : "/tmp/job_exe_231/outputs/output2.csv"
}
]
}
],
"parse_results": [
{
"filename" : "myfile.h5",
"data_started" : "2015-05-15T10:34:12Z",
"data_ended" : "2015-05-15T10:36:12Z",
"data_types" : ["H5", "VEG"]
}
],
"errors": []
}
manifest = ResultsManifest(json_manifest)
converted = manifest._convert_schema(json_manifest)
self.assertEqual(converted, new_format)
def test_convert_1_0_to_1_1(self):
json_manifest = {
"version":
"1.0",
"files": [{
"name": "output_file",
"path": "/tmp/job_exe_231/outputs/output.csv"
}, {
"name":
"output_files",
"paths": [
"/tmp/job_exe_231/outputs/output.csv",
"/tmp/job_exe_231/outputs/output2.csv"
]
}],
"parse_results": [{
"filename": "myfile.h5",
"data_started": "2015-05-15T10:34:12Z",
"data_ended": "2015-05-15T10:36:12Z",
"data_types": ["H5", "VEG"]
}],
"errors": []
}
new_format = {
"version":
"1.1",
"output_data": [{
"name": "output_file",
"file": {
"path": "/tmp/job_exe_231/outputs/output.csv"
}
}, {
"name":
"output_files",
"files": [{
"path": "/tmp/job_exe_231/outputs/output.csv"
}, {
"path": "/tmp/job_exe_231/outputs/output2.csv"
}]
}],
"parse_results": [{
"filename": "myfile.h5",
"data_started": "2015-05-15T10:34:12Z",
"data_ended": "2015-05-15T10:36:12Z",
"data_types": ["H5", "VEG"]
}],
"errors": []
}
manifest = ResultsManifest(json_manifest)
converted = manifest._convert_schema(json_manifest)
self.assertEqual(converted, new_format)
def test_simple_validation(self):
json_manifest = {
"version": "1.0",
"files": [{
"name": "foo",
"path": "nfs:server//myfile.txt"
}]
}
input_files = {"input_file": False}
output_files = {"foo": (False, True)}
manifest = ResultsManifest(json_manifest)
manifest.validate(output_files)
def test_simple_validation(self):
json_manifest = {
"version": "1.0",
"files": [
{"name":"foo", "path":"nfs:server//myfile.txt"}
]
}
input_files = {
"input_file": False
}
output_files = {
"foo": (False, True)
}
manifest = ResultsManifest(json_manifest)
manifest.validate(output_files)
def test_missing_optional_is_ok(self):
json_manifest = {
"version": "1.0",
"files": [{
"name": "foo",
"path": "nfs:server//myfile.txt"
}]
}
input_files = {"input_file": False}
output_files = {
"foo": (False, True),
"bar": (False, False) #This is an optional file
}
manifest = ResultsManifest(json_manifest)
try:
manifest.validate(output_files)
except ResultsManifestAndInterfaceDontMatch:
self.fail(u'The missing an optional file')
def test_output_does_not_match(self):
json_manifest = {
"version": "1.0",
"files": [{
"name": "foo",
"path": "nfs:server//myfile.txt"
}]
}
input_files = {"input_file": False}
output_files = {"bar": (False, True)}
manifest = ResultsManifest(json_manifest)
try:
manifest.validate(output_files)
self.fail(
u'The outputs do not match the manifest, there should be a failure'
)
except ResultsManifestAndInterfaceDontMatch:
pass
def test_output_does_not_match(self):
json_manifest = {
"version": "1.0",
"files": [
{"name":"foo", "path":"nfs:server//myfile.txt"}
]
}
input_files = {
"input_file": False
}
output_files = {
"bar": (False, True)
}
manifest = ResultsManifest(json_manifest)
try:
manifest.validate(output_files)
self.fail(u'The outputs do not match the manifest, there should be a failure')
except ResultsManifestAndInterfaceDontMatch:
pass
def test_missing_optional_is_ok(self):
json_manifest = {
"version": "1.0",
"files": [
{"name":"foo", "path":"nfs:server//myfile.txt"}
]
}
input_files = {
"input_file": False
}
output_files = {
"foo": (False, True),
"bar": (False, False) #This is an optional file
}
manifest = ResultsManifest(json_manifest)
try:
manifest.validate(output_files)
except ResultsManifestAndInterfaceDontMatch:
self.fail(u'The missing an optional file')
def test_missing_required_is_bad(self):
json_manifest = {
"version": "1.0",
"files": [
{"name":"foo", "path":"nfs:server//myfile.txt"}
]
}
input_files = {
"input_file": False
}
output_files = {
"foo": (False, True),
"bar": (False, True) #This is a missing required file
}
manifest = ResultsManifest(json_manifest)
try:
manifest.validate(output_files)
self.fail(u'There is a missing required file. Validation should have failed')
except ResultsManifestAndInterfaceDontMatch:
pass
def test_missing_required_is_bad(self):
json_manifest = {
"version": "1.0",
"files": [
{"name":"foo", "path":"nfs:server//myfile.txt"}
]
}
input_files = {
"input_file": False
}
output_files = {
"foo": (False, True),
"bar": (False, True) #This is a missing required file
}
manifest = ResultsManifest(json_manifest)
try:
manifest.validate(output_files)
self.fail(u'There is a missing required file. Validation should have failed')
except MissingRequiredOutput:
pass
def test_manifest_supports_file_with_paths(self):
json_manifest = {
"version": "1.0",
"files": [
{"name":"foo", "paths":["nfs:server//myfile.txt"]}
]
}
try:
#This should not throw an exception since it is valid
ResultsManifest(json_manifest)
except InvalidResultsManifest:
self.fail(u'This simple json_manifest is valid')
def test_invalid_results_manifest(self):
json_manifest = {
"version": "1.0",
"files": [
{"name":"foo", "path":"nfs:server//myfile.txt", "paths": ["nfs:server//why_do_i_have_path_and_paths"]}
]
}
try:
ResultsManifest(json_manifest)
self.fail(u'files in a results manifest should not have both path and paths')
except InvalidResultsManifest:
#This should throw an exception since it is invalid
pass
def test_manifest_version_1_1(self):
json_manifest = {
"version":
"1.1",
"output_data": [{
"name": "output_file",
"file": {
"path": "/tmp/job_exe_231/outputs/output.csv",
"geo_metadata": {
"data_started": "2015-05-15T10:34:12Z",
"data_ended": "2015-05-15T10:36:12Z",
"geo_json": {
"type":
"Polygon",
"coordinates": [[[1.0, 10.0], [2.0, 10.0],
[2.0, 20.0], [1.0, 20.0],
[1.0, 10.0]]]
}
}
}
}, {
"name":
"output_files",
"files": [{
"path": "/tmp/job_exe_231/outputs/output.csv",
"geo_metadata": {
"data_started": "2015-05-15T10:34:12Z",
"data_ended": "2015-05-15T10:36:12Z",
"geo_json": {
"type":
"Polygon",
"coordinates": [[[1.0, 10.0], [2.0, 10.0],
[2.0, 20.0], [1.0, 20.0],
[1.0, 10.0]]]
}
}
}, {
"path": "/tmp/job_exe_231/outputs/output2.csv"
}]
}],
"parse_results": [{
"filename": "myfile.h5",
"data_started": "2015-05-15T10:34:12Z",
"data_ended": "2015-05-15T10:36:12Z",
"data_types": ["H5", "VEG"]
}]
}
manifest = ResultsManifest(json_manifest)
def test_empty_results_manifest(self):
json_manifest = {}
#This should not throw an exception since it is valid
ResultsManifest(json_manifest)
def perform_post_steps(self, job_exe, job_data, stdoutAndStderr):
"""Stores the files and deletes any working directories
:param job_exe: The job execution model with related job and job_type fields
:type job_exe: :class:`job.models.JobExecution`
:param job_data: The job data
:type job_data: :class:`job.configuration.data.job_data.JobData`
:param stdoutAndStderr: the standard out from the job execution
:type stdoutAndStderr: str
:return: A tuple of the job results and the results manifest generated by the job execution
:rtype: (:class:`job.configuration.results.job_results.JobResults`,
:class:`job.configuration.results.results_manifest.results_manifest.ResultsManifest`)
"""
manifest_data = {}
path_to_manifest_file = os.path.join(SCALE_JOB_EXE_OUTPUT_PATH,
'results_manifest.json')
if os.path.exists(path_to_manifest_file):
logger.info('Opening results manifest...')
with open(path_to_manifest_file, 'r') as manifest_file:
manifest_data = json.loads(manifest_file.read())
logger.info('Results manifest:')
logger.info(manifest_data)
else:
logger.info('No results manifest found')
results_manifest = ResultsManifest(manifest_data)
stdout_files = self._get_artifacts_from_stdout(stdoutAndStderr)
results_manifest.add_files(stdout_files)
results_manifest.validate(self._output_file_manifest_dict)
files_to_store = {}
for manifest_file_entry in results_manifest.get_files():
param_name = manifest_file_entry['name']
media_type = None
output_data_item = self._get_output_data_item_by_name(param_name)
if output_data_item:
media_type = output_data_item.get('media_type')
msg = 'Output %s has invalid/missing file path "%s"'
if 'file' in manifest_file_entry:
file_entry = manifest_file_entry['file']
if not os.path.isfile(file_entry['path']):
raise InvalidResultsManifest(
msg % (param_name, file_entry['path']))
if 'geo_metadata' in file_entry:
files_to_store[param_name] = (file_entry['path'],
media_type,
file_entry['geo_metadata'])
else:
files_to_store[param_name] = (file_entry['path'],
media_type)
elif 'files' in manifest_file_entry:
file_tuples = []
for file_entry in manifest_file_entry['files']:
if not os.path.isfile(file_entry['path']):
raise InvalidResultsManifest(
msg % (param_name, file_entry['path']))
if 'geo_metadata' in file_entry:
file_tuples.append((file_entry['path'], media_type,
file_entry['geo_metadata']))
else:
file_tuples.append((file_entry['path'], media_type))
files_to_store[param_name] = file_tuples
job_data_parse_results = {} # parse results formatted for job_data
for parse_result in results_manifest.get_parse_results():
filename = parse_result['filename']
assert filename not in job_data_parse_results
geo_metadata = parse_result.get('geo_metadata', {})
geo_json = geo_metadata.get('geo_json', None)
data_started = geo_metadata.get('data_started', None)
data_ended = geo_metadata.get('data_ended', None)
data_types = parse_result.get('data_types', [])
new_workspace_path = parse_result.get('new_workspace_path', None)
if new_workspace_path:
new_workspace_path = os.path.join(new_workspace_path, filename)
job_data_parse_results[filename] = (geo_json, data_started,
data_ended, data_types,
new_workspace_path)
job_data.save_parse_results(job_data_parse_results)
return (job_data.store_output_data_files(files_to_store,
job_exe), results_manifest)
from django.db.utils import DatabaseError, OperationalError
from django.utils.timezone import now
from django.test import TransactionTestCase
from mock import patch
from error.exceptions import ScaleDatabaseError, ScaleIOError, ScaleOperationalError
from job.configuration.results.exceptions import InvalidResultsManifest, MissingRequiredOutput
from job.configuration.results.job_results import JobResults
from job.configuration.results.results_manifest.results_manifest import ResultsManifest
from job.management.commands.scale_post_steps import Command as PostCommand
from job.models import JobExecutionOutput
from job.test import utils as job_utils
from trigger.models import TriggerEvent
JOB_RESULTS = JobResults()
RESULTS_MANIFEST = ResultsManifest()
RESULTS = (JOB_RESULTS, RESULTS_MANIFEST)
class TestPostJobSteps(TransactionTestCase):
def setUp(self):
django.setup()
cmd = 'command'
cmd_args = 'args'
interface = {
'version':
'1.0',
'command':
cmd,
'command_arguments':
def perform_post_steps(self, job_exe, job_data, stdoutAndStderr):
"""Stores the files and deletes any working directories
:param job_exe: The job execution model with related job and job_type fields
:type job_exe: :class:`job.models.JobExecution`
:param job_data: The job data
:type job_data: :class:`job.configuration.data.job_data.JobData`
:param stdoutAndStderr: the standard out from the job execution
:type stdoutAndStderr: str
:return: A tuple of the job results and the results manifest generated by the job execution
:rtype: (:class:`job.configuration.results.job_results.JobResults`,
:class:`job.configuration.results.results_manifest.results_manifest.ResultsManifest`)
"""
manifest_data = {}
path_to_manifest_file = os.path.join(SCALE_JOB_EXE_OUTPUT_PATH, 'results_manifest.json')
if os.path.exists(path_to_manifest_file):
logger.info('Opening results manifest...')
with open(path_to_manifest_file, 'r') as manifest_file:
manifest_data = json.loads(manifest_file.read())
logger.info('Results manifest:')
logger.info(manifest_data)
else:
logger.info('No results manifest found')
results_manifest = ResultsManifest(manifest_data)
stdout_files = self._get_artifacts_from_stdout(stdoutAndStderr)
results_manifest.add_files(stdout_files)
results_manifest.validate(self._output_file_manifest_dict)
files_to_store = {}
for manifest_file_entry in results_manifest.get_files():
param_name = manifest_file_entry['name']
media_type = None
output_data_item = self._get_output_data_item_by_name(param_name)
if output_data_item:
media_type = output_data_item.get('media_type')
if 'file' in manifest_file_entry:
file_entry = manifest_file_entry['file']
if 'geo_metadata' in file_entry:
files_to_store[param_name] = (file_entry['path'], media_type, file_entry['geo_metadata'])
else:
files_to_store[param_name] = (file_entry['path'], media_type)
elif 'files' in manifest_file_entry:
file_tuples = []
for file_entry in manifest_file_entry['files']:
if 'geo_metadata' in file_entry:
file_tuples.append((file_entry['path'], media_type, file_entry['geo_metadata']))
else:
file_tuples.append((file_entry['path'], media_type))
files_to_store[param_name] = file_tuples
job_data_parse_results = {} # parse results formatted for job_data
for parse_result in results_manifest.get_parse_results():
filename = parse_result['filename']
assert filename not in job_data_parse_results
geo_metadata = parse_result.get('geo_metadata', {})
geo_json = geo_metadata.get('geo_json', None)
data_started = geo_metadata.get('data_started', None)
data_ended = geo_metadata.get('data_ended', None)
data_types = parse_result.get('data_types', [])
new_workspace_path = parse_result.get('new_workspace_path', None)
if new_workspace_path:
new_workspace_path = os.path.join(new_workspace_path, filename)
job_data_parse_results[filename] = (geo_json, data_started, data_ended, data_types, new_workspace_path)
job_data.save_parse_results(job_data_parse_results)
return (job_data.store_output_data_files(files_to_store, job_exe), results_manifest)
