def test_simple_validation_1_1(self):
json_manifest = {
"version":
"1.1",
"output_data": [{
"name": "output_file",
"file": {
"path": "/tmp/job_exe_231/outputs/output.csv",
"geo_metadata": {
"data_started": "2015-05-15T10:34:12Z",
"data_ended": "2015-05-15T10:36:12Z",
"geo_json": {
"type":
"Polygon",
"coordinates": [[[1.0, 10.0], [2.0, 10.0],
[2.0, 20.0], [1.0, 20.0],
[1.0, 10.0]]]
}
}
}
}]
}
input_files = {"input_file": False}
output_files = {"output_file": (False, True)}
manifest = ResultsManifest(json_manifest)
manifest.validate(output_files)
def test_simple_validation_1_1(self):
json_manifest = {
"version": "1.1",
"output_data": [
{
"name" : "output_file",
"file": {
"path" : "/tmp/job_exe_231/outputs/output.csv",
"geo_metadata": {
"data_started": "2015-05-15T10:34:12Z",
"data_ended" : "2015-05-15T10:36:12Z",
"geo_json": {
"type": "Polygon",
"coordinates": [ [ [ 1.0, 10.0 ], [ 2.0, 10.0 ], [ 2.0, 20.0 ],[ 1.0, 20.0 ], [ 1.0, 10.0 ] ] ]
}
}
}
}
]
}
input_files = {
"input_file": False
}
output_files = {
"output_file": (False, True)
}
manifest = ResultsManifest(json_manifest)
manifest.validate(output_files)
def test_simple_validation(self):
json_manifest = {
"version": "1.0",
"files": [{
"name": "foo",
"path": "nfs:server//myfile.txt"
}]
}
input_files = {"input_file": False}
output_files = {"foo": (False, True)}
manifest = ResultsManifest(json_manifest)
manifest.validate(output_files)
def test_simple_validation(self):
json_manifest = {
"version": "1.0",
"files": [
{"name":"foo", "path":"nfs:server//myfile.txt"}
]
}
input_files = {
"input_file": False
}
output_files = {
"foo": (False, True)
}
manifest = ResultsManifest(json_manifest)
manifest.validate(output_files)
def test_missing_optional_is_ok(self):
json_manifest = {
"version": "1.0",
"files": [{
"name": "foo",
"path": "nfs:server//myfile.txt"
}]
}
input_files = {"input_file": False}
output_files = {
"foo": (False, True),
"bar": (False, False) #This is an optional file
}
manifest = ResultsManifest(json_manifest)
try:
manifest.validate(output_files)
except ResultsManifestAndInterfaceDontMatch:
self.fail(u'The missing an optional file')
def test_output_does_not_match(self):
json_manifest = {
"version": "1.0",
"files": [{
"name": "foo",
"path": "nfs:server//myfile.txt"
}]
}
input_files = {"input_file": False}
output_files = {"bar": (False, True)}
manifest = ResultsManifest(json_manifest)
try:
manifest.validate(output_files)
self.fail(
u'The outputs do not match the manifest, there should be a failure'
)
except ResultsManifestAndInterfaceDontMatch:
pass
def test_missing_optional_is_ok(self):
json_manifest = {
"version": "1.0",
"files": [
{"name":"foo", "path":"nfs:server//myfile.txt"}
]
}
input_files = {
"input_file": False
}
output_files = {
"foo": (False, True),
"bar": (False, False) #This is an optional file
}
manifest = ResultsManifest(json_manifest)
try:
manifest.validate(output_files)
except ResultsManifestAndInterfaceDontMatch:
self.fail(u'The missing an optional file')
def test_output_does_not_match(self):
json_manifest = {
"version": "1.0",
"files": [
{"name":"foo", "path":"nfs:server//myfile.txt"}
]
}
input_files = {
"input_file": False
}
output_files = {
"bar": (False, True)
}
manifest = ResultsManifest(json_manifest)
try:
manifest.validate(output_files)
self.fail(u'The outputs do not match the manifest, there should be a failure')
except ResultsManifestAndInterfaceDontMatch:
pass
def test_missing_required_is_bad(self):
json_manifest = {
"version": "1.0",
"files": [
{"name":"foo", "path":"nfs:server//myfile.txt"}
]
}
input_files = {
"input_file": False
}
output_files = {
"foo": (False, True),
"bar": (False, True) #This is a missing required file
}
manifest = ResultsManifest(json_manifest)
try:
manifest.validate(output_files)
self.fail(u'There is a missing required file. Validation should have failed')
except ResultsManifestAndInterfaceDontMatch:
pass
def test_missing_required_is_bad(self):
json_manifest = {
"version": "1.0",
"files": [
{"name":"foo", "path":"nfs:server//myfile.txt"}
]
}
input_files = {
"input_file": False
}
output_files = {
"foo": (False, True),
"bar": (False, True) #This is a missing required file
}
manifest = ResultsManifest(json_manifest)
try:
manifest.validate(output_files)
self.fail(u'There is a missing required file. Validation should have failed')
except MissingRequiredOutput:
pass
def perform_post_steps(self, job_exe, job_data, stdoutAndStderr):
"""Stores the files and deletes any working directories
:param job_exe: The job execution model with related job and job_type fields
:type job_exe: :class:`job.models.JobExecution`
:param job_data: The job data
:type job_data: :class:`job.configuration.data.job_data.JobData`
:param stdoutAndStderr: the standard out from the job execution
:type stdoutAndStderr: str
:return: A tuple of the job results and the results manifest generated by the job execution
:rtype: (:class:`job.configuration.results.job_results.JobResults`,
:class:`job.configuration.results.results_manifest.results_manifest.ResultsManifest`)
"""
manifest_data = {}
path_to_manifest_file = os.path.join(SCALE_JOB_EXE_OUTPUT_PATH,
'results_manifest.json')
if os.path.exists(path_to_manifest_file):
logger.info('Opening results manifest...')
with open(path_to_manifest_file, 'r') as manifest_file:
manifest_data = json.loads(manifest_file.read())
logger.info('Results manifest:')
logger.info(manifest_data)
else:
logger.info('No results manifest found')
results_manifest = ResultsManifest(manifest_data)
stdout_files = self._get_artifacts_from_stdout(stdoutAndStderr)
results_manifest.add_files(stdout_files)
results_manifest.validate(self._output_file_manifest_dict)
files_to_store = {}
for manifest_file_entry in results_manifest.get_files():
param_name = manifest_file_entry['name']
media_type = None
output_data_item = self._get_output_data_item_by_name(param_name)
if output_data_item:
media_type = output_data_item.get('media_type')
msg = 'Output %s has invalid/missing file path "%s"'
if 'file' in manifest_file_entry:
file_entry = manifest_file_entry['file']
if not os.path.isfile(file_entry['path']):
raise InvalidResultsManifest(
msg % (param_name, file_entry['path']))
if 'geo_metadata' in file_entry:
files_to_store[param_name] = (file_entry['path'],
media_type,
file_entry['geo_metadata'])
else:
files_to_store[param_name] = (file_entry['path'],
media_type)
elif 'files' in manifest_file_entry:
file_tuples = []
for file_entry in manifest_file_entry['files']:
if not os.path.isfile(file_entry['path']):
raise InvalidResultsManifest(
msg % (param_name, file_entry['path']))
if 'geo_metadata' in file_entry:
file_tuples.append((file_entry['path'], media_type,
file_entry['geo_metadata']))
else:
file_tuples.append((file_entry['path'], media_type))
files_to_store[param_name] = file_tuples
job_data_parse_results = {} # parse results formatted for job_data
for parse_result in results_manifest.get_parse_results():
filename = parse_result['filename']
assert filename not in job_data_parse_results
geo_metadata = parse_result.get('geo_metadata', {})
geo_json = geo_metadata.get('geo_json', None)
data_started = geo_metadata.get('data_started', None)
data_ended = geo_metadata.get('data_ended', None)
data_types = parse_result.get('data_types', [])
new_workspace_path = parse_result.get('new_workspace_path', None)
if new_workspace_path:
new_workspace_path = os.path.join(new_workspace_path, filename)
job_data_parse_results[filename] = (geo_json, data_started,
data_ended, data_types,
new_workspace_path)
job_data.save_parse_results(job_data_parse_results)
return (job_data.store_output_data_files(files_to_store,
job_exe), results_manifest)
def perform_post_steps(self, job_exe, job_data, stdoutAndStderr):
"""Stores the files and deletes any working directories
:param job_exe: The job execution model with related job and job_type fields
:type job_exe: :class:`job.models.JobExecution`
:param job_data: The job data
:type job_data: :class:`job.configuration.data.job_data.JobData`
:param stdoutAndStderr: the standard out from the job execution
:type stdoutAndStderr: str
:return: A tuple of the job results and the results manifest generated by the job execution
:rtype: (:class:`job.configuration.results.job_results.JobResults`,
:class:`job.configuration.results.results_manifest.results_manifest.ResultsManifest`)
"""
manifest_data = {}
path_to_manifest_file = os.path.join(SCALE_JOB_EXE_OUTPUT_PATH, 'results_manifest.json')
if os.path.exists(path_to_manifest_file):
logger.info('Opening results manifest...')
with open(path_to_manifest_file, 'r') as manifest_file:
manifest_data = json.loads(manifest_file.read())
logger.info('Results manifest:')
logger.info(manifest_data)
else:
logger.info('No results manifest found')
results_manifest = ResultsManifest(manifest_data)
stdout_files = self._get_artifacts_from_stdout(stdoutAndStderr)
results_manifest.add_files(stdout_files)
results_manifest.validate(self._output_file_manifest_dict)
files_to_store = {}
for manifest_file_entry in results_manifest.get_files():
param_name = manifest_file_entry['name']
media_type = None
output_data_item = self._get_output_data_item_by_name(param_name)
if output_data_item:
media_type = output_data_item.get('media_type')
if 'file' in manifest_file_entry:
file_entry = manifest_file_entry['file']
if 'geo_metadata' in file_entry:
files_to_store[param_name] = (file_entry['path'], media_type, file_entry['geo_metadata'])
else:
files_to_store[param_name] = (file_entry['path'], media_type)
elif 'files' in manifest_file_entry:
file_tuples = []
for file_entry in manifest_file_entry['files']:
if 'geo_metadata' in file_entry:
file_tuples.append((file_entry['path'], media_type, file_entry['geo_metadata']))
else:
file_tuples.append((file_entry['path'], media_type))
files_to_store[param_name] = file_tuples
job_data_parse_results = {} # parse results formatted for job_data
for parse_result in results_manifest.get_parse_results():
filename = parse_result['filename']
assert filename not in job_data_parse_results
geo_metadata = parse_result.get('geo_metadata', {})
geo_json = geo_metadata.get('geo_json', None)
data_started = geo_metadata.get('data_started', None)
data_ended = geo_metadata.get('data_ended', None)
data_types = parse_result.get('data_types', [])
new_workspace_path = parse_result.get('new_workspace_path', None)
if new_workspace_path:
new_workspace_path = os.path.join(new_workspace_path, filename)
job_data_parse_results[filename] = (geo_json, data_started, data_ended, data_types, new_workspace_path)
job_data.save_parse_results(job_data_parse_results)
return (job_data.store_output_data_files(files_to_store, job_exe), results_manifest)