def test_output_file(self, mock_loads, mock_open, mock_exists, mock_isfile):
job_interface_dict, job_data_dict = self._get_simple_interface_data()
job_interface_dict['output_data'] = [{
'name': 'output_file',
'type': 'file',
'required': True,
}]
job_data_dict['output_data'].append({
'name': 'output_file',
'workspace_id': self.workspace.id,
})
results_manifest = {
'version': '1.0',
'files': [{
'name': 'output_file',
'path': '/some/path/foo.txt',
}]
}
mock_loads.return_value = results_manifest
mock_exists.return_value = True
mock_isfile.return_value = True
job_exe = MagicMock()
job_interface = JobInterface(job_interface_dict)
job_data = Mock(spec=JobData)
job_data.save_parse_results = Mock()
fake_stdout = ''
job_interface.perform_post_steps(job_exe, job_data, fake_stdout)
job_data.store_output_data_files.assert_called_with({
'output_file': ('/some/path/foo.txt', None),
}, job_exe)
def test_extra_products_are_fine(self, mock_loads, mock_open, mock_exists):
job_interface_dict, job_data_dict= self._get_simple_interface_data()
job_interface_dict[u'input_data'] = [{"name":"input_file", "type":"file", "required": True}]
job_interface_dict[u'output_data'] = [{"name":"output_file", "type":"file", "required": True}]
job_data_dict[u'input_data'].append({"name":"input_file", "file_id":1})
results_manifest={"version" : "1.0", "files" : [{"name":"output_file", "path":"/new/path/foo.txt"}]}
mock_loads.return_value = results_manifest
mock_exists.return_value = True
job_exe = MagicMock()
job_interface = JobInterface(job_interface_dict)
job_data = Mock(spec=JobData)
job_data.save_parse_results = Mock()
fake_stdout = '''
This text is supposed to mimick the output
of a program we should see artifacts registered with
the format: ARTIFACT:<input-name>:path, but it needs be at the beginning of a line
so the example above won't match, but this will
ARTIFACT:output_file:/path/to/foo.txt
We should also be able to have text after the artifact and multiple artifacts.
ARTIFACT:output_file_2:/path/to/foo_2.txt
'''
job_interface.perform_post_steps(job_exe, job_data, fake_stdout)
job_data.store_output_data_files.assert_called_with({u'output_file': (u'/new/path/foo.txt', None)}, job_exe)
def test_output_files(self, mock_loads, mock_open, mock_exists):
job_interface_dict, job_data_dict = self._get_simple_interface_data()
job_interface_dict['output_data'] = [{
'name': 'output_files',
'type': 'files',
'required': True,
}]
job_data_dict['output_data'].append({
'name': 'output_files',
'workspace_id': self.workspace.id,
})
results_manifest = {
'version': '1.0',
'files': [{
'name':'output_files',
'paths': ['/some/path/foo.txt', '/other/path/foo.txt'],
}]
}
mock_loads.return_value = results_manifest
mock_exists.return_value = True
job_exe = MagicMock()
job_interface = JobInterface(job_interface_dict)
job_data = Mock(spec=JobData)
job_data.save_parse_results = Mock()
fake_stdout = ''
job_interface.perform_post_steps(job_exe, job_data, fake_stdout)
job_data.store_output_data_files.assert_called_with({
'output_files': [
('/some/path/foo.txt', None),
('/other/path/foo.txt', None),
]
}, job_exe)
def test_output_files_with_geo_metadata(self, mock_loads, mock_open,
mock_exists, mock_isfile):
job_interface_dict, job_data_dict = self._get_simple_interface_data()
job_interface_dict['output_data'] = [{
'name': 'output_files',
'type': 'files',
'required': True,
}]
job_data_dict['output_data'].append({
'name': 'output_files',
'workspace_id': self.workspace.id,
})
geo_metadata = {
'data_started': '2015-05-15T10:34:12Z',
'data_ended': '2015-05-15T10:36:12Z',
'geo_json': {
'type':
'Polygon',
'coordinates': [[[1.0, 10.0], [2.0, 10.0], [2.0, 20.0],
[1.0, 20.0], [1.0, 10.0]]],
}
}
results_manifest = {
'version':
'1.1',
'output_data': [{
'name':
'output_files',
'files': [{
'path': '/some/path/foo.txt',
'geo_metadata': geo_metadata,
}, {
'path': '/other/path/foo.txt',
'geo_metadata': geo_metadata,
}]
}]
}
mock_loads.return_value = results_manifest
mock_exists.return_value = True
mock_isfile.return_value = True
job_exe = MagicMock()
job_interface = JobInterface(job_interface_dict)
job_data = Mock(spec=JobData)
job_data.save_parse_results = Mock()
fake_stdout = ''
job_interface.perform_post_steps(job_exe, job_data, fake_stdout)
job_data.store_output_data_files.assert_called_with(
{
'output_files': [
('/some/path/foo.txt', None, geo_metadata),
('/other/path/foo.txt', None, geo_metadata),
]
}, job_exe)
def test_manifest_overrides_stdout(self, mock_loads, mock_open,
mock_exists, mock_isfile):
job_interface_dict, job_data_dict = self._get_simple_interface_data()
job_interface_dict['input_data'] = [{
'name': 'input_file',
'type': 'file',
'required': True,
}]
job_interface_dict['output_data'] = [{
'name': 'output_file',
'type': 'file',
'required': True,
}, {
'name': 'output_file_2',
'type': 'file',
'required': True,
}]
job_data_dict['input_data'].append({
'name': 'input_file',
'file_id': self.file.id,
})
results_manifest = {
'version': '1.0',
'files': [{
'name': 'output_file',
'path': '/new/path/foo.txt',
}]
}
mock_loads.return_value = results_manifest
mock_exists.return_value = True
mock_isfile.return_value = True
job_exe = MagicMock()
job_interface = JobInterface(job_interface_dict)
job_data = Mock(spec=JobData)
job_data.save_parse_results = Mock()
fake_stdout = """
This text is supposed to mimic the output
of a program we should see artifacts registered with
the format: ARTIFACT:<input-name>:path, but it needs be at the beginning of a line
so the example above won't match, but this will
ARTIFACT:output_file:/path/to/foo.txt
We should also be able to have text after the artifact and multiple artifacts.
ARTIFACT:output_file_2:/path/to/foo_2.txt
"""
job_interface.perform_post_steps(job_exe, job_data, fake_stdout)
job_data.store_output_data_files.assert_called_with(
{
'output_file': ('/new/path/foo.txt', None),
'output_file_2': ('/path/to/foo_2.txt', None),
}, job_exe)
def test_manifest_overrides_stdout(self, mock_loads, mock_open, mock_exists, mock_isfile):
job_interface_dict, job_data_dict = self._get_simple_interface_data()
job_interface_dict['input_data'] = [{
'name': 'input_file',
'type': 'file',
'required': True,
}]
job_interface_dict['output_data'] = [{
'name': 'output_file',
'type': 'file',
'required': True,
}, {
'name': 'output_file_2',
'type':'file',
'required': True,
}]
job_data_dict['input_data'].append({
'name': 'input_file',
'file_id': self.file.id,
})
results_manifest = {
'version': '1.0',
'files': [{
'name': 'output_file',
'path': '/new/path/foo.txt',
}]
}
mock_loads.return_value = results_manifest
mock_exists.return_value = True
mock_isfile.return_value = True
job_exe = MagicMock()
job_interface = JobInterface(job_interface_dict)
job_data = Mock(spec=JobData)
job_data.save_parse_results = Mock()
fake_stdout = """
This text is supposed to mimic the output
of a program we should see artifacts registered with
the format: ARTIFACT:<input-name>:path, but it needs be at the beginning of a line
so the example above won't match, but this will
ARTIFACT:output_file:/path/to/foo.txt
We should also be able to have text after the artifact and multiple artifacts.
ARTIFACT:output_file_2:/path/to/foo_2.txt
"""
job_interface.perform_post_steps(job_exe, job_data, fake_stdout)
job_data.store_output_data_files.assert_called_with({
'output_file': ('/new/path/foo.txt', None),
'output_file_2': ('/path/to/foo_2.txt', None),
}, job_exe)
def test_output_files_with_geo_metadata(self, mock_loads, mock_open, mock_exists, mock_isfile):
job_interface_dict, job_data_dict = self._get_simple_interface_data()
job_interface_dict['output_data'] = [{
'name': 'output_files',
'type': 'files',
'required': True,
}]
job_data_dict['output_data'].append({
'name': 'output_files',
'workspace_id': self.workspace.id,
})
geo_metadata = {
'data_started': '2015-05-15T10:34:12Z',
'data_ended': '2015-05-15T10:36:12Z',
'geo_json': {
'type': 'Polygon',
'coordinates': [[[1.0, 10.0], [2.0, 10.0], [2.0, 20.0], [1.0, 20.0], [1.0, 10.0]]],
}
}
results_manifest = {
'version': '1.1',
'output_data': [{
'name': 'output_files',
'files': [{
'path': '/some/path/foo.txt',
'geo_metadata': geo_metadata,
}, {
'path': '/other/path/foo.txt',
'geo_metadata': geo_metadata,
}]
}]
}
mock_loads.return_value = results_manifest
mock_exists.return_value = True
mock_isfile.return_value = True
job_exe = MagicMock()
job_interface = JobInterface(job_interface_dict)
job_data = Mock(spec=JobData)
job_data.save_parse_results = Mock()
fake_stdout = ''
job_interface.perform_post_steps(job_exe, job_data, fake_stdout)
job_data.store_output_data_files.assert_called_with({
'output_files': [
('/some/path/foo.txt', None, geo_metadata),
('/other/path/foo.txt', None, geo_metadata),
]
}, job_exe)
def test_output_files(self, mock_loads, mock_open, mock_exists):
job_interface_dict, job_data_dict= self._get_simple_interface_data()
job_interface_dict[u'output_data'] = [{"name":"output_files", "type":"files", "required": True}]
job_data_dict[u'output_data'].append({"name":"output_files", "workspace_id":1})
results_manifest={"version" : "1.0", "files" : [{"name":"output_files", "paths": ["/some/path/foo.txt", "/other/path/foo.txt"]}]}
mock_loads.return_value = results_manifest
mock_exists.return_value = True
job_exe = MagicMock()
job_interface = JobInterface(job_interface_dict)
job_data = Mock(spec=JobData)
job_data.save_parse_results = Mock()
fake_stdout = u''
job_interface.perform_post_steps(job_exe, job_data, fake_stdout)
job_data.store_output_data_files.assert_called_with({u'output_files': [(u'/some/path/foo.txt', None),(u'/other/path/foo.txt', None)]}, job_exe)
def test_output_files_with_geo_metadata(self, mock_loads, mock_open, mock_exists):
job_interface_dict, job_data_dict= self._get_simple_interface_data()
job_interface_dict[u'output_data'] = [{"name":"output_files", "type":"files", "required": True}]
job_data_dict[u'output_data'].append({"name":"output_files", "workspace_id":1})
geo_metadata = {
"data_started": "2015-05-15T10:34:12Z",
"data_ended": "2015-05-15T10:36:12Z",
"geo_json": {
"type": "Polygon",
"coordinates": [[[ 1.0, 10.0 ], [ 2.0, 10.0 ], [ 2.0, 20.0 ],[ 1.0, 20.0 ], [ 1.0, 10.0 ]]]
}
}
results_manifest = {
"version": "1.1",
"output_data": [
{
"name" : "output_files",
"files": [
{
"path" : "/some/path/foo.txt",
"geo_metadata": geo_metadata
},
{
"path" : "/other/path/foo.txt",
"geo_metadata": geo_metadata
}
]
}
]
}
mock_loads.return_value = results_manifest
mock_exists.return_value = True
job_exe = MagicMock()
job_interface = JobInterface(job_interface_dict)
job_data = Mock(spec=JobData)
job_data.save_parse_results = Mock()
fake_stdout = u''
job_interface.perform_post_steps(job_exe, job_data, fake_stdout)
job_data.store_output_data_files.assert_called_with(
{u'output_files': [(u'/some/path/foo.txt', None, geo_metadata),(u'/other/path/foo.txt', None, geo_metadata)]}, job_exe)
def test_parse_data(self, mock_loads, mock_open, mock_exists):
job_interface_dict, job_data_dict= self._get_simple_interface_data()
job_interface_dict[u'input_data'] = [{"name":"input_file", "type":"file", "required": True}]
job_data_dict[u'input_data'].append({"name":"input_file", "file_id":1})
geo_json = {"type": 'Feature'}
geo_metadata = {
"data_started": '2015-01-01T00:00:00Z',
"geo_json": geo_json
}
results_manifest={"version" : "1.1", "parse_results" : [{"filename":"/some/path/foo.txt", "geo_metadata":geo_metadata}]}
mock_loads.return_value = results_manifest
mock_exists.return_value = True
job_exe = MagicMock()
job_interface = JobInterface(job_interface_dict)
job_data = Mock(spec=JobData)
job_data.save_parse_results = Mock()
fake_stdout = u''
job_interface.perform_post_steps(job_exe, job_data, fake_stdout)
job_data.save_parse_results.assert_called_with({u'/some/path/foo.txt': (geo_json, '2015-01-01T00:00:00Z', None, [], None, None)})
def test_parse_data(self, mock_loads, mock_open, mock_exists):
job_interface_dict, job_data_dict = self._get_simple_interface_data()
job_interface_dict['input_data'] = [{
'name': 'input_file',
'type': 'file',
'required': True,
}]
job_data_dict['input_data'].append({
'name': 'input_file',
'file_id': self.file.id,
})
geo_json = {'type': 'Feature'}
geo_metadata = {
'data_started': '2015-01-01T00:00:00Z',
'geo_json': geo_json
}
results_manifest = {
'version':
'1.1',
'parse_results': [{
'filename': '/some/path/foo.txt',
'geo_metadata': geo_metadata,
}]
}
mock_loads.return_value = results_manifest
mock_exists.return_value = True
job_exe = MagicMock()
job_interface = JobInterface(job_interface_dict)
job_data = Mock(spec=JobData)
job_data.save_parse_results = Mock()
fake_stdout = ''
job_interface.perform_post_steps(job_exe, job_data, fake_stdout)
job_data.save_parse_results.assert_called_with({
'/some/path/foo.txt':
(geo_json, '2015-01-01T00:00:00Z', None, [], None),
})
def test_parse_data(self, mock_loads, mock_open, mock_exists):
job_interface_dict, job_data_dict = self._get_simple_interface_data()
job_interface_dict['input_data'] = [{
'name': 'input_file',
'type': 'file',
'required': True,
}]
job_data_dict['input_data'].append({
'name': 'input_file',
'file_id': self.file.id,
})
geo_json = {'type': 'Feature'}
geo_metadata = {
'data_started': '2015-01-01T00:00:00Z',
'geo_json': geo_json
}
results_manifest = {
'version': '1.1',
'parse_results': [{
'filename': '/some/path/foo.txt',
'geo_metadata': geo_metadata,
}]
}
mock_loads.return_value = results_manifest
mock_exists.return_value = True
job_exe = MagicMock()
job_interface = JobInterface(job_interface_dict)
job_data = Mock(spec=JobData)
job_data.save_parse_results = Mock()
fake_stdout = ''
job_interface.perform_post_steps(job_exe, job_data, fake_stdout)
job_data.save_parse_results.assert_called_with({
'/some/path/foo.txt': (geo_json, '2015-01-01T00:00:00Z', None, [], None, None),
})
