def test_output_file(self, mock_loads, mock_open, mock_exists, mock_isfile): job_interface_dict, job_data_dict = self._get_simple_interface_data() job_interface_dict['output_data'] = [{ 'name': 'output_file', 'type': 'file', 'required': True, }] job_data_dict['output_data'].append({ 'name': 'output_file', 'workspace_id': self.workspace.id, }) results_manifest = { 'version': '1.0', 'files': [{ 'name': 'output_file', 'path': '/some/path/foo.txt', }] } mock_loads.return_value = results_manifest mock_exists.return_value = True mock_isfile.return_value = True job_exe = MagicMock() job_interface = JobInterface(job_interface_dict) job_data = Mock(spec=JobData) job_data.save_parse_results = Mock() fake_stdout = '' job_interface.perform_post_steps(job_exe, job_data, fake_stdout) job_data.store_output_data_files.assert_called_with({ 'output_file': ('/some/path/foo.txt', None), }, job_exe)
def test_extra_products_are_fine(self, mock_loads, mock_open, mock_exists): job_interface_dict, job_data_dict= self._get_simple_interface_data() job_interface_dict[u'input_data'] = [{"name":"input_file", "type":"file", "required": True}] job_interface_dict[u'output_data'] = [{"name":"output_file", "type":"file", "required": True}] job_data_dict[u'input_data'].append({"name":"input_file", "file_id":1}) results_manifest={"version" : "1.0", "files" : [{"name":"output_file", "path":"/new/path/foo.txt"}]} mock_loads.return_value = results_manifest mock_exists.return_value = True job_exe = MagicMock() job_interface = JobInterface(job_interface_dict) job_data = Mock(spec=JobData) job_data.save_parse_results = Mock() fake_stdout = ''' This text is supposed to mimick the output of a program we should see artifacts registered with the format: ARTIFACT:<input-name>:path, but it needs be at the beginning of a line so the example above won't match, but this will ARTIFACT:output_file:/path/to/foo.txt We should also be able to have text after the artifact and multiple artifacts. ARTIFACT:output_file_2:/path/to/foo_2.txt ''' job_interface.perform_post_steps(job_exe, job_data, fake_stdout) job_data.store_output_data_files.assert_called_with({u'output_file': (u'/new/path/foo.txt', None)}, job_exe)
def test_output_files(self, mock_loads, mock_open, mock_exists): job_interface_dict, job_data_dict = self._get_simple_interface_data() job_interface_dict['output_data'] = [{ 'name': 'output_files', 'type': 'files', 'required': True, }] job_data_dict['output_data'].append({ 'name': 'output_files', 'workspace_id': self.workspace.id, }) results_manifest = { 'version': '1.0', 'files': [{ 'name':'output_files', 'paths': ['/some/path/foo.txt', '/other/path/foo.txt'], }] } mock_loads.return_value = results_manifest mock_exists.return_value = True job_exe = MagicMock() job_interface = JobInterface(job_interface_dict) job_data = Mock(spec=JobData) job_data.save_parse_results = Mock() fake_stdout = '' job_interface.perform_post_steps(job_exe, job_data, fake_stdout) job_data.store_output_data_files.assert_called_with({ 'output_files': [ ('/some/path/foo.txt', None), ('/other/path/foo.txt', None), ] }, job_exe)
def test_output_files_with_geo_metadata(self, mock_loads, mock_open, mock_exists, mock_isfile): job_interface_dict, job_data_dict = self._get_simple_interface_data() job_interface_dict['output_data'] = [{ 'name': 'output_files', 'type': 'files', 'required': True, }] job_data_dict['output_data'].append({ 'name': 'output_files', 'workspace_id': self.workspace.id, }) geo_metadata = { 'data_started': '2015-05-15T10:34:12Z', 'data_ended': '2015-05-15T10:36:12Z', 'geo_json': { 'type': 'Polygon', 'coordinates': [[[1.0, 10.0], [2.0, 10.0], [2.0, 20.0], [1.0, 20.0], [1.0, 10.0]]], } } results_manifest = { 'version': '1.1', 'output_data': [{ 'name': 'output_files', 'files': [{ 'path': '/some/path/foo.txt', 'geo_metadata': geo_metadata, }, { 'path': '/other/path/foo.txt', 'geo_metadata': geo_metadata, }] }] } mock_loads.return_value = results_manifest mock_exists.return_value = True mock_isfile.return_value = True job_exe = MagicMock() job_interface = JobInterface(job_interface_dict) job_data = Mock(spec=JobData) job_data.save_parse_results = Mock() fake_stdout = '' job_interface.perform_post_steps(job_exe, job_data, fake_stdout) job_data.store_output_data_files.assert_called_with( { 'output_files': [ ('/some/path/foo.txt', None, geo_metadata), ('/other/path/foo.txt', None, geo_metadata), ] }, job_exe)
def test_manifest_overrides_stdout(self, mock_loads, mock_open, mock_exists, mock_isfile): job_interface_dict, job_data_dict = self._get_simple_interface_data() job_interface_dict['input_data'] = [{ 'name': 'input_file', 'type': 'file', 'required': True, }] job_interface_dict['output_data'] = [{ 'name': 'output_file', 'type': 'file', 'required': True, }, { 'name': 'output_file_2', 'type': 'file', 'required': True, }] job_data_dict['input_data'].append({ 'name': 'input_file', 'file_id': self.file.id, }) results_manifest = { 'version': '1.0', 'files': [{ 'name': 'output_file', 'path': '/new/path/foo.txt', }] } mock_loads.return_value = results_manifest mock_exists.return_value = True mock_isfile.return_value = True job_exe = MagicMock() job_interface = JobInterface(job_interface_dict) job_data = Mock(spec=JobData) job_data.save_parse_results = Mock() fake_stdout = """ This text is supposed to mimic the output of a program we should see artifacts registered with the format: ARTIFACT:<input-name>:path, but it needs be at the beginning of a line so the example above won't match, but this will ARTIFACT:output_file:/path/to/foo.txt We should also be able to have text after the artifact and multiple artifacts. ARTIFACT:output_file_2:/path/to/foo_2.txt """ job_interface.perform_post_steps(job_exe, job_data, fake_stdout) job_data.store_output_data_files.assert_called_with( { 'output_file': ('/new/path/foo.txt', None), 'output_file_2': ('/path/to/foo_2.txt', None), }, job_exe)
def test_manifest_overrides_stdout(self, mock_loads, mock_open, mock_exists, mock_isfile): job_interface_dict, job_data_dict = self._get_simple_interface_data() job_interface_dict['input_data'] = [{ 'name': 'input_file', 'type': 'file', 'required': True, }] job_interface_dict['output_data'] = [{ 'name': 'output_file', 'type': 'file', 'required': True, }, { 'name': 'output_file_2', 'type':'file', 'required': True, }] job_data_dict['input_data'].append({ 'name': 'input_file', 'file_id': self.file.id, }) results_manifest = { 'version': '1.0', 'files': [{ 'name': 'output_file', 'path': '/new/path/foo.txt', }] } mock_loads.return_value = results_manifest mock_exists.return_value = True mock_isfile.return_value = True job_exe = MagicMock() job_interface = JobInterface(job_interface_dict) job_data = Mock(spec=JobData) job_data.save_parse_results = Mock() fake_stdout = """ This text is supposed to mimic the output of a program we should see artifacts registered with the format: ARTIFACT:<input-name>:path, but it needs be at the beginning of a line so the example above won't match, but this will ARTIFACT:output_file:/path/to/foo.txt We should also be able to have text after the artifact and multiple artifacts. ARTIFACT:output_file_2:/path/to/foo_2.txt """ job_interface.perform_post_steps(job_exe, job_data, fake_stdout) job_data.store_output_data_files.assert_called_with({ 'output_file': ('/new/path/foo.txt', None), 'output_file_2': ('/path/to/foo_2.txt', None), }, job_exe)
def test_output_files_with_geo_metadata(self, mock_loads, mock_open, mock_exists, mock_isfile): job_interface_dict, job_data_dict = self._get_simple_interface_data() job_interface_dict['output_data'] = [{ 'name': 'output_files', 'type': 'files', 'required': True, }] job_data_dict['output_data'].append({ 'name': 'output_files', 'workspace_id': self.workspace.id, }) geo_metadata = { 'data_started': '2015-05-15T10:34:12Z', 'data_ended': '2015-05-15T10:36:12Z', 'geo_json': { 'type': 'Polygon', 'coordinates': [[[1.0, 10.0], [2.0, 10.0], [2.0, 20.0], [1.0, 20.0], [1.0, 10.0]]], } } results_manifest = { 'version': '1.1', 'output_data': [{ 'name': 'output_files', 'files': [{ 'path': '/some/path/foo.txt', 'geo_metadata': geo_metadata, }, { 'path': '/other/path/foo.txt', 'geo_metadata': geo_metadata, }] }] } mock_loads.return_value = results_manifest mock_exists.return_value = True mock_isfile.return_value = True job_exe = MagicMock() job_interface = JobInterface(job_interface_dict) job_data = Mock(spec=JobData) job_data.save_parse_results = Mock() fake_stdout = '' job_interface.perform_post_steps(job_exe, job_data, fake_stdout) job_data.store_output_data_files.assert_called_with({ 'output_files': [ ('/some/path/foo.txt', None, geo_metadata), ('/other/path/foo.txt', None, geo_metadata), ] }, job_exe)
def test_output_files(self, mock_loads, mock_open, mock_exists): job_interface_dict, job_data_dict= self._get_simple_interface_data() job_interface_dict[u'output_data'] = [{"name":"output_files", "type":"files", "required": True}] job_data_dict[u'output_data'].append({"name":"output_files", "workspace_id":1}) results_manifest={"version" : "1.0", "files" : [{"name":"output_files", "paths": ["/some/path/foo.txt", "/other/path/foo.txt"]}]} mock_loads.return_value = results_manifest mock_exists.return_value = True job_exe = MagicMock() job_interface = JobInterface(job_interface_dict) job_data = Mock(spec=JobData) job_data.save_parse_results = Mock() fake_stdout = u'' job_interface.perform_post_steps(job_exe, job_data, fake_stdout) job_data.store_output_data_files.assert_called_with({u'output_files': [(u'/some/path/foo.txt', None),(u'/other/path/foo.txt', None)]}, job_exe)
def test_output_files_with_geo_metadata(self, mock_loads, mock_open, mock_exists): job_interface_dict, job_data_dict= self._get_simple_interface_data() job_interface_dict[u'output_data'] = [{"name":"output_files", "type":"files", "required": True}] job_data_dict[u'output_data'].append({"name":"output_files", "workspace_id":1}) geo_metadata = { "data_started": "2015-05-15T10:34:12Z", "data_ended": "2015-05-15T10:36:12Z", "geo_json": { "type": "Polygon", "coordinates": [[[ 1.0, 10.0 ], [ 2.0, 10.0 ], [ 2.0, 20.0 ],[ 1.0, 20.0 ], [ 1.0, 10.0 ]]] } } results_manifest = { "version": "1.1", "output_data": [ { "name" : "output_files", "files": [ { "path" : "/some/path/foo.txt", "geo_metadata": geo_metadata }, { "path" : "/other/path/foo.txt", "geo_metadata": geo_metadata } ] } ] } mock_loads.return_value = results_manifest mock_exists.return_value = True job_exe = MagicMock() job_interface = JobInterface(job_interface_dict) job_data = Mock(spec=JobData) job_data.save_parse_results = Mock() fake_stdout = u'' job_interface.perform_post_steps(job_exe, job_data, fake_stdout) job_data.store_output_data_files.assert_called_with( {u'output_files': [(u'/some/path/foo.txt', None, geo_metadata),(u'/other/path/foo.txt', None, geo_metadata)]}, job_exe)
def test_parse_data(self, mock_loads, mock_open, mock_exists): job_interface_dict, job_data_dict= self._get_simple_interface_data() job_interface_dict[u'input_data'] = [{"name":"input_file", "type":"file", "required": True}] job_data_dict[u'input_data'].append({"name":"input_file", "file_id":1}) geo_json = {"type": 'Feature'} geo_metadata = { "data_started": '2015-01-01T00:00:00Z', "geo_json": geo_json } results_manifest={"version" : "1.1", "parse_results" : [{"filename":"/some/path/foo.txt", "geo_metadata":geo_metadata}]} mock_loads.return_value = results_manifest mock_exists.return_value = True job_exe = MagicMock() job_interface = JobInterface(job_interface_dict) job_data = Mock(spec=JobData) job_data.save_parse_results = Mock() fake_stdout = u'' job_interface.perform_post_steps(job_exe, job_data, fake_stdout) job_data.save_parse_results.assert_called_with({u'/some/path/foo.txt': (geo_json, '2015-01-01T00:00:00Z', None, [], None, None)})
def test_parse_data(self, mock_loads, mock_open, mock_exists): job_interface_dict, job_data_dict = self._get_simple_interface_data() job_interface_dict['input_data'] = [{ 'name': 'input_file', 'type': 'file', 'required': True, }] job_data_dict['input_data'].append({ 'name': 'input_file', 'file_id': self.file.id, }) geo_json = {'type': 'Feature'} geo_metadata = { 'data_started': '2015-01-01T00:00:00Z', 'geo_json': geo_json } results_manifest = { 'version': '1.1', 'parse_results': [{ 'filename': '/some/path/foo.txt', 'geo_metadata': geo_metadata, }] } mock_loads.return_value = results_manifest mock_exists.return_value = True job_exe = MagicMock() job_interface = JobInterface(job_interface_dict) job_data = Mock(spec=JobData) job_data.save_parse_results = Mock() fake_stdout = '' job_interface.perform_post_steps(job_exe, job_data, fake_stdout) job_data.save_parse_results.assert_called_with({ '/some/path/foo.txt': (geo_json, '2015-01-01T00:00:00Z', None, [], None), })
def test_parse_data(self, mock_loads, mock_open, mock_exists): job_interface_dict, job_data_dict = self._get_simple_interface_data() job_interface_dict['input_data'] = [{ 'name': 'input_file', 'type': 'file', 'required': True, }] job_data_dict['input_data'].append({ 'name': 'input_file', 'file_id': self.file.id, }) geo_json = {'type': 'Feature'} geo_metadata = { 'data_started': '2015-01-01T00:00:00Z', 'geo_json': geo_json } results_manifest = { 'version': '1.1', 'parse_results': [{ 'filename': '/some/path/foo.txt', 'geo_metadata': geo_metadata, }] } mock_loads.return_value = results_manifest mock_exists.return_value = True job_exe = MagicMock() job_interface = JobInterface(job_interface_dict) job_data = Mock(spec=JobData) job_data.save_parse_results = Mock() fake_stdout = '' job_interface.perform_post_steps(job_exe, job_data, fake_stdout) job_data.save_parse_results.assert_called_with({ '/some/path/foo.txt': (geo_json, '2015-01-01T00:00:00Z', None, [], None, None), })