def test_post_create(self):
'''it enqueues a job after being created'''
bundle = JobBundle(id=7)
with patch.object(theia.tasks, 'process_bundle',
autospec=True) as mockProcess:
with patch.object(mockProcess.return_value, 'delay',
autospec=True) as mockDelay:
JobBundle.post_save(None, bundle, False)
mockDelay.assert_not_called()
JobBundle.post_save(None, bundle, True)
mockDelay.assert_called_once_with(bundle.id)
def test__connect(self, mockSecret, mockId, mockUrl, mockConnect):
operation = UploadSubject(JobBundle())
operation._connect()
mockUrl.assert_called_once()
mockId.assert_called_once()
mockSecret.assert_called_once()
mockConnect.assert_called_once_with(endpoint='sample url',
client_id='sample id',
client_secret='sample secret')
def test__resolve_name(mock_resolve, mock_locate, mock_glob):
stage = PipelineStage(select_images=['infrared'],
operation='noop',
sort_order=5)
bundle = JobBundle(local_path='/tmp')
adapter = Adapter
tasks._resolve_name(adapter, stage, bundle, 'infrared')
mock_resolve.assert_called_once_with(bundle, 'infrared')
mock_locate.assert_called_once_with('/tmp/literal name', 5)
def test_retrieve(self, mockExtract, mockRetrieve, mockSave, mockUnameNode, mockIsFile):
scene = RequestedScene(scene_url='https://example.org')
bundle = JobBundle(scene_entity_id='test_id', requested_scene=scene)
Adapter().retrieve(bundle)
mockUnameNode.assert_called_once()
assert(bundle.hostname=='testhostname')
mockSave.assert_called_once()
mockIsFile.assert_called_once_with('tmp/test_id.tar.gz')
mockRetrieve.assert_called_once_with('https://example.org', 'tmp/test_id.tar.gz')
mockExtract.assert_called_once_with('tmp/test_id.tar.gz', bundle.local_path)
def test_apply(self, mock_open, mock_get_name):
stage = PipelineStage(config={'width': 10, 'height': 20}, sort_order=3)
bundle = JobBundle(current_stage=stage)
operation = ResizeImage(bundle)
operation.apply(['literal filename'])
mock_get_name.assert_called_once_with('literal filename')
mock_open.assert_called_once_with('literal filename')
mock_open.return_value.thumbnail.assert_called_once_with(
(10, 20), Image.ANTIALIAS)
mock_open.return_value.save.assert_called_once_with('new filename')
def test__get_subject_set(self, mock_client, mockFind, mockCreateSet,
*args):
mockFind.reset_mock()
mockCreateSet.reset_mock()
emptyJobBundle = JobBundle()
linkedJobBundle = JobBundle(subject_set_id=3)
emptyPipeline = Pipeline()
linkedPipeline = Pipeline(subject_set_id=3)
operation = UploadSubject(emptyJobBundle)
result = operation._get_subject_set(emptyJobBundle, 8, 'some name')
mockFind.assert_not_called()
mockCreateSet.assert_called_once_with(8, 'some name')
mockFind.reset_mock()
mockCreateSet.reset_mock()
operation = UploadSubject(linkedJobBundle)
result = operation._get_subject_set(linkedJobBundle, 8, 'some name')
mockFind.assert_called_once_with(3)
mockCreateSet.assert_not_called()
mockFind.reset_mock()
mockCreateSet.reset_mock()
operation = UploadSubject(emptyPipeline)
result = operation._get_subject_set(emptyPipeline, 8, 'some name')
mockFind.assert_not_called()
mockCreateSet.assert_called_once_with(8, 'some name')
mockFind.reset_mock()
mockCreateSet.reset_mock()
operation = UploadSubject(linkedPipeline)
result = operation._get_subject_set(linkedPipeline, 8, 'some name')
mockFind.assert_called_once_with(3)
mockCreateSet.assert_not_called()
def test_apply_multiple(self, mockAdd, mockCreate, mockGet, mockConnect,
mockGetName, *args):
project = Project(id=8)
pipeline = Pipeline(project=project, multiple_subject_sets=True)
bundle = JobBundle(pipeline=pipeline)
operation = UploadSubject(bundle)
operation.apply(['some_file'])
mockConnect.assert_called_once()
mockGetName.assert_called_once()
mockGet.assert_called_once_with(bundle, 8, 'bundle name')
mockCreate.assert_called_once_with(8, 'some_file')
mockAdd.assert_called_once_with(mockCreate.return_value)
def test_apply_single(self, mockOpen, mockAdd, mockCreate, mockGet,
mockGetName, mockIncludeMetadata, *args):
project = Project(id=8)
pipeline = Pipeline(project=project)
bundle = JobBundle(pipeline=pipeline)
operation = UploadSubject(bundle)
operation.apply(['some_file'])
mockOpen.assert_called_once()
mockGetName.assert_called_once()
mockGet.assert_called_once_with(pipeline, 8, 'pipeline name')
mockCreate.assert_called_once_with(8, 'some_file', metadata={})
mockAdd.assert_called_once_with(mockCreate.return_value)
def setUp(self):
self.request = ImageryRequest(adapter_name='dummy')
self.stage = PipelineStage(
select_images=['composite'],
sort_order=3,
config={
'tile_size': 100,
'tile_overlap': 10,
'output_filename': 'foo.csv',
},
)
self.bundle = JobBundle(current_stage=self.stage,
imagery_request=self.request)
self.operation = ComputeCorners(self.bundle)
def test__resolve_name_wildcard(mock_resolve, mock_absolute, mock_locate,
mock_glob):
stage = PipelineStage(select_images=['infrared'],
operation='noop',
sort_order=5)
bundle = JobBundle(local_path='/tmp')
adapter = Adapter
tasks._resolve_name(adapter, stage, bundle, 'tiles/*.jpg')
mock_resolve.assert_not_called()
mock_absolute.assert_called_once_with(bundle=bundle,
filename='tiles/*.jpg')
mock_glob.assert_called_once_with('tmp/tiles/*.jpg')
mock_locate.assert_has_calls([
call('/tmp/literal name', 5),
call('/tmp/another.jpg', 5),
])
def test_do_apply(self, mockFromArray, mockOpen, mockRename, mockRemap):
mockRemap.return_value = self.dummy_array
mockOpen.return_value.read_image.return_value = self.dummy_array
request = ImageryRequest(adapter_name='dummy')
stage = PipelineStage(config={}, sort_order=3)
bundle = JobBundle(current_stage=stage, imagery_request=request)
operation = RemapImage(bundle)
operation.apply(['literal filename'])
mockRename.assert_called_once_with('literal filename')
mockOpen.assert_called_once_with('literal filename')
mockOpen.return_value.read_image.assert_called_once_with()
mockRemap.assert_called_once_with(self.dummy_array)
mockFromArray.assert_called_once_with(self.dummy_array)
mockFromArray.return_value.save.assert_called_once_with(
'versioned filename')
def test_apply(self, mock_merge, mock_open, mock_get_new, mock_get_version):
request = ImageryRequest(adapter_name='dummy')
stage = PipelineStage(
select_images=['green_channel', 'red_channel', 'blue_channel'],
config={"red": "red_channel", "green": "green_channel", "blue": "blue_channel", "filename": "newish name"},
sort_order=3
)
bundle = JobBundle(current_stage=stage, imagery_request=request)
operation = ComposeImages(bundle)
operation.apply(['green_tif', 'red_tif', 'blue_tif'])
mock_open.assert_has_calls([call('green_tif'), call('red_tif'), call('blue_tif')])
mock_merge.assert_called_once_with('RGB', (ANY, ANY, ANY))
mock_merge.return_value.save.assert_called_once()
first_method_call = mock_merge.return_value.save.call_args_list[0]
args = first_method_call[0]
assert str.endswith(args[0], '/theia/3_/totally new name')
def test_apply(self, mock_merge, mock_open, mock_get_new,
mock_get_version):
request = ImageryRequest(adapter_name='dummy')
stage = PipelineStage(select_images=['ggggg', 'rrr', 'bbbb'],
config={
'red': 'rrr',
'green': 'ggggg',
'blue': 'bbbb',
'filename': 'newish name'
},
sort_order=3)
bundle = JobBundle(current_stage=stage, imagery_request=request)
operation = ComposeImages(bundle)
operation.apply(['neerg', 'erd', 'eulb'])
mock_get_new.assert_called_once_with('newish name')
mock_get_version.assert_called_once_with('new name')
mock_open.assert_has_calls([call('erd'), call('neerg'), call('eulb')])
mock_merge.assert_called_once_with('RGB', (ANY, ANY, ANY))
mock_merge.return_value.save.assert_called_once_with(
'totally new name')
def test_process_bundle(mock_input_files, mock_resolve, mock_apply,
mock_retrieve):
project = Project(id=4)
project.save()
pipeline = Pipeline(project=project)
pipeline.save()
stage_1 = PipelineStage(operation='noop',
pipeline=pipeline,
sort_order=1,
config={})
stage_1.save()
stage_2 = PipelineStage(operation='noop',
select_images=['blue'],
pipeline=pipeline,
sort_order=2,
config={})
stage_2.save()
request = ImageryRequest(adapter_name='dummy',
pipeline=pipeline,
project=project)
request.save()
bundle = JobBundle(id=3, total_stages=2)
bundle.imagery_request = request
bundle.pipeline = pipeline
bundle.save()
tasks.process_bundle(3)
assert (mock_apply.call_count == 2)
second_method_call = mock_apply.call_args_list[1]
arguments = second_method_call[0]
file_list = arguments[0]
assert str.endswith(file_list[0], '/theia/1_noop/input_file')
mock_retrieve.assert_called_once()
def test___str__(self):
bundle = JobBundle(scene_entity_id='entity', hostname='some.host')
assert (bundle.__str__() == '[JobBundle entity on some.host]')
bundle = JobBundle(scene_entity_id='entity')
assert (bundle.__str__() == '[JobBundle entity on no host]')
def test_name_subject_set(self):
bundle = JobBundle(scene_entity_id='entity')
assert (bundle.name_subject_set() == 'entity')
def test_resolve_relative_image(self):
request = ImageryRequest(adapter_name='usgs', dataset_name='LANDSAT_8_C1')
bundle = JobBundle(scene_entity_id='LC08', imagery_request=request, local_path='tmp/')
assert(Adapter().resolve_relative_image(bundle, 'red') == 'LC08_sr_band4.tif')
def test_construct_filename(self):
bundle = JobBundle(scene_entity_id='LC08', local_path='tmp/')
assert(Adapter().construct_filename(bundle, 'aerosol')=='LC08_sr_aerosol.tif')
def test_get_metadata(self, mock_resolve, mock_get_tree):
patch.object(mock_get_tree.return_value, 'xpath', return_value='some value')
bundle = JobBundle(scene_entity_id='full')
Adapter().get_metadata(bundle, 'some_field')
mock_resolve.assert_called_once_with('some_field')
import theia.tasks as tasks
from theia.api.models import ImageryRequest, JobBundle, Pipeline, PipelineStage, Project
from theia.adapters.dummy import Adapter
@patch('theia.api.models.ImageryRequest.objects.get',
return_value=ImageryRequest(adapter_name='dummy'))
@patch('theia.adapters.dummy.Adapter.process_request')
def test_locate_scenes(mock_process, mock_get):
tasks.locate_scenes(7)
mock_get.assert_called_once_with(pk=7)
mock_process.assert_called_once_with(mock_get.return_value)
@patch('theia.adapters.dummy.Adapter.retrieve')
@patch('theia.api.models.JobBundle.objects.get', return_value=JobBundle(id=3))
@patch('theia.api.models.JobBundle.save')
@patch('theia.operations.noop.NoOp.apply')
@patch('theia.tasks._resolve_name', return_value=['blue_resolved'])
def test_process_bundle(mock_resolve, mock_apply, mock_save, mock_get,
mock_retrieve):
project = Project()
pipeline = Pipeline(project=project)
stage_1 = PipelineStage(operation='noop')
stage_2 = PipelineStage(operation='noop', select_images=['blue'])
request = ImageryRequest(adapter_name='dummy', pipeline=pipeline)
with patch('theia.api.models.Pipeline.get_stages',
return_value=[stage_1, stage_2]) as mockStages:
bundle = mock_get.return_value
bundle.imagery_request = request