def testExportTableToCloudStorageCloudApi(self):
"""Verifies the Cloud Storage task created by Export.table()."""
with apitestcase.UsingCloudApi():
task = ee.batch.Export.table.toCloudStorage(
collection=ee.FeatureCollection('foo'),
outputBucket='test-bucket',
maxVertices=1e6)
self.assertIsNone(task.id)
self.assertIsNone(task.name)
self.assertEqual('EXPORT_FEATURES', task.task_type)
self.assertEqual('UNSUBMITTED', task.state)
self.assertEqual(
{
'expression': ee.FeatureCollection('foo'),
'description': 'myExportTableTask',
'fileExportOptions': {
'fileFormat': 'CSV',
'gcsDestination': {
'bucket': 'test-bucket',
'filenamePrefix': 'myExportTableTask',
},
},
'maxVertices': {
'value': int(1e6)
},
}, task.config)
def testPrepareForExport_withRegionDimensionsCrsAndTransform(self):
with apitestcase.UsingCloudApi():
polygon = ee.Geometry.Polygon(9, 8, 7, 6, 3, 2)
image, params = self.base_image.prepare_for_export({
'crs':
'ABCD',
'crs_transform':
'[1,2,3,4,5,6]',
'dimensions': [3, 2],
'region':
polygon.toGeoJSONString(),
'something':
'else'
})
expected_polygon = ee.Geometry(polygon.toGeoJSON(),
opt_geodesic=False)
projected = self.base_image.reproject(
crs='ABCD', crsTransform=[1, 2, 3, 4, 5, 6])
self.assertImageEqual(
projected.clipToBoundsAndScale(width=3,
height=2,
geometry=expected_polygon),
image)
self.assertEqual({'something': 'else'}, params)
def testThumb_withDimensionsRegionCrs(self):
"""Verifies Thumbnail ID and URL generation in the Cloud API."""
with apitestcase.UsingCloudApi(
cloud_api_resource=self.cloud_api_resource):
url = self.base_image.getThumbURL({
'dimensions': [13, 42],
'region': self.geo_json,
'crs': 'EPSG:4326',
})
self.assertEqual('/v1alpha/thumbName:getPixels', url)
_, kwargs = self.cloud_api_resource.projects().thumbnails(
).create.call_args
self.assertEqual(
kwargs['body']['expression'],
serializer.encode(self.base_image.setDefaultProjection(
crs='EPSG:4326',
crsTransform=[1, 0, 0, 0, -1, 0]).clipToBoundsAndScale(
geometry=ee.Geometry(self.geo_json,
opt_geodesic=False),
width=13,
height=42),
for_cloud_api=True))
self.assertEqual(kwargs['parent'], 'projects/earthengine-legacy')
def testCloudErrorTranslation(self):
mock_http = mock.MagicMock(httplib2.Http)
mock_http.request.return_value = (httplib2.Response({'status': 400}),
b'{"error": {"message": "errorly"} }')
with apitestcase.UsingCloudApi(mock_http=mock_http):
with self.assertRaisesRegex(ee.ee_exception.EEException, '^errorly$'):
ee.data.listImages({'parent': 'projects/earthengine-public/assets/q'})
def testPrepareForExport(self):
"""Verifies proper handling of export-related parameters."""
with apitestcase.UsingCloudApi():
base_collection = ee.ImageCollection(ee.Image(1))
collection, params = base_collection.prepare_for_export(
{'something': 'else'})
self.assertEqual(base_collection, collection)
self.assertEqual({'something': 'else'}, params)
collection, params = base_collection.prepare_for_export({
'crs':
'ABCD',
'crs_transform':
'1,2,3,4,5,6'
})
# Need to do a serialized comparison for the collection because
# custom functions don't implement equality comparison.
def expected_preparation_function(img):
return img.reproject(
crs='ABCD', crsTransform=[1.0, 2.0, 3.0, 4.0, 5.0, 6.0])
expected_collection = base_collection.map(
expected_preparation_function)
self.assertEqual(expected_collection.serialize(for_cloud_api=True),
collection.serialize(for_cloud_api=True))
self.assertEqual({}, params)
def testPrepareForExport_simple(self):
"""Verifies proper handling of export-related parameters."""
with apitestcase.UsingCloudApi():
image, params = self.base_image.prepare_for_export({'something': 'else'})
self.assertImageEqual(self.base_image, image)
self.assertEqual({'something': 'else'}, params)
def testComplexGetListViaCloudApi(self):
cloud_api_resource = mock.MagicMock()
with apitestcase.UsingCloudApi(cloud_api_resource=cloud_api_resource):
mock_result = {
'images': [{
'name': 'id1',
'size_bytes': 1234
}]
}
cloud_api_resource.projects().assets().listImages(
).execute.return_value = mock_result
actual_result = ee.data.getList({
'id': 'glam',
'num': 3,
'starttime': 3612345,
'filter': 'foo'
})
expected_params = {
'parent': 'projects/earthengine-public/assets/glam',
'pageSize': 3,
'startTime': '1970-01-01T01:00:12.345000Z',
'view': 'BASIC',
'filter': 'foo'
}
expected_result = [{'id': 'id1', 'type': 'Image'}]
cloud_api_resource.projects().assets().listImages.assert_called_with(
**expected_params)
self.assertEqual(expected_result, actual_result)
def testExportTableCloudApiBogusParameter(self):
"""Verifies that bogus parameters are rejected."""
with apitestcase.UsingCloudApi():
with self.assertRaisesRegexp(ee.EEException,
'Unknown configuration options.*'):
ee.batch.Export.table.toDrive(
ee.FeatureCollection('drive test FC'), framesPerSecond=30)
def testListOperationsEmptyList(self):
# Empty lists don't appear at all in the result.
mock_http = mock.MagicMock(httplib2.Http)
mock_http.request.return_value = (httplib2.Response({'status':
200}), b'{}')
with apitestcase.UsingCloudApi(mock_http=mock_http):
self.assertEqual([], ee.data.listOperations())
def testExportImageToAssetCloudApi(self):
"""Verifies the Asset export task created by Export.image.toAsset()."""
with apitestcase.UsingCloudApi():
config = dict(
image=ee.Image(1),
assetId='users/foo/bar',
pyramidingPolicy={'B1': 'min'})
expected_expression = ee.Image(1)
# Test keyed parameters.
task_keyed = ee.batch.Export.image.toAsset(
image=config['image'],
assetId=config['assetId'],
pyramidingPolicy=config['pyramidingPolicy'])
self.assertIsNone(task_keyed.id)
self.assertIsNone(task_keyed.name)
self.assertEqual('EXPORT_IMAGE', task_keyed.task_type)
self.assertEqual('UNSUBMITTED', task_keyed.state)
self.assertEqual({
'expression': expected_expression,
'description': 'myExportImageTask',
'assetExportOptions': {
'earthEngineDestination': {
'name': 'projects/earthengine-legacy/assets/users/foo/bar'
},
'pyramidingPolicyOverrides': {
'B1': 'MIN'
}
},
}, task_keyed.config)
task_ordered = ee.batch.Export.image.toAsset(
config['image'],
'TestDescription',
config['assetId'],
maxPixels=1000,
maxWorkers=100,
tileSize=4)
self.assertEqual('EXPORT_IMAGE', task_ordered.task_type)
self.assertEqual('UNSUBMITTED', task_ordered.state)
self.assertEqual({
'expression': expected_expression,
'description': 'TestDescription',
'assetExportOptions': {
'earthEngineDestination': {
'name': 'projects/earthengine-legacy/assets/users/foo/bar'
},
'tileSize': {
'value': 4
}
},
'maxPixels': {
'value': '1000'
},
'maxWorkerCount': {
'value': 100
}
}, task_ordered.config)
def testGetDownloadId_withSerializedImage(self):
cloud_api_resource = mock.MagicMock()
with apitestcase.UsingCloudApi(cloud_api_resource=cloud_api_resource):
with self.assertRaisesRegex(ee.ee_exception.EEException,
'^Image as JSON string not supported.'):
ee.data.getDownloadId({
'image': image.Image('my-image').serialize(),
'name': 'dummy'
})
def testPrepareForExport_invalidCrsAndTransform(self):
with apitestcase.UsingCloudApi():
with self.assertRaises(ee_exception.EEException):
self.base_image.prepare_for_export({'crs_transform': '1,2,3,4,5,6'})
with self.assertRaises(ValueError):
self.base_image.prepare_for_export({
'crs': 'ABCD',
'crs_transform': 'x'
})
def testListBuckets(self):
cloud_api_resource = mock.MagicMock()
with apitestcase.UsingCloudApi(cloud_api_resource=cloud_api_resource):
mock_result = {'assets': [{'name': 'id1', 'type': 'FOLDER'}]}
cloud_api_resource.projects().listAssets(
).execute.return_value = mock_result
actual_result = ee.data.listBuckets()
cloud_api_resource.projects().listAssets().execute.assert_called_once()
self.assertEqual(mock_result, actual_result)
def testPrepareForExport_withCrsAndTransform(self):
with apitestcase.UsingCloudApi():
image, params = self.base_image.prepare_for_export({
'crs': 'ABCD',
'crs_transform': '1,2,3,4,5,6'
})
self.assertImageEqual(
self.base_image.reproject(
crs='ABCD', crsTransform=[1, 2, 3, 4, 5, 6]), image)
self.assertEqual({}, params)
def testCloudProfilingEnabled(self):
seen = []
def ProfileHook(profile_id):
seen.append(profile_id)
with ee.data.profiling(ProfileHook):
with apitestcase.UsingCloudApi(), DoCloudProfileStubHttp(self, True):
ee.data.listImages({'parent': 'projects/earthengine-public/assets/q'})
self.assertEqual(['someProfileId'], seen)
def testExportImageToGoogleDriveCloudApi(self):
"""Verifies the Drive destined task created by Export.image.toDrive()."""
with apitestcase.UsingCloudApi():
region = ee.Geometry.Rectangle(1, 2, 3, 4)
drive_task_by_keys = ee.batch.Export.image.toDrive(
image=ee.Image(1), region=region['coordinates'], folder='foo',
maxPixels=10**10, crs='foo', crsTransform='[9,8,7,6,5,4]')
expected_expression = ee.Image(1).reproject(
'foo', crsTransform=[9.0, 8.0, 7.0, 6.0, 5.0,
4.0]).clipToBoundsAndScale(geometry=region)
self.assertIsNone(drive_task_by_keys.id)
self.assertIsNone(drive_task_by_keys.name)
self.assertEqual('EXPORT_IMAGE', drive_task_by_keys.task_type)
self.assertEqual('UNSUBMITTED', drive_task_by_keys.state)
self.assertEqual({
'expression': expected_expression,
'description': 'myExportImageTask',
'fileExportOptions': {
'fileFormat': 'GEO_TIFF',
'driveDestination': {
'folder': 'foo',
'filenamePrefix': 'myExportImageTask'
}
},
'maxPixels': {
'value': '10000000000'
},
}, drive_task_by_keys.config)
drive_task_with_old_keys = ee.batch.Export.image.toDrive(
image=ee.Image(1), region=region['coordinates'], driveFolder='foo',
driveFileNamePrefix='fooExport', maxPixels=10**10,
crs='foo', crs_transform='[9,8,7,6,5,4]')
self.assertIsNone(drive_task_with_old_keys.id)
self.assertIsNone(drive_task_by_keys.name)
self.assertEqual('EXPORT_IMAGE', drive_task_with_old_keys.task_type)
self.assertEqual('UNSUBMITTED', drive_task_with_old_keys.state)
self.assertEqual({
'expression': expected_expression,
'description': 'myExportImageTask',
'fileExportOptions': {
'fileFormat': 'GEO_TIFF',
'driveDestination': {
'folder': 'foo',
'filenamePrefix': 'fooExport'
}
},
'maxPixels': {
'value': '10000000000'
},
}, drive_task_with_old_keys.config)
with self.assertRaisesRegexp(ee.EEException,
'Unknown configuration options.*'):
ee.batch.Export.image.toDrive(image=ee.Image(1), framesPerSecond=30)
def testPrepareForExport_withPolygon(self):
with apitestcase.UsingCloudApi():
polygon = ee.Geometry.Polygon(9, 8, 7, 6, 3, 2)
image, params = self.base_image.prepare_for_export({
'dimensions': '3x2',
'region': polygon
})
expected = self.base_image.clipToBoundsAndScale(
width=3, height=2, geometry=polygon)
self.assertImageEqual(expected, image)
self.assertEqual({}, params)
def testListImages(self):
cloud_api_resource = mock.MagicMock()
with apitestcase.UsingCloudApi(cloud_api_resource=cloud_api_resource):
mock_result = {'images': [{'path': 'id1', 'type': 'type1'}]}
cloud_api_resource.projects().assets().listImages(
).execute.return_value = mock_result
cloud_api_resource.projects().assets().listImages_next.return_value = None
actual_result = ee.data.listImages({'p': 'q'})
cloud_api_resource.projects().assets().listImages(
).execute.assert_called_once()
self.assertEqual(mock_result, actual_result)
def testPrepareForExport_withScaleAndRegion(self):
with apitestcase.UsingCloudApi():
polygon = ee.Geometry.Polygon(9, 8, 7, 6, 3, 2)
image, params = self.base_image.prepare_for_export({
'scale': 8,
'region': polygon.toGeoJSONString(),
'something': 'else'
})
expected_polygon = ee.Geometry(polygon.toGeoJSON(), opt_geodesic=False)
self.assertImageEqual(
self.base_image.clipToBoundsAndScale(
scale=8, geometry=expected_polygon), image)
self.assertEqual({'something': 'else'}, params)
def testExportVideoToCloudStorageCloudApi(self):
"""Verifies the task created by Export.video.toCloudStorage()."""
with apitestcase.UsingCloudApi():
region = ee.Geometry.Rectangle(1, 2, 3, 4)
collection = ee.ImageCollection([ee.Image(1), ee.Image(2)])
def expected_preparation_function(img):
img = img.reproject(
crs='foo', crsTransform=[9.0, 8.0, 7.0, 6.0, 5.0, 4.0])
img = img.clipToBoundsAndScale(geometry=region, maxDimension=16)
return img
expected_collection = collection.map(expected_preparation_function)
expected_config = {
'description': 'TestVideoName',
'fileExportOptions': {
'fileFormat': 'MP4',
'gcsDestination': {
'bucket': 'test-bucket',
'filenamePrefix': 'TestVideoName',
}
}
}
# Test keyed parameters.
task_keyed = ee.batch.Export.video.toCloudStorage(
collection=collection,
description='TestVideoName',
bucket='test-bucket',
dimensions=16,
region=region['coordinates'],
crsTransform='[9,8,7,6,5,4]',
crs='foo')
self.assertIsNone(task_keyed.id)
self.assertIsNone(task_keyed.name)
self.assertEqual('EXPORT_VIDEO', task_keyed.task_type)
self.assertEqual('UNSUBMITTED', task_keyed.state)
self.assertEqual(
expected_collection.serialize(for_cloud_api=True),
task_keyed.config.pop('expression').serialize(for_cloud_api=True))
self.assertEqual(expected_config, task_keyed.config)
# Test ordered parameters.
task_ordered = ee.batch.Export.video.toCloudStorage(
collection, 'TestVideoName', 'test-bucket', None, None, 16,
region['coordinates'], None, 'foo', '[9,8,7,6,5,4]')
self.assertEqual('EXPORT_VIDEO', task_ordered.task_type)
self.assertEqual('UNSUBMITTED', task_ordered.state)
self.assertEqual(
expected_collection.serialize(for_cloud_api=True),
task_ordered.config.pop('expression').serialize(for_cloud_api=True))
self.assertEqual(expected_config, task_ordered.config)
def testPrepareForExport_withCrsNoTransform(self):
with apitestcase.UsingCloudApi():
# CRS with no crs_transform causes a "soft" reprojection. Make sure that
# the (crs, crsTransform, dimensions) special case doesn't trigger.
image, params = self.base_image.prepare_for_export({
'crs': 'ABCD',
'dimensions': [3, 2],
'something': 'else'
})
projected = self.base_image.setDefaultProjection(
crs='ABCD', crsTransform=[1, 0, 0, 0, -1, 0])
self.assertEqual(projected.clipToBoundsAndScale(width=3, height=2), image)
self.assertEqual({'something': 'else'}, params)
def testTaskStartCloudApi(self):
"""Verifies that Task.start() calls the server appropriately."""
mock_cloud_api_resource = mock.MagicMock()
mock_cloud_api_resource.projects().table().export().execute.return_value = {
'name': 'projects/earthengine-legacy/operations/foo',
'metadata': {},
}
with apitestcase.UsingCloudApi(cloud_api_resource=mock_cloud_api_resource):
task = ee.batch.Export.table(ee.FeatureCollection('foo'), 'bar')
task.start()
export_args = mock_cloud_api_resource.projects().table().export.call_args
self.assertEqual(task.id, 'foo')
self.assertTrue(export_args[1]['body']['requestId'])
self.assertEqual(export_args[1]['body']['description'], 'bar')
def testDownloadURL(self):
"""Verifies that the getDownloadURL request is constructed correctly."""
with apitestcase.UsingCloudApi(cloud_api_resource=self.cloud_api_resource):
url = self.base_image.getDownloadURL()
_, kwargs = self.cloud_api_resource.projects().thumbnails(
).create.call_args
self.assertEqual(
serializer.encode(self.base_image, for_cloud_api=True),
kwargs['body']['expression'])
self.assertEqual('ZIPPED_GEO_TIFF_PER_BAND', kwargs['body']['fileFormat'])
self.assertEqual('projects/earthengine-legacy', kwargs['parent'])
self.assertEqual('/%s/thumbName:getPixels' % _cloud_api_utils.VERSION,
url)
def testExportTableToGoogleDriveCloudApi(self):
"""Verifies the Drive destined task created by Export.table.toDrive()."""
with apitestcase.UsingCloudApi():
test_collection = ee.FeatureCollection('foo')
test_description = 'TestDescription'
test_file_name_prefix = 'fooDriveFileNamePrefix'
test_format = 'KML'
expected_config = {
'expression': test_collection,
'description': test_description,
'fileExportOptions': {
'fileFormat': test_format,
'driveDestination': {
'filenamePrefix': test_file_name_prefix,
}
}
}
# Ordered parameters
task_ordered = ee.batch.Export.table.toDrive(
test_collection, test_description,
None, test_file_name_prefix, test_format)
self.assertIsNone(task_ordered.id)
self.assertIsNone(task_ordered.name)
self.assertEqual('EXPORT_FEATURES', task_ordered.task_type)
self.assertEqual('UNSUBMITTED', task_ordered.state)
self.assertEqual(expected_config, task_ordered.config)
# Updating expectations to test keyed parameters
expected_config['description'] = 'myExportTableTask'
expected_config['fileExportOptions']['fileFormat'] = 'CSV'
expected_config['fileExportOptions']['driveDestination'][
'folder'] = 'fooFolder'
# Test that deprecated parameters (driveFolder and driveFileNamePrefix)
# still work.
task_old_keys = ee.batch.Export.table.toDrive(
collection=test_collection, driveFolder='fooFolder',
driveFileNamePrefix='fooDriveFileNamePrefix')
self.assertEqual('EXPORT_FEATURES', task_old_keys.task_type)
self.assertEqual('UNSUBMITTED', task_old_keys.state)
self.assertEqual(expected_config, task_old_keys.config)
# Test that new parameters work
task_new_keys = ee.batch.Export.table.toDrive(
collection=test_collection, folder='fooFolder',
fileNamePrefix='fooDriveFileNamePrefix')
self.assertEqual('EXPORT_FEATURES', task_new_keys.task_type)
self.assertEqual('UNSUBMITTED', task_new_keys.state)
self.assertEqual(expected_config, task_new_keys.config)
def testSimpleGetListViaCloudApi(self):
cloud_api_resource = mock.MagicMock()
with apitestcase.UsingCloudApi(cloud_api_resource=cloud_api_resource):
mock_result = {'assets': [{'name': 'id1', 'type': 'IMAGE_COLLECTION'}]}
cloud_api_resource.projects().assets().listAssets(
).execute.return_value = mock_result
actual_result = ee.data.getList({'id': 'glam', 'num': 3})
expected_params = {
'parent': 'projects/earthengine-public/assets/glam',
'pageSize': 3
}
expected_result = [{'id': 'id1', 'type': 'ImageCollection'}]
cloud_api_resource.projects().assets().listAssets.assert_called_with(
**expected_params)
self.assertEqual(expected_result, actual_result)
def testDownloadTableWithCloudApi(self):
cloud_api_resource = mock.MagicMock()
with apitestcase.UsingCloudApi(cloud_api_resource=cloud_api_resource):
create_table_response = {'name': 'table_name'}
cloud_api_resource.projects().tables().create(
).execute.return_value = (create_table_response)
fc = ee.FeatureCollection([ee.Feature(None, {'foo': 'bar'})])
result = ee.data.getTableDownloadId({
'table': fc,
'selectors': 'foo',
'format': 'CSV',
})
url = ee.data.makeTableDownloadUrl(result)
self.assertDictEqual(result, {'docid': '5', 'token': '6'})
self.assertEqual(url, '/v1alpha/5:getFeatures')
def testExportImageCloudApi(self):
"""Verifies the task created by Export.image()."""
with apitestcase.UsingCloudApi():
region = ee.Geometry.Rectangle(1, 2, 3, 4)
config = dict(
region=region['coordinates'],
maxPixels=10**10,
crs='foo',
crs_transform='[9,8,7,6,5,4]',
tiffCloudOptimized=True,
shardSize=512,
fileDimensions=1024,
)
task = ee.batch.Export.image(ee.Image(1), 'TestDescription', config)
expected_expression = ee.Image(1).reproject(
'foo', crsTransform=[
9.0, 8.0, 7.0, 6.0, 5.0, 4.0
]).clip(region)
self.assertIsNone(task.id)
self.assertIsNone(task.name)
self.assertEqual('EXPORT_IMAGE', task.task_type)
self.assertEqual('UNSUBMITTED', task.state)
self.assertEqual(
{
'expression': expected_expression,
'description': 'TestDescription',
'fileExportOptions': {
'fileFormat': 'GEO_TIFF',
'driveDestination': {
'filenamePrefix': 'TestDescription'
},
'geoTiffOptions': {
'cloudOptimized': True,
'tileDimensions': {
'width': 1024,
'height': 1024
},
'tileSize': {
'value': 512
}
},
},
'maxPixels': {
'value': '10000000000'
},
}, task.config)
def testThumb_withDimensionsRegionJson(self):
# Try it with the region as a GeoJSON string.
with apitestcase.UsingCloudApi(
cloud_api_resource=self.cloud_api_resource):
self.base_image.getThumbURL({
'dimensions': [13, 42],
'region': json.dumps(self.geo_json),
})
_, kwargs = self.cloud_api_resource.projects().thumbnails(
).create.call_args
self.assertEqual(
kwargs['body']['expression'],
serializer.encode(self.base_image.clipToBoundsAndScale(
geometry=self.expected_geometry, width=13, height=42),
for_cloud_api=True))
self.assertEqual(kwargs['parent'], 'projects/earthengine-legacy')
def testThumb_withVisualizationParams(self):
with apitestcase.UsingCloudApi(
cloud_api_resource=self.cloud_api_resource):
self.base_image.getThumbURL({
'dimensions': [13, 42],
'region': self.geo_json,
'min': 0
})
_, kwargs = self.cloud_api_resource.projects().thumbnails(
).create.call_args
self.assertEqual(
kwargs['body']['expression'],
serializer.encode(self.base_image.clipToBoundsAndScale(
geometry=self.expected_geometry, width=13,
height=42).visualize(min=0),
for_cloud_api=True))
def testGetDownloadId(self, _):
cloud_api_resource = mock.MagicMock()
with apitestcase.UsingCloudApi(cloud_api_resource=cloud_api_resource):
mock_result = {'name': 'projects/earthengine-legacy/thumbnails/DOCID'}
cloud_api_resource.projects().thumbnails().create(
).execute.return_value = mock_result
actual_result = ee.data.getDownloadId({
'image': image.Image('my-image'),
'name': 'dummy'
})
cloud_api_resource.projects().thumbnails().create(
).execute.assert_called_once()
self.assertEqual(
{
'docid': 'projects/earthengine-legacy/thumbnails/DOCID',
'token': ''
}, actual_result)
