def extract_state(status):
state = status['state']
task.state = state
if state == 'FAILED':
return throw(ee.EEException(status.get('error_message')))
else:
return of(state)
def run(self, args, config):
"""Runs the asset update."""
config.ee_init()
properties = _decode_property_flags(args)
if not properties and args.time_start is None and args.time_end is None:
raise ee.EEException('No properties specified.')
if config.use_cloud_api:
update_mask = [
'properties.' + property_name for property_name in properties
]
asset = {}
if properties:
asset['properties'] = {
k: v for k, v in six.iteritems(properties) if v is not None
}
# args.time_start and .time_end could have any of three falsy values, with
# different meanings:
# None: the --time_start flag was not provided at all
# '': the --time_start flag was explicitly set to the empty string
# 0: the --time_start flag was explicitly set to midnight 1 Jan 1970.
# pylint:disable=g-explicit-bool-comparison
if args.time_start is not None:
update_mask.append('start_time')
if args.time_start != '':
asset['start_time'] = _cloud_timestamp_for_timestamp_ms(
args.time_start)
if args.time_end is not None:
update_mask.append('end_time')
if args.time_end != '':
asset['end_time'] = _cloud_timestamp_for_timestamp_ms(args.time_end)
# pylint:enable=g-explicit-bool-comparison
ee.data.updateAsset(args.asset_id, asset, update_mask)
return
properties.update(_decode_timestamp_flags(args))
ee.data.setAssetProperties(args.asset_id, properties)
def _decode_property_flags(args):
"""Decodes metadata properties from args as a name->value dict."""
property_list = list(args.property or [])
names = [name for name, _ in property_list]
duplicates = [name for name, count in Counter(names).items() if count > 1]
if duplicates:
raise ee.EEException('Duplicate property name(s): %s.' % duplicates)
return dict(property_list)
def run(self, args, config):
config.ee_init()
info = ee.data.getInfo(args.asset_id)
if info:
_pretty_print_json(info)
else:
raise ee.EEException(
'Asset does not exist or is not accessible: %s' % args.asset_id)
def _upload(args, request, ingestion_function):
if 0 <= args.wait < 10:
raise ee.EEException('Wait time should be at least 10 seconds.')
task_id = ee.data.newTaskId()[0]
ingestion_function(task_id, request)
print('Started upload task with ID: %s' % task_id)
if args.wait >= 0:
print('Waiting for the upload task to complete...')
utils.wait_for_task(task_id, args.wait)
def filter_collection(collection, coords, period=None):
filtered = collection
if period is not None:
filtered = filtered.filterDate(*period) # filter time
filtered = filtered.filterBounds(
ee.Geometry.Point(coords)) # filter region
if filtered.size().getInfo() == 0:
raise ee.EEException(
f'ImageCollection.filter: No suitable images found in ({coords[1]:.4f}, {coords[0]:.4f}) between {period[0]} and {period[1]}.'
)
return filtered
def _parse_permissions(self, args):
"""Decodes and sanity-checks the permissions in the arguments."""
# A dictionary mapping from user ids to one of 'R', 'W', or 'D'.
permissions = {}
if args.u:
for grant in args.u:
parts = grant.split(':')
if len(parts) != 2 or parts[1] not in ['R', 'W']:
raise ee.EEException('Invalid permission "%s".' % grant)
user, role = parts
if user in permissions:
raise ee.EEException('Multiple permission settings for "%s".' % user)
if user == ALL_USERS and role == 'W':
raise ee.EEException('Cannot grant write permissions to AllUsers.')
permissions[user] = role
if args.d:
for user in args.d:
if user in permissions:
raise ee.EEException('Multiple permission settings for "%s".' % user)
permissions[user] = 'D'
return permissions
def _decode_property_flags(args):
"""Decodes metadata properties from args as a list of (name,value) pairs."""
property_list = list(args.property or [])
if args.time_start:
property_list.append((SYSTEM_TIME_START, args.time_start))
if args.time_end:
property_list.append((SYSTEM_TIME_END, args.time_end))
names = [name for name, _ in property_list]
duplicates = [name for name, count in Counter(names).items() if count > 1]
if duplicates:
raise ee.EEException('Duplicate property name(s): %s.' % duplicates)
return dict(property_list)
def _get_size(asset):
"""Returns the size of the given asset in bytes."""
size_parsers = {
'Folder': _get_size_folder,
'ImageCollection': _get_size_image_collection,
}
if asset['type'] not in size_parsers:
raise ee.EEException('Cannot get size for asset type "%s"' %
asset['type'])
return size_parsers[asset['type']](asset)
def get_credentials():
try:
tokens = json.load(open(credential_path))
refresh_token = tokens['refresh_token']
return Credentials(None,
refresh_token=refresh_token,
token_uri=ee.oauth.TOKEN_URI,
client_id=ee.oauth.CLIENT_ID,
client_secret=ee.oauth.CLIENT_SECRET,
scopes=ee.oauth.SCOPES)
except IOError:
raise ee.EEException(
'Please authorize access to your Earth Engine account by '
'running\n\nearthengine authenticate\n\nin your command line, and then '
'retry.')
def run(self, args, config):
config.ee_init()
cancel_all = args.task_ids == ['all']
if cancel_all:
statuses = ee.data.getTaskList()
else:
statuses = ee.data.getTaskStatus(args.task_ids)
for status in statuses:
state = status['state']
task_id = status['id']
if state == 'UNKNOWN':
raise ee.EEException('Unknown task id "%s"' % task_id)
elif state == 'READY' or state == 'RUNNING':
print('Canceling task "%s"' % task_id)
ee.data.cancelTask(task_id)
elif not cancel_all:
print('Task "%s" already in state "%s".' % (status['id'], state))
def _get_size(self, asset):
"""Returns the size of the given asset in bytes."""
size_parsers = {
'Image': self._get_size_asset,
'Folder': self._get_size_folder,
'ImageCollection': self._get_size_image_collection,
'Table': self._get_size_asset,
'IMAGE': self._get_size_asset,
'FOLDER': self._get_size_folder,
'IMAGE_COLLECTION': self._get_size_image_collection,
'TABLE': self._get_size_asset,
}
if asset['type'] not in size_parsers:
raise ee.EEException(
'Cannot get size for asset type "%s"' % asset['type'])
return size_parsers[asset['type']](asset)
def run(self, args, config):
"""Waits on the given tasks to complete or for a timeout to pass."""
config.ee_init()
task_ids = []
if args.task_ids == ['all']:
tasks = ee.data.getTaskList()
for task in tasks:
if task['state'] not in utils.TASK_FINISHED_STATES:
task_ids.append(task['id'])
else:
statuses = ee.data.getTaskStatus(args.task_ids)
for status in statuses:
state = status['state']
task_id = status['id']
if state == 'UNKNOWN':
raise ee.EEException('Unknown task id "%s"' % task_id)
else:
task_ids.append(task_id)
utils.wait_for_tasks(task_ids, args.timeout, log_progress=args.verbose)
def build_map(**kwargs):
"""
Creates a map in Google Earth Engine using the python api and returns the map id and token.
:param kwargs:
:return: mapid object
"""
reducer = getattr(ee.Reducer, kwargs.get('reducer', 'mode'))()
if 'collection' in kwargs:
collection = ee.ImageCollection(kwargs['collection'])
collection = collection.select(kwargs.get('band', ['.*']))
if 'id' in kwargs:
collection = collection.filterMetadata('id', 'equals',
kwargs['id'])
vis_params = get_vis_params(None, collection, **kwargs)
else:
vis_params = get_vis_params(None, None, **kwargs)
if 'year' in kwargs:
collection = collection.filterMetadata('year', 'equals',
int(kwargs['year']))
image = ee.Image(collection.reduce(reducer))
elif 'image' in kwargs:
image = ee.Image(kwargs['image'])
image = image.select(kwargs.get('band', ['.*']))
vis_params = get_vis_params(image, None, **kwargs)
else:
raise ee.EEException("No image or collection specified")
mapid = image.getMapId(vis_params=vis_params)
del mapid['image']
return mapid
def run(self, args, config):
properties = _decode_property_flags(args)
config.ee_init()
if not properties:
raise ee.EEException('No properties specified.')
ee.data.setAssetProperties(args.asset_id, properties)
def _check_valid_files(filenames):
"""Returns true if the given filenames are valid upload file URIs."""
for filename in filenames:
if not filename.startswith('gs://'):
raise ee.EEException('Invalid Cloud Storage URL: ' + filename)
def manifest_from_args(self, args, config):
"""Constructs an upload manifest from the command-line flags."""
if args.manifest:
with open(args.manifest) as fh:
return json.loads(fh.read())
if not args.asset_id:
raise ValueError('Flag --asset_id must be set.')
_check_valid_files(args.src_file)
source_files = list(utils.expand_gcs_wildcards(args.src_file))
if len(source_files) != 1:
raise ValueError('Exactly one file must be specified.')
if config.use_cloud_api:
properties = _decode_property_flags(args)
args.asset_id = ee.data.convert_asset_id_to_asset_name(args.asset_id)
source = {'uris': source_files}
if args.max_error:
source['maxErrorMeters'] = args.max_error
if args.max_vertices:
source['maxVertices'] = args.max_vertices
if args.max_failed_features:
raise ee.EEException(
'--max_failed_features is not supported with the Cloud API')
manifest = {
'name': args.asset_id,
'sources': [source],
'properties': properties
}
# pylint:disable=g-explicit-bool-comparison
if args.time_start is not None and args.time_start != '':
manifest['start_time'] = _cloud_timestamp_for_timestamp_ms(
args.time_start)
if args.time_end is not None and args.time_end != '':
manifest['end_time'] = _cloud_timestamp_for_timestamp_ms(args.time_end)
# pylint:enable=g-explicit-bool-comparison
return manifest
# non-cloud API section
source = {'primaryPath': source_files[0]}
if args.max_error:
source['max_error'] = args.max_error
if args.max_vertices:
source['max_vertices'] = args.max_vertices
if args.max_failed_features:
source['max_failed_features'] = args.max_failed_features
if args.crs:
source['crs'] = args.crs
if args.geodesic:
source['geodesic'] = args.geodesic
if args.primary_geometry_column:
source['primary_geometry_column'] = args.primary_geometry_column
if args.x_column:
source['x_column'] = args.x_column
if args.y_column:
source['y_column'] = args.y_column
if args.date_format:
source['date_format'] = args.date_format
if args.csv_delimiter:
source['csv_delimiter'] = args.csv_delimiter
if args.csv_qualifier:
source['csv_qualifier'] = args.csv_qualifier
return {
'id': args.asset_id,
'sources': [source]
}
