def run(self, args, config): """Starts the upload task, and waits for completion if requested.""" _check_valid_files(args.src_files) config.ee_init() if args.last_band_alpha and args.nodata_value: raise ValueError( 'last_band_alpha and nodata_value are mutually exclusive.') properties = _decode_property_flags(args) request = {'id': args.asset_id, 'properties': properties} source_files = utils.expand_gcs_wildcards(args.src_files) sources = [{'primaryPath': source} for source in source_files] tileset = {'sources': sources} if args.last_band_alpha: tileset['bandMappings'] = [{ 'fileBandIndex': -1, 'maskForAllBands': True }] request['tilesets'] = [tileset] if args.pyramiding_policy: if len(args.pyramiding_policy) == 1: request['pyramidingPolicy'] = args.pyramiding_policy[0].upper() else: bands = [] for index, policy in enumerate(args.pyramiding_policy): bands.append({ 'id': index, 'pyramidingPolicy': policy.upper() }) request['bands'] = bands if args.nodata_value: if len(args.nodata_value) == 1: request['missingData'] = {'value': args.nodata_value[0]} else: if 'bands' in request: if len(request['bands']) != len(args.nodata_value): raise ValueError( 'Inconsistent number of bands: {} and {}'.format( args.pyramiding_policy, args.nodata_value)) else: request['bands'] = [] bands = request['bands'] for index, nodata in enumerate(args.nodata_value): if index < len(bands): bands[index]['missingData'] = {'value': nodata} else: bands.append({ 'id': index, 'missingData': { 'value': nodata } }) _upload(args, request, ee.data.startIngestion)
def run(self, args, config): """Starts the upload task, and waits for completion if requested.""" _check_valid_files(args.src_file) config.ee_init() source_files = list(utils.expand_gcs_wildcards(args.src_file)) if len(source_files) != 1: raise ValueError('Exactly one file must be specified.') request = { 'id': args.asset_id, 'sources': [{'primaryPath': source_files[0]}] } _upload(args, request, ee.data.startTableIngestion)
def run(self, args, config): """Starts the upload task, and waits for completion if requested.""" _check_valid_files(args.src_files) config.ee_init() if args.last_band_alpha and args.nodata_value: raise ValueError( 'last_band_alpha and nodata_value are mutually exclusive.') properties = _decode_property_flags(args) source_files = utils.expand_gcs_wildcards(args.src_files) bands = args.bands if args.pyramiding_policy and len(args.pyramiding_policy) != 1: bands = self._check_num_bands(bands, len(args.pyramiding_policy), 'pyramiding_policy') if args.nodata_value and len(args.nodata_value) != 1: bands = self._check_num_bands(bands, len(args.nodata_value), 'nodata_value') request = { 'id': args.asset_id, 'properties': properties } sources = [{'primaryPath': source} for source in source_files] tileset = {'sources': sources} if args.last_band_alpha: tileset['fileBands'] = [{'fileBandIndex': -1, 'maskForAllBands': True}] request['tilesets'] = [tileset] if bands: request['bands'] = [{'id': name} for name in bands] if args.pyramiding_policy: if len(args.pyramiding_policy) == 1: request['pyramidingPolicy'] = args.pyramiding_policy[0] else: for index, policy in enumerate(args.pyramiding_policy): request['bands'][index]['pyramidingPolicy'] = policy if args.nodata_value: if len(args.nodata_value) == 1: request['missingData'] = {'value': args.nodata_value[0]} else: for index, nodata in enumerate(args.nodata_value): request['bands'][index]['missingData'] = {'value': nodata} if args.crs: request['crs'] = args.crs _upload(args, request, ee.data.startIngestion)
def run(self, args, config): """Starts the upload task, and waits for completion if requested.""" _check_valid_files(args.src_files) config.ee_init() if args.last_band_alpha and args.nodata_value: raise ValueError( 'last_band_alpha and nodata_value are mutually exclusive.') properties = _decode_property_flags(args) request = { 'id': args.asset_id, 'properties': properties } source_files = utils.expand_gcs_wildcards(args.src_files) sources = [{'primaryPath': source} for source in source_files] tileset = {'sources': sources} if args.last_band_alpha: tileset['fileBands'] = [{'fileBandIndex': -1, 'maskForAllBands': True}] request['tilesets'] = [tileset] if args.bands: request['bands'] = [{'id': name} for name in args.bands] if args.pyramiding_policy: if len(args.pyramiding_policy) == 1: request['pyramidingPolicy'] = args.pyramiding_policy[0].upper() else: self._check_num_bands(request, len(args.pyramiding_policy), 'pyramiding_policy') for index, policy in enumerate(args.pyramiding_policy): request['bands'][index]['pyramidingPolicy'] = policy.upper() if args.nodata_value: if len(args.nodata_value) == 1: request['missingData'] = {'value': args.nodata_value[0]} else: self._check_num_bands(request, len(args.nodata_value), 'nodata_value') for index, nodata in enumerate(args.nodata_value): request['bands'][index]['missingData'] = {'value': nodata} if args.crs: request['crs'] = args.crs _upload(args, request, ee.data.startIngestion)
def manifest_from_args(self, args, config): """Constructs an upload manifest from the command-line flags.""" if args.manifest: with open(args.manifest) as fh: return json.loads(fh.read()) if not args.asset_id: raise ValueError('Flag --asset_id must be set.') _check_valid_files(args.src_file) source_files = list(utils.expand_gcs_wildcards(args.src_file)) if len(source_files) != 1: raise ValueError('Exactly one file must be specified.') if config.use_cloud_api: properties = _decode_property_flags(args) args.asset_id = ee.data.convert_asset_id_to_asset_name(args.asset_id) source = {'uris': source_files} if args.max_error: source['maxErrorMeters'] = args.max_error if args.max_vertices: source['maxVertices'] = args.max_vertices if args.max_failed_features: raise ee.EEException( '--max_failed_features is not supported with the Cloud API') manifest = { 'name': args.asset_id, 'sources': [source], 'properties': properties } # pylint:disable=g-explicit-bool-comparison if args.time_start is not None and args.time_start != '': manifest['start_time'] = _cloud_timestamp_for_timestamp_ms( args.time_start) if args.time_end is not None and args.time_end != '': manifest['end_time'] = _cloud_timestamp_for_timestamp_ms(args.time_end) # pylint:enable=g-explicit-bool-comparison return manifest # non-cloud API section source = {'primaryPath': source_files[0]} if args.max_error: source['max_error'] = args.max_error if args.max_vertices: source['max_vertices'] = args.max_vertices if args.max_failed_features: source['max_failed_features'] = args.max_failed_features if args.crs: source['crs'] = args.crs if args.geodesic: source['geodesic'] = args.geodesic if args.primary_geometry_column: source['primary_geometry_column'] = args.primary_geometry_column if args.x_column: source['x_column'] = args.x_column if args.y_column: source['y_column'] = args.y_column if args.date_format: source['date_format'] = args.date_format if args.csv_delimiter: source['csv_delimiter'] = args.csv_delimiter if args.csv_qualifier: source['csv_qualifier'] = args.csv_qualifier return { 'id': args.asset_id, 'sources': [source] }
def manifest_from_args(self, args, config): """Constructs an upload manifest from the command-line flags.""" if args.manifest: with open(args.manifest) as fh: return json.loads(fh.read()) if not args.asset_id: raise ValueError('Flag --asset_id must be set.') _check_valid_files(args.src_files) if args.last_band_alpha and args.nodata_value: raise ValueError( 'last_band_alpha and nodata_value are mutually exclusive.') properties = _decode_property_flags(args) source_files = utils.expand_gcs_wildcards(args.src_files) if not source_files: raise ValueError('At least one file must be specified.') bands = args.bands if args.pyramiding_policy and len(args.pyramiding_policy) != 1: bands = self._check_num_bands(bands, len(args.pyramiding_policy), 'pyramiding_policy') if args.nodata_value and len(args.nodata_value) != 1: bands = self._check_num_bands(bands, len(args.nodata_value), 'nodata_value') if config.use_cloud_api: args.asset_id = ee.data.convert_asset_id_to_asset_name(args.asset_id) tileset = { 'id': 'ts', 'sources': [{'uris': [source]} for source in source_files] } manifest = { 'name': args.asset_id, 'properties': properties, 'tilesets': [tileset] } # pylint:disable=g-explicit-bool-comparison if args.time_start is not None and args.time_start != '': manifest['start_time'] = _cloud_timestamp_for_timestamp_ms( args.time_start) if args.time_end is not None and args.time_end != '': manifest['end_time'] = _cloud_timestamp_for_timestamp_ms(args.time_end) # pylint:enable=g-explicit-bool-comparison if bands: file_bands = [] for i, band in enumerate(bands): file_bands.append({ 'id': band, 'tilesetId': tileset['id'], 'tilesetBandIndex': i }) manifest['bands'] = file_bands if args.pyramiding_policy: if len(args.pyramiding_policy) == 1: manifest['pyramidingPolicy'] = args.pyramiding_policy[0] else: for index, policy in enumerate(args.pyramiding_policy): file_bands[index]['pyramidingPolicy'] = policy if args.nodata_value: if len(args.nodata_value) == 1: manifest['missingData'] = {'values': [args.nodata_value[0]]} else: for index, value in enumerate(args.nodata_value): file_bands[index]['missingData'] = {'values': [value]} if args.last_band_alpha: manifest['maskBands'] = {'tilesetId': tileset['id']} return manifest # non-cloud API section properties.update(_decode_timestamp_flags(args)) manifest = { 'id': args.asset_id, 'properties': properties } sources = [{'primaryPath': source} for source in source_files] tileset = {'sources': sources} if args.last_band_alpha: tileset['fileBands'] = [{'fileBandIndex': -1, 'maskForAllBands': True}] manifest['tilesets'] = [tileset] if bands: manifest['bands'] = [{'id': name} for name in bands] if args.pyramiding_policy: if len(args.pyramiding_policy) == 1: manifest['pyramidingPolicy'] = args.pyramiding_policy[0] else: for index, policy in enumerate(args.pyramiding_policy): manifest['bands'][index]['pyramidingPolicy'] = policy if args.nodata_value: if len(args.nodata_value) == 1: manifest['missingData'] = {'value': args.nodata_value[0]} else: for index, nodata in enumerate(args.nodata_value): manifest['bands'][index]['missingData'] = {'value': nodata} if args.crs: manifest['crs'] = args.crs return manifest