processed_data_file = '{}_edit.tif'.format(dataset_name) gdal_translate_command = ( 'gdal_translate {} {} -co COMPRESS=LZW -co BIGTIFF=YES').format( vrt_name, processed_data_file) print('Merging geotiffs') subprocess.check_output(gdal_translate_command, shell=True) ''' Upload processed data to Google Earth Engine ''' # set up Google Cloud Storage project and bucket objects gcs_client = storage.Client(os.environ.get("CLOUDSDK_CORE_PROJECT")) gcs_bucket = gcs_client.bucket(os.environ.get("GEE_STAGING_BUCKET")) logger.info('Uploading processed data to Google Cloud Storage.') gcs_uris = util_cloud.gcs_upload(processed_data_file, dataset_name, gcs_bucket=gcs_bucket) logger.info('Uploading processed data to Google Earth Engine.') # initialize ee and eeUtil modules for uploading to Google Earth Engine auth = ee.ServiceAccountCredentials( os.getenv('GEE_SERVICE_ACCOUNT'), os.getenv('GOOGLE_APPLICATION_CREDENTIALS')) ee.Initialize(auth) # set pyramiding policy for GEE upload pyramiding_policy = 'MEAN' #check # Upload processed data file to GEE asset_name = f'projects/resource-watch-gee/{dataset_name}'
'Merge masked, single-band GeoTIFFs into single, multiband GeoTIFF.') # merge multiple masked single-band tifs as layers in single multiband tif multitif = os.path.abspath(os.path.join(data_dir, dataset_name + '.tif')) util_files.merge_geotiffs(masked_tifs, multitif) processed_data_file = multitif ''' Upload processed data to Google Earth Engine ''' logger.info('Uploading processed data to Google Cloud Storage.') # set up Google Cloud Storage project and bucket objects gcsClient = storage.Client(os.environ.get("CLOUDSDK_CORE_PROJECT")) gcsBucket = gcsClient.bucket(os.environ.get("GEE_STAGING_BUCKET")) # upload files to Google Cloud Storage gcs_uris = util_cloud.gcs_upload(multitif, dataset_name, gcs_bucket=gcsBucket) logger.info('Uploading processed data to Google Earth Engine.') # initialize ee and eeUtil modules for uploading to Google Earth Engine auth = ee.ServiceAccountCredentials( os.getenv('GEE_SERVICE_ACCOUNT'), os.getenv('GOOGLE_APPLICATION_CREDENTIALS')) ee.Initialize(auth) # set pyramiding policy for GEE upload pyramiding_policy = 'MEAN' #check missing_data_values = [-128.0] # set asset name to be used in GEE asset_name = f'projects/resource-watch-gee/{dataset_name}'
task_id = [] # Upload processed data files to GEE # if upload is timing out, uncomment the following lines # storage.blob._DEFAULT_CHUNKSIZE = 10 * 1024* 1024 # 10 MB # storage.blob._MAX_MULTIPART_SIZE = 10 * 1024* 1024 # 10 MB #loop though the processed data files to upload to Google Cloud Storage and Google Earth Engine for i in range(len(data_dict['tifs'])): logger.info('Uploading ' + data_dict['processed_data_file'][i] + ' to Google Cloud Storage.') # upload files to Google Cloud Storage gcs_uri = util_cloud.gcs_upload(data_dict['raw_data_file'][i], dataset_name, gcs_bucket=gcsBucket) logger.info('Uploading ' + data_dict['processed_data_file'][i] + ' Google Earth Engine.') # generate an asset name for the current file by using the filename (minus the file type extension) file_name = data_dict['processed_data_file'][i].split('.')[0].split('/')[1] asset_name = f'projects/resource-watch-gee/{dataset_name}/{file_name}' # create the band manifest for this asset tileset_id = data_dict['tifs'][i].split('.')[0] missing_values = [ 65533, ] mf_bands = [{ 'id': band_id,
# sum and export annual total precipitation as tif file for i in range(int(src0.meta['count']/12)): with rasterio.open(processed_data_annual[sum(n_layers[:id])+i], 'w', **meta) as dst: dst.write_band(1,np.sum(src0.read(range((i*12+1), (i*12+13))), axis = 0)) ''' Upload processed data to Google Earth Engine ''' logger.info('Uploading processed data to Google Cloud Storage.') # set up Google Cloud Storage project and bucket objects gcsClient = storage.Client(os.environ.get("CLOUDSDK_CORE_PROJECT")) gcsBucket = gcsClient.bucket(os.environ.get("GEE_STAGING_BUCKET")) # upload files to Google Cloud Storage gcs_uris= util_cloud.gcs_upload(processed_data_annual, dataset_name, gcs_bucket=gcsBucket) logger.info('Uploading processed data to Google Earth Engine.') # initialize ee and eeUtil modules for uploading to Google Earth Engine auth = ee.ServiceAccountCredentials(os.getenv('GEE_SERVICE_ACCOUNT'), os.getenv('GOOGLE_APPLICATION_CREDENTIALS')) ee.Initialize(auth) # set pyramiding policy for GEE upload pyramiding_policy = 'MEAN' #check # create an image collection where we will put the processed data files in GEE image_collection = f'projects/resource-watch-gee/Facebook/PrecipitationAnalysis/GPCC_annual' ee.data.createAsset({'type': 'ImageCollection'}, image_collection) # set image collection's privacy to public acl = {"all_users_can_read": True}
ee.Initialize(auth) # Create an image collection where we will put the processed data files in GEE image_collection = f'projects/resource-watch-gee/{dataset_name}' ee.data.createAsset({'type': 'ImageCollection'}, image_collection) # set image collection's privacy to public acl = {"all_users_can_read": True} ee.data.setAssetAcl(image_collection, acl) logging.info('Privacy set to public.') # loop through data files in data_dict for data_file in data_dict.keys(): # set image name as basename of the file image_name = os.path.basename(data_file.split('.')[0]) # upload to google cloud gcs_uris = util_cloud.gcs_upload(data_file, image_name, gcs_bucket=gcsBucket) # define asset name asset_name = 'projects/resource-watch-gee/{}/{}'.format( dataset_name, image_name) # define band dictionary for manifest upload, with the missing data, b1 as band name, and pyramiding pollicy as mean upload_data_dict = OrderedDict() upload_data_dict[image_name] = { 'missing_data': [ data_dict.get(data_file), ], 'pyramiding_policy': 'MEAN', 'band_ids': ['b1'] } # upload to google earth engine mf_bands = util_cloud.gee_manifest_bands(upload_data_dict,
y1 = y - 15 y2 = y + 15 for change in changes: # get file name f = raster_template(change, indicator, scenario, y1, y2, dataset) file = os.path.join(data_dir, f) dataset_name = f'{dataset}_{scenario}_{indicator}_{change}_{y1}_{y2}' ''' Upload processed data to Google Earth Engine ''' logger.info( 'Uploading processed data to Google Cloud Storage.') # upload local files to Google Cloud Storage gs_uris = util_cloud.gcs_upload( file, os.path.join(dataset, dataset_name)) logger.info( 'Uploading processed data to Google Earth Engine.') # set pyramiding policy for GEE upload pyramiding_policy = 'MEAN' # check # name bands according to variable names in original netcdf bands = ['q25', 'q50', 'q75'] # create manifests for upload to GEE band_manifest = [{ 'id': band, 'tileset_band_index': bands.index(band),