processed_data_file = '{}_edit.tif'.format(dataset_name)
gdal_translate_command = (
'gdal_translate {} {} -co COMPRESS=LZW -co BIGTIFF=YES').format(
vrt_name, processed_data_file)
print('Merging geotiffs')
subprocess.check_output(gdal_translate_command, shell=True)
'''
Upload processed data to Google Earth Engine
'''
# set up Google Cloud Storage project and bucket objects
gcs_client = storage.Client(os.environ.get("CLOUDSDK_CORE_PROJECT"))
gcs_bucket = gcs_client.bucket(os.environ.get("GEE_STAGING_BUCKET"))
logger.info('Uploading processed data to Google Cloud Storage.')
gcs_uris = util_cloud.gcs_upload(processed_data_file,
dataset_name,
gcs_bucket=gcs_bucket)
logger.info('Uploading processed data to Google Earth Engine.')
# initialize ee and eeUtil modules for uploading to Google Earth Engine
auth = ee.ServiceAccountCredentials(
os.getenv('GEE_SERVICE_ACCOUNT'),
os.getenv('GOOGLE_APPLICATION_CREDENTIALS'))
ee.Initialize(auth)
# set pyramiding policy for GEE upload
pyramiding_policy = 'MEAN' #check
# Upload processed data file to GEE
asset_name = f'projects/resource-watch-gee/{dataset_name}'
'Merge masked, single-band GeoTIFFs into single, multiband GeoTIFF.')
# merge multiple masked single-band tifs as layers in single multiband tif
multitif = os.path.abspath(os.path.join(data_dir, dataset_name + '.tif'))
util_files.merge_geotiffs(masked_tifs, multitif)
processed_data_file = multitif
'''
Upload processed data to Google Earth Engine
'''
logger.info('Uploading processed data to Google Cloud Storage.')
# set up Google Cloud Storage project and bucket objects
gcsClient = storage.Client(os.environ.get("CLOUDSDK_CORE_PROJECT"))
gcsBucket = gcsClient.bucket(os.environ.get("GEE_STAGING_BUCKET"))
# upload files to Google Cloud Storage
gcs_uris = util_cloud.gcs_upload(multitif, dataset_name, gcs_bucket=gcsBucket)
logger.info('Uploading processed data to Google Earth Engine.')
# initialize ee and eeUtil modules for uploading to Google Earth Engine
auth = ee.ServiceAccountCredentials(
os.getenv('GEE_SERVICE_ACCOUNT'),
os.getenv('GOOGLE_APPLICATION_CREDENTIALS'))
ee.Initialize(auth)
# set pyramiding policy for GEE upload
pyramiding_policy = 'MEAN' #check
missing_data_values = [-128.0]
# set asset name to be used in GEE
asset_name = f'projects/resource-watch-gee/{dataset_name}'
task_id = []
# Upload processed data files to GEE
# if upload is timing out, uncomment the following lines
# storage.blob._DEFAULT_CHUNKSIZE = 10 * 1024* 1024 # 10 MB
# storage.blob._MAX_MULTIPART_SIZE = 10 * 1024* 1024 # 10 MB
#loop though the processed data files to upload to Google Cloud Storage and Google Earth Engine
for i in range(len(data_dict['tifs'])):
logger.info('Uploading ' + data_dict['processed_data_file'][i] +
' to Google Cloud Storage.')
# upload files to Google Cloud Storage
gcs_uri = util_cloud.gcs_upload(data_dict['raw_data_file'][i],
dataset_name,
gcs_bucket=gcsBucket)
logger.info('Uploading ' + data_dict['processed_data_file'][i] +
' Google Earth Engine.')
# generate an asset name for the current file by using the filename (minus the file type extension)
file_name = data_dict['processed_data_file'][i].split('.')[0].split('/')[1]
asset_name = f'projects/resource-watch-gee/{dataset_name}/{file_name}'
# create the band manifest for this asset
tileset_id = data_dict['tifs'][i].split('.')[0]
missing_values = [
65533,
]
mf_bands = [{
'id': band_id,
# sum and export annual total precipitation as tif file
for i in range(int(src0.meta['count']/12)):
with rasterio.open(processed_data_annual[sum(n_layers[:id])+i], 'w', **meta) as dst:
dst.write_band(1,np.sum(src0.read(range((i*12+1), (i*12+13))), axis = 0))
'''
Upload processed data to Google Earth Engine
'''
logger.info('Uploading processed data to Google Cloud Storage.')
# set up Google Cloud Storage project and bucket objects
gcsClient = storage.Client(os.environ.get("CLOUDSDK_CORE_PROJECT"))
gcsBucket = gcsClient.bucket(os.environ.get("GEE_STAGING_BUCKET"))
# upload files to Google Cloud Storage
gcs_uris= util_cloud.gcs_upload(processed_data_annual, dataset_name, gcs_bucket=gcsBucket)
logger.info('Uploading processed data to Google Earth Engine.')
# initialize ee and eeUtil modules for uploading to Google Earth Engine
auth = ee.ServiceAccountCredentials(os.getenv('GEE_SERVICE_ACCOUNT'), os.getenv('GOOGLE_APPLICATION_CREDENTIALS'))
ee.Initialize(auth)
# set pyramiding policy for GEE upload
pyramiding_policy = 'MEAN' #check
# create an image collection where we will put the processed data files in GEE
image_collection = f'projects/resource-watch-gee/Facebook/PrecipitationAnalysis/GPCC_annual'
ee.data.createAsset({'type': 'ImageCollection'}, image_collection)
# set image collection's privacy to public
acl = {"all_users_can_read": True}
ee.Initialize(auth)
# Create an image collection where we will put the processed data files in GEE
image_collection = f'projects/resource-watch-gee/{dataset_name}'
ee.data.createAsset({'type': 'ImageCollection'}, image_collection)
# set image collection's privacy to public
acl = {"all_users_can_read": True}
ee.data.setAssetAcl(image_collection, acl)
logging.info('Privacy set to public.')
# loop through data files in data_dict
for data_file in data_dict.keys():
# set image name as basename of the file
image_name = os.path.basename(data_file.split('.')[0])
# upload to google cloud
gcs_uris = util_cloud.gcs_upload(data_file,
image_name,
gcs_bucket=gcsBucket)
# define asset name
asset_name = 'projects/resource-watch-gee/{}/{}'.format(
dataset_name, image_name)
# define band dictionary for manifest upload, with the missing data, b1 as band name, and pyramiding pollicy as mean
upload_data_dict = OrderedDict()
upload_data_dict[image_name] = {
'missing_data': [
data_dict.get(data_file),
],
'pyramiding_policy': 'MEAN',
'band_ids': ['b1']
}
# upload to google earth engine
mf_bands = util_cloud.gee_manifest_bands(upload_data_dict,
y1 = y - 15
y2 = y + 15
for change in changes:
# get file name
f = raster_template(change, indicator, scenario, y1, y2,
dataset)
file = os.path.join(data_dir, f)
dataset_name = f'{dataset}_{scenario}_{indicator}_{change}_{y1}_{y2}'
'''
Upload processed data to Google Earth Engine
'''
logger.info(
'Uploading processed data to Google Cloud Storage.')
# upload local files to Google Cloud Storage
gs_uris = util_cloud.gcs_upload(
file, os.path.join(dataset, dataset_name))
logger.info(
'Uploading processed data to Google Earth Engine.')
# set pyramiding policy for GEE upload
pyramiding_policy = 'MEAN' # check
# name bands according to variable names in original netcdf
bands = ['q25', 'q50', 'q75']
# create manifests for upload to GEE
band_manifest = [{
'id':
band,
'tileset_band_index':
bands.index(band),
