def main(): etl() input_file_path = "{}/{}".format(ec2_input_path, INPUT_FILE_NAME) nc_fid = netCDF4.Dataset(input_file_path, 'r') nc_attrs, nc_dims, nc_vars = aqueduct3.ncdump(nc_fid) print(nc_attrs, nc_dims, nc_vars) y_dimension = nc_fid.variables["lat"].shape[0] x_dimension = nc_fid.variables["lon"].shape[0] default_geotransform, default_geoprojection = aqueduct3.get_global_georeference( np.ones([y_dimension, x_dimension])) Z = nc_fid.variables[EXPORT_VARIABLE][:, :] Z = np.flipud(Z) aqueduct3.write_geotiff(output_file_path, default_geotransform, default_geoprojection, Z, nodata_value=-9999, datatype=gdal.GDT_Int32) get_ipython().system( 'aws s3 cp {ec2_output_path} {s3_output_path} --recursive')
default_geotransform, default_geoprojection = aqueduct3.get_global_georeference( np.ones([Y_DIMENSION_5MIN, X_DIMENSION_5MIN])) for root, dirs, file_names in os.walk(ec2_input_path): for file_name in file_names: print(file_name) base, extension = file_name.split(".") output_path = ec2_output_path + base + ".tif" input_path = os.path.join(root, file_name) xsize, ysize, geotransform, geoproj, Z = aqueduct3.read_gdal_file( input_path) Z[Z < -9990] = -9999 Z[Z > 1e19] = -9999 aqueduct3.write_geotiff(output_path, default_geotransform, default_geoprojection, Z, nodata_value=-9999, datatype=gdal.GDT_Float32) # In[8]: for old_name, new_name in RENAME_DICT.items(): get_ipython().system( 'mv {ec2_output_path}{old_name} {ec2_output_path}{new_name}') assert len(new_name) < 100, "new key should not exceed 100 characters" # In[9]: get_ipython().system('gsutil -m cp {ec2_output_path}/*.tif {gcs_output_path}') # In[10]:
files = os.listdir(ec2_input_path) # In[7]: # Convert geotiff to all Float64 and with -9999 as NoData value for one_file in files: input_file_path = os.path.join(ec2_input_path,one_file) output_file_path = os.path.join(ec2_output_path,one_file) xsize,ysize,geotransform,geoproj,Z = aqueduct3.read_gdal_file(input_file_path) Z[Z<-9990]= NODATA_VALUE datatype=gdal.GDT_Float64 aqueduct3.write_geotiff(output_file_path,geotransform,geoproj,Z,NODATA_VALUE,datatype) # In[8]: get_ipython().system('gsutil cp -r {ec2_output_path} {GCS_OUTPUT_PATH}') # In[9]: aqueduct3.earthengine.create_ee_folder_recursive(ee_output_path) # In[10]:
input_path = os.path.join(root, file_name) xsize, ysize, geotransform, geoproj, Z = aqueduct3.read_gdal_file( input_path) Z[Z < -9990] = -9999 Z[Z > 1e19] = -9999 if file_name == "global_lddsound_numpad_05min.map" or file_name == "global_outletendorheicbasins_boolean_05min.map": nodata_value = 255 datatype = gdal.GDT_Int32 # Could probably use byte type as well. else: nodata_value = -9999 datatype = gdal.GDT_Float32 aqueduct3.write_geotiff(output_path, default_geotransform, default_geoprojection, Z, nodata_value, datatype) # In[10]: get_ipython().system('gsutil -m cp {ec2_output_path}/*.tif {gcs_output_path}') # In[11]: get_ipython().system( 'aws s3 cp {ec2_output_path} {s3_output_path} --recursive') # In[12]: end = datetime.datetime.now() elapsed = end - start