def main():
etl()
input_file_path = "{}/{}".format(ec2_input_path, INPUT_FILE_NAME)
nc_fid = netCDF4.Dataset(input_file_path, 'r')
nc_attrs, nc_dims, nc_vars = aqueduct3.ncdump(nc_fid)
print(nc_attrs, nc_dims, nc_vars)
y_dimension = nc_fid.variables["lat"].shape[0]
x_dimension = nc_fid.variables["lon"].shape[0]
default_geotransform, default_geoprojection = aqueduct3.get_global_georeference(
np.ones([y_dimension, x_dimension]))
Z = nc_fid.variables[EXPORT_VARIABLE][:, :]
Z = np.flipud(Z)
aqueduct3.write_geotiff(output_file_path,
default_geotransform,
default_geoprojection,
Z,
nodata_value=-9999,
datatype=gdal.GDT_Int32)
get_ipython().system(
'aws s3 cp {ec2_output_path} {s3_output_path} --recursive')
default_geotransform, default_geoprojection = aqueduct3.get_global_georeference(
np.ones([Y_DIMENSION_5MIN, X_DIMENSION_5MIN]))
for root, dirs, file_names in os.walk(ec2_input_path):
for file_name in file_names:
print(file_name)
base, extension = file_name.split(".")
output_path = ec2_output_path + base + ".tif"
input_path = os.path.join(root, file_name)
xsize, ysize, geotransform, geoproj, Z = aqueduct3.read_gdal_file(
input_path)
Z[Z < -9990] = -9999
Z[Z > 1e19] = -9999
aqueduct3.write_geotiff(output_path,
default_geotransform,
default_geoprojection,
Z,
nodata_value=-9999,
datatype=gdal.GDT_Float32)
# In[8]:
for old_name, new_name in RENAME_DICT.items():
get_ipython().system(
'mv {ec2_output_path}{old_name} {ec2_output_path}{new_name}')
assert len(new_name) < 100, "new key should not exceed 100 characters"
# In[9]:
get_ipython().system('gsutil -m cp {ec2_output_path}/*.tif {gcs_output_path}')
# In[10]:
files = os.listdir(ec2_input_path)
# In[7]:
# Convert geotiff to all Float64 and with -9999 as NoData value
for one_file in files:
input_file_path = os.path.join(ec2_input_path,one_file)
output_file_path = os.path.join(ec2_output_path,one_file)
xsize,ysize,geotransform,geoproj,Z = aqueduct3.read_gdal_file(input_file_path)
Z[Z<-9990]= NODATA_VALUE
datatype=gdal.GDT_Float64
aqueduct3.write_geotiff(output_file_path,geotransform,geoproj,Z,NODATA_VALUE,datatype)
# In[8]:
get_ipython().system('gsutil cp -r {ec2_output_path} {GCS_OUTPUT_PATH}')
# In[9]:
aqueduct3.earthengine.create_ee_folder_recursive(ee_output_path)
# In[10]:
input_path = os.path.join(root, file_name)
xsize, ysize, geotransform, geoproj, Z = aqueduct3.read_gdal_file(
input_path)
Z[Z < -9990] = -9999
Z[Z > 1e19] = -9999
if file_name == "global_lddsound_numpad_05min.map" or file_name == "global_outletendorheicbasins_boolean_05min.map":
nodata_value = 255
datatype = gdal.GDT_Int32 # Could probably use byte type as well.
else:
nodata_value = -9999
datatype = gdal.GDT_Float32
aqueduct3.write_geotiff(output_path, default_geotransform,
default_geoprojection, Z, nodata_value,
datatype)
# In[10]:
get_ipython().system('gsutil -m cp {ec2_output_path}/*.tif {gcs_output_path}')
# In[11]:
get_ipython().system(
'aws s3 cp {ec2_output_path} {s3_output_path} --recursive')
# In[12]:
end = datetime.datetime.now()
elapsed = end - start