def convert_data(args):
"""
"""
# Unpack the arguments
latlim, lonlim, date, \
product, \
username, password, apitoken, \
url_server, url_dir, \
remote_fname, temp_fname, local_fname,\
remote_file, temp_file, local_file,\
y_id, x_id, pixel_size, pixel_w, pixel_h, \
data_ndv, data_type, data_multiplier, data_variable = args
# Define local variable
status_cod = -1
nparts = 4
# post-process remote (from server)
# -> temporary (unzip)
# -> local (gis)
msg = 'Converting "{f}"'.format(f=local_file)
print('\33[94m{}\33[0m'.format(msg))
__this.Log.write(datetime.datetime.now(), msg=msg)
# --------- #
# Load data #
# --------- #
# From downloaded remote file
# From generated temporary file
for i in range(0, nparts):
# Band number of the grib data which is converted in .nc
temp_file_part = temp_file.format(dtime=date, ipart=str(i + 1))
temp_file_band = (int(date.strftime('%d')) - 1) * 28 + (i + 1) * 7
Convert_grb2_to_nc(remote_file, temp_file_part, temp_file_band)
# Generate temporary files
data_variable = 'Band1'
data_sum = np.zeros([y_id[1] - y_id[0], x_id[1] - x_id[0]])
for i in range(0, nparts):
temp_file_part = temp_file.format(dtime=date, ipart=str(i + 1))
fh = Dataset(temp_file_part, mode='r')
data_group = product['data']['ftype']['r'].split('.')
if len(data_group) > 1:
data_raw = fh.groups[data_group[1]].variables[data_variable]
else:
data_raw = fh.variables[data_variable]
if 'missing_value' in data_raw.ncattrs():
# ASCAT
data_raw_missing = data_raw.getncattr('missing_value')
elif 'CodeMissingValue' in data_raw.ncattrs():
# GPM
data_raw_missing = data_raw.getncattr('CodeMissingValue')
else:
data_raw_missing = data_raw.getncattr('_FillValue')
if 'scale_factor' in data_raw.ncattrs():
data_raw_scale = data_raw.getncattr('scale_factor')
else:
data_raw_scale = 1.0
# Convert meta data to float
if np.logical_or(isinstance(data_raw_missing, str),
isinstance(data_raw_scale, str)):
data_raw_missing = float(data_raw_missing)
data_raw_scale = float(data_raw_scale)
# --------- #
# Clip data #
# --------- #
# get data to 2D matrix
data_raw_tmp = np.zeros([pixel_h, pixel_w])
data_raw_tmp[:,
0:int(pixel_w / 2)] = data_raw[:,
int(pixel_w / 2):pixel_w]
data_raw_tmp[:,
int(pixel_w / 2):pixel_w] = data_raw[:,
0:int(pixel_w / 2)]
data_tmp = data_raw_tmp[y_id[0]:y_id[1], x_id[0]:x_id[1]]
# data_tmp = np.squeeze(data_tmp, axis=0)
# check data type
# filled numpy.ma.MaskedArray as numpy.ndarray
if isinstance(data_tmp, np.ma.MaskedArray):
data = data_tmp.filled()
else:
data = np.asarray(data_tmp)
# transfer matrix to GTiff matrix
# [w,n]--[e,n]
# | |
# [w,s]--[e,s]
# data = np.asarray(data)
# [w,s]--[e,s]
# | |
# [w,n]--[e,n]
data = np.flipud(data)
# [w,n]--[w,s]
# | |
# [e,n]--[e,s]
# data = np.transpose(a=data, axes=(1, 0))
# [w,s]--[w,n]
# | |
# [e,s]--[e,n]
# data = np.rot90(data, k=1, axes=(0, 1))
# close file
fh.close()
# ------- #
# Convert #
# ------- #
# scale, units
data = np.where(data == data_raw_missing, np.nan, data)
data = data * data_raw_scale * data_multiplier
# novalue data
data = np.where(np.isnan(data), data_ndv, data)
data_sum = data_sum + data
# calculate the average
data = data_sum / float(nparts)
# Save as GTiff
geo = [lonlim[0], pixel_size, 0, latlim[1], 0, -pixel_size]
Save_as_tiff(name=local_file, data=data, geo=geo, projection="WGS84")
if __this.conf['is_save_remote']:
pass
else:
path = os.path.dirname(os.path.realpath(remote_file))
if 'remote' != path[-6:]:
path = os.path.join(path, 'remote')
clean(path)
if __this.conf['is_save_temp']:
pass
else:
path = os.path.dirname(os.path.realpath(temp_file))
if 'temporary' != path[-9:]:
path = os.path.join(path, 'temporary')
clean(path)
status_cod = 0
return status_cod
def convert_data(args):
"""
"""
# Unpack the arguments
latlim, lonlim, date,\
product, \
username, password, apitoken, \
url_server, url_dir, \
remote_fname, temp_fname, local_fname,\
remote_file, temp_file, local_file,\
y_id, x_id, pixel_size, pixel_w, pixel_h, \
data_ndv, data_type, data_multiplier, data_variable = args
# Define local variable
status_cod = -1
# post-process remote (from server)
# -> temporary (unzip)
# -> local (gis)
msg = 'Converting "{f}"'.format(f=local_file)
print('\33[94m{}\33[0m'.format(msg))
__this.Log.write(datetime.datetime.now(), msg=msg)
# --------- #
# Load data #
# --------- #
# From downloaded remote file
# From generated temporary file
temp_file_part = temp_file.format(dtime=date)
temp_folder = os.path.dirname(temp_file_part)
# Generate temporary files
Extract_Data_zip(remote_file, temp_folder)
data_raw = Open_tiff_array(temp_file_part)
# Convert meta data to float
# if np.logical_or(isinstance(data_raw_missing, str),
# isinstance(data_raw_scale, str)):
# data_raw_missing = float(data_raw_missing)
# data_raw_scale = float(data_raw_scale)
# --------- #
# Clip data #
# --------- #
# get data to 2D matrix
data_tmp = data_raw[y_id[0]:y_id[1], x_id[0]:x_id[1]]
# check data type
# filled numpy.ma.MaskedArray as numpy.ndarray
if isinstance(data_tmp, np.ma.MaskedArray):
data = data_tmp.filled()
else:
data = np.asarray(data_tmp)
# transfer matrix to GTiff matrix
# [w,n]--[e,n]
# | |
# [w,s]--[e,s]
data = np.asarray(data)
# [w,s]--[e,s]
# | |
# [w,n]--[e,n]
# data = np.flipud(data)
# [w,n]--[w,s]
# | |
# [e,n]--[e,s]
# data = np.transpose(a=data, axes=(1, 0))
# [w,s]--[w,n]
# | |
# [e,s]--[e,n]
# data = np.rot90(data, k=1, axes=(0, 1))
# close file
# fh.close()
# ------- #
# Convert #
# ------- #
# scale, units
data = np.where(data < 0, np.nan, data)
data = data * data_multiplier
# novalue data
data = np.where(np.isnan(data), data_ndv, data)
# ------------ #
# Saveas GTiff #
# ------------ #
geo = [lonlim[0], pixel_size, 0,
latlim[1], 0, -pixel_size]
Save_as_tiff(name=local_file, data=data, geo=geo, projection="WGS84")
path = os.path.dirname(os.path.realpath(remote_file))
if 'remote' != path[-6:]:
path = os.path.join(path, 'remote')
clean(path)
path = os.path.dirname(os.path.realpath(temp_file))
if 'temporary' != path[-9:]:
path = os.path.join(path, 'temporary')
clean(path)
status_cod = 0
return status_cod
def convert_data(args):
"""
"""
# Unpack the arguments
latlim, lonlim, date, \
product, \
username, password, apitoken, \
url_server, url_dir, \
remote_fname, temp_fname, local_fname,\
remote_file, temp_file, local_file,\
y_id, x_id, pixel_size, pixel_w, pixel_h, \
data_ndv, data_type, data_multiplier, data_variable = args
# Define local variable
status_cod = -1
# post-process remote (from server)
# -> temporary (unzip)
# -> local (gis)
msg = 'Converting "{f}"'.format(f=local_file)
print('\33[94m{}\33[0m'.format(msg))
__this.Log.write(datetime.datetime.now(), msg=msg)
# --------- #
# Load data #
# --------- #
# From downloaded remote file
fh = Dataset(remote_file, mode='r')
data_group = product['data']['ftype']['r'].split('.')
if len(data_group) > 1:
data_raw = fh.groups[data_group[1]].variables[data_variable]
else:
data_raw = fh.variables[data_variable]
if 'missing_value' in data_raw.ncattrs():
# ASCAT
data_raw_missing = data_raw.getncattr('missing_value')
elif 'CodeMissingValue' in data_raw.ncattrs():
# GPM
data_raw_missing = data_raw.getncattr('CodeMissingValue')
else:
data_raw_missing = data_raw.getncattr('_FillValue')
if 'scale_factor' in data_raw.ncattrs():
data_raw_scale = data_raw.getncattr('scale_factor')
else:
data_raw_scale = 1.0
# From generated temporary file
# Generate temporary files
# Convert meta data to float
if np.logical_or(isinstance(data_raw_missing, str),
isinstance(data_raw_scale, str)):
data_raw_missing = float(data_raw_missing)
data_raw_scale = float(data_raw_scale)
# --------- #
# Clip data #
# --------- #
# get data to 2D matrix
date_id = 0
data_tmp = data_raw[date_id, y_id[0]:y_id[1], x_id[0]:x_id[1]]
# data_tmp = np.squeeze(data_tmp, axis=0)
# check data type
# filled numpy.ma.MaskedArray as numpy.ndarray
if isinstance(data_tmp, np.ma.MaskedArray):
data = data_tmp.filled()
else:
data = np.asarray(data_tmp)
# transfer matrix to GTiff matrix
# [w,n]--[e,n]
# | |
# [w,s]--[e,s]
# data = np.asarray(data)
# [w,s]--[e,s]
# | |
# [w,n]--[e,n]
data = np.flipud(data)
# [w,n]--[w,s]
# | |
# [e,n]--[e,s]
# data = np.transpose(a=data, axes=(1, 0))
# [w,s]--[w,n]
# | |
# [e,s]--[e,n]
# data = np.rot90(data, k=1, axes=(0, 1))
# close file
fh.close()
# ------- #
# Convert #
# ------- #
# scale, units
data = np.where(data == data_raw_missing, np.nan, data)
data = data * data_raw_scale * data_multiplier
# novalue data
data = np.where(np.isnan(data), data_ndv, data)
# ------------ #
# Saveas GTiff #
# ------------ #
geo = [
lonlim[0] - pixel_size / 10., pixel_size, 0,
latlim[1] + pixel_size / 10., 0, -pixel_size
]
Save_as_tiff(name=local_file, data=data, geo=geo, projection="WGS84")
path = os.path.dirname(os.path.realpath(remote_file))
if 'remote' != path[-6:]:
path = os.path.join(path, 'remote')
clean(path)
path = os.path.dirname(os.path.realpath(temp_file))
if 'temporary' != path[-9:]:
path = os.path.join(path, 'temporary')
clean(path)
status_cod = 0
return status_cod
def convert_data(args):
"""
"""
# Unpack the arguments
latlim, lonlim, date, \
product, \
username, password, apitoken, \
url_server, url_dir, \
remote_fname, temp_fname, local_fname,\
remote_file, temp_file, local_file,\
y_id, x_id, pixel_size, pixel_w, pixel_h, \
data_ndv, data_type, data_multiplier, data_variable = args
# Define local variable
status_cod = -1
# post-process remote (from server)
# -> temporary (unzip)
# -> local (gis)
msg = 'Converting "{f}"'.format(f=local_file)
print('\33[94m{}\33[0m'.format(msg))
__this.Log.write(datetime.datetime.now(), msg=msg)
# --------- #
# Load data #
# --------- #
# From downloaded remote file
remote_fnames, remote_files = start_download_tiles(latlim, lonlim,
remote_fname,
remote_file)
temp_file_part = []
# temp_file_part_4326 = []
for ifile in range(len(remote_fnames)):
# From downloaded remote file
# From generated temporary file
temp_file_part.append(
temp_file.format(ipart=str(ifile + 1)))
# temp_file_part_4326.append(
# temp_file.format(dtime=date, ipart='{}_4326'.format(str(ifile + 1))))
# Generate temporary files
# Convert_hdf5_to_tiff(remote_files[ifile], temp_file_part[ifile],
# data_variable)
#
# reproject_MODIS(temp_file_part[ifile], temp_file_part_4326[ifile], '4326')
# Clip_Dataset_GDAL(remote_files[ifile], temp_file_part[ifile],
# latlim, lonlim)
geo_trans, geo_proj, size_x, size_y = Open_array_info(remote_files[ifile])
lat_min_merge = np.maximum(latlim[0], geo_trans[3] + size_y * geo_trans[5])
lat_max_merge = np.minimum(latlim[1], geo_trans[3])
lon_min_merge = np.maximum(lonlim[0], geo_trans[0])
lon_max_merge = np.minimum(lonlim[1], geo_trans[0] + size_x * geo_trans[1])
lonmerge = [lon_min_merge, lon_max_merge]
latmerge = [lat_min_merge, lat_max_merge]
Clip_Dataset_GDAL(remote_files[ifile], temp_file_part[ifile],
latmerge, lonmerge)
# Convert meta data to float
# if np.logical_or(isinstance(data_raw_missing, str),
# isinstance(data_raw_scale, str)):
# data_raw_missing = float(data_raw_missing)
# data_raw_scale = float(data_raw_scale)
temp_file_part_all = temp_file.format(ipart=0)
Merge_Dataset_GDAL(temp_file_part, temp_file_part_all)
# get data to 2D matrix
geo_trans, geo_proj, \
size_x, size_y = Open_array_info(temp_file_part_all)
lat_min_merge = np.maximum(latlim[0], geo_trans[3] + size_y * geo_trans[5])
lat_max_merge = np.minimum(latlim[1], geo_trans[3])
lon_min_merge = np.maximum(lonlim[0], geo_trans[0])
lon_max_merge = np.minimum(lonlim[1], geo_trans[0] + size_x * geo_trans[1])
lonmerge = [lon_min_merge, lon_max_merge]
latmerge = [lat_min_merge, lat_max_merge]
data_tmp = Open_tiff_array(temp_file_part_all)
# check data type
# filled numpy.ma.MaskedArray as numpy.ndarray
if isinstance(data_tmp, np.ma.MaskedArray):
data = data_tmp.filled()
else:
data = np.asarray(data_tmp)
# transfer matrix to GTiff matrix
# [w,n]--[e,n]
# | |
# [w,s]--[e,s]
data = np.asarray(data)
# [w,s]--[e,s]
# | |
# [w,n]--[e,n]
# data = np.flipud(data)
# [w,n]--[w,s]
# | |
# [e,n]--[e,s]
# data = np.transpose(a=data, axes=(1, 0))
# [w,s]--[w,n]
# | |
# [e,s]--[e,n]
# data = np.rot90(data, k=1, axes=(0, 1))
# close file
# fh.close()
# ------- #
# Convert #
# ------- #
# scale, units
# data = np.where(data < 0, np.nan, data)
data = data * data_multiplier
# novalue data
data = np.where(np.isnan(data), data_ndv, data)
# ------------ #
# Saveas GTiff #
# ------------ #
geo = [lonmerge[0] - geo_trans[1], geo_trans[1], 0,
latmerge[1] - geo_trans[5] / 2., 0, geo_trans[5]]
Save_as_tiff(name=local_file, data=data, geo=geo, projection="WGS84")
if __this.conf['is_save_remote']:
pass
else:
path = os.path.dirname(os.path.realpath(remote_file))
if 'remote' != path[-6:]:
path = os.path.join(path, 'remote')
clean(path)
if __this.conf['is_save_temp']:
pass
else:
path = os.path.dirname(os.path.realpath(temp_file))
if 'temporary' != path[-9:]:
path = os.path.join(path, 'temporary')
clean(path)
status_cod = 0
return status_cod
def DownloadData(status, conf) -> int:
"""This is main interface.
Args:
status (dict): Status.
conf (dict): Configuration.
"""
__this.account = conf['account']
__this.product = conf['product']
__this.Log = Log(conf['log'])
Waitbar = 0
cores = 1
bbox = conf['product']['bbox']
Startdate = conf['product']['period']['s']
Enddate = conf['product']['period']['e']
para_name = conf['product']['parameter']
resolution = conf['product']['resolution']
variable = conf['product']['variable']
TimeFreq = conf['product']['freq']
latlim = conf['product']['data']['lat']
lonlim = conf['product']['data']['lon']
folder = conf['folder']
# Define parameter depedent variables
parameter = para_name.lower()
unit = conf['product']['data']['units']['l']
latlim = [bbox['s'], bbox['n']]
lonlim = [bbox['w'], bbox['e']]
# converts the latlim and lonlim into names of the tiles which must be
# downloaded
if resolution == '3s':
name, rangeLon, rangeLat = Find_Document_Names(latlim, lonlim, parameter)
# Memory for the map x and y shape (starts with zero)
size_X_tot = 0
size_Y_tot = 0
if resolution == '15s' or resolution == '30s':
name = Find_Document_names_15s_30s(latlim, lonlim, parameter, resolution)
nameResults = []
# Create a temporary folder for processing
output_folder = folder['l']
output_folder_trash = folder['t']
# Download, extract, and converts all the files to tiff files
for nameFile in name:
try:
# Download the data from
# http://earlywarning.usgs.gov/hydrodata/
output_file, file_name = Download_Data(nameFile,
output_folder_trash, parameter,
para_name, resolution)
# extract zip data
Extract_Data_zip(output_file, output_folder_trash)
# Converts the data with a adf extention to a tiff extension.
# The input is the file name and in which directory the data must be stored
file_name_tiff = file_name.split('.')[0] + '_trans_temporary.tif'
file_name_extract = file_name.split('_')[0:3]
if resolution == '3s':
file_name_extract2 = file_name_extract[0] + '_' + file_name_extract[1]
if resolution == '15s':
file_name_extract2 = file_name_extract[0] + '_' + file_name_extract[1] + '_15s'
if resolution == '30s':
file_name_extract2 = file_name_extract[0] + '_' + file_name_extract[1] + '_30s'
output_tiff = os.path.join(output_folder_trash, file_name_tiff)
# convert data from adf to a tiff file
if (resolution == "15s" or resolution == "3s"):
input_adf = os.path.join(output_folder_trash, file_name_extract2,
file_name_extract2, 'hdr.adf')
output_tiff = Convert_adf_to_tiff(input_adf, output_tiff)
# convert data from adf to a tiff file
if resolution == "30s":
input_bil = os.path.join(output_folder_trash,
'%s.bil' % file_name_extract2)
output_tiff = Convert_bil_to_tiff(input_bil, output_tiff)
geo_out, proj, size_X, size_Y = Open_array_info(output_tiff)
if (resolution == "3s" and (
int(size_X) != int(6000) or int(size_Y) != int(6000))):
data = np.ones((6000, 6000)) * -9999
# Create the latitude bound
Vfile = str(nameFile)[1:3]
SignV = str(nameFile)[0]
SignVer = 1
# If the sign before the filename is a south sign than latitude is negative
if SignV is "s":
SignVer = -1
Bound2 = int(SignVer) * int(Vfile)
# Create the longitude bound
Hfile = str(nameFile)[4:7]
SignH = str(nameFile)[3]
SignHor = 1
# If the sign before the filename is a west sign than longitude is negative
if SignH is "w":
SignHor = -1
Bound1 = int(SignHor) * int(Hfile)
Expected_X_min = Bound1
Expected_Y_max = Bound2 + 5
Xid_start = int(np.round((geo_out[0] - Expected_X_min) / geo_out[1]))
Xid_end = int(np.round(
((geo_out[0] + size_X * geo_out[1]) - Expected_X_min) / geo_out[1]))
Yid_start = int(np.round((Expected_Y_max - geo_out[3]) / (-geo_out[5])))
Yid_end = int(np.round(
(Expected_Y_max - (geo_out[3] + (size_Y * geo_out[5]))) / (
-geo_out[5])))
data[Yid_start:Yid_end, Xid_start:Xid_end] = Open_tiff_array(
output_tiff)
if np.max(data) == 255:
data[data == 255] = -9999
data[data < -9999] = -9999
geo_in = [Bound1, 0.00083333333333333, 0.0, int(Bound2 + 5),
0.0, -0.0008333333333333333333]
# save chunk as tiff file
Save_as_tiff(name=output_tiff, data=data, geo=geo_in, projection="WGS84")
except:
if resolution == '3s':
# If tile not exist create a replacing zero tile (sea tiles)
output = nameFile.split('.')[0] + "_trans_temporary.tif"
output_tiff = os.path.join(output_folder_trash, output)
file_name = nameFile
data = np.ones((6000, 6000)) * -9999
data = data.astype(np.float32)
# Create the latitude bound
Vfile = str(file_name)[1:3]
SignV = str(file_name)[0]
SignVer = 1
# If the sign before the filename is a south sign than latitude is negative
if SignV is "s":
SignVer = -1
Bound2 = int(SignVer) * int(Vfile)
# Create the longitude bound
Hfile = str(file_name)[4:7]
SignH = str(file_name)[3]
SignHor = 1
# If the sign before the filename is a west sign than longitude is negative
if SignH is "w":
SignHor = -1
Bound1 = int(SignHor) * int(Hfile)
# Geospatial data for the tile
geo_in = [Bound1, 0.00083333333333333, 0.0, int(Bound2 + 5),
0.0, -0.0008333333333333333333]
# save chunk as tiff file
Save_as_tiff(name=output_tiff, data=data, geo=geo_in, projection="WGS84")
if resolution == '15s':
print('no 15s data is in dataset')
if resolution == '3s':
# clip data
Data, Geo_data = Clip_Data(output_tiff, latlim, lonlim)
size_Y_out = int(np.shape(Data)[0])
size_X_out = int(np.shape(Data)[1])
# Total size of the product so far
size_Y_tot = int(size_Y_tot + size_Y_out)
size_X_tot = int(size_X_tot + size_X_out)
if nameFile is name[0]:
Geo_x_end = Geo_data[0]
Geo_y_end = Geo_data[3]
else:
Geo_x_end = np.min([Geo_x_end, Geo_data[0]])
Geo_y_end = np.max([Geo_y_end, Geo_data[3]])
# create name for chunk
FileNameEnd = "%s_temporary.tif" % (nameFile)
nameForEnd = os.path.join(output_folder_trash, FileNameEnd)
nameResults.append(str(nameForEnd))
# save chunk as tiff file
Save_as_tiff(name=nameForEnd, data=Data, geo=Geo_data, projection="WGS84")
if resolution == '3s':
# size_X_end = int(size_X_tot) #!
# size_Y_end = int(size_Y_tot) #!
size_X_end = int(size_X_tot / len(rangeLat)) + 1 # !
size_Y_end = int(size_Y_tot / len(rangeLon)) + 1 # !
# Define the georeference of the end matrix
geo_out = [Geo_x_end, Geo_data[1], 0, Geo_y_end, 0, Geo_data[5]]
latlim_out = [geo_out[3] + geo_out[5] * size_Y_end, geo_out[3]]
lonlim_out = [geo_out[0], geo_out[0] + geo_out[1] * size_X_end]
# merge chunk together resulting in 1 tiff map
datasetTot = Merge_DEM(latlim_out, lonlim_out, nameResults, size_Y_end,
size_X_end)
datasetTot[datasetTot < -9999] = -9999
if resolution == '15s':
output_file_merged = os.path.join(output_folder_trash, 'merged.tif')
datasetTot, geo_out = Merge_DEM_15s_30s(output_folder_trash, output_file_merged,
latlim, lonlim, resolution)
if resolution == '30s':
output_file_merged = os.path.join(output_folder_trash, 'merged.tif')
datasetTot, geo_out = Merge_DEM_15s_30s(output_folder_trash, output_file_merged,
latlim, lonlim, resolution)
# name of the end result
output_DEM_name = "%s_HydroShed_%s_%s.tif" % (para_name, unit, resolution)
Save_name = os.path.join(output_folder, output_DEM_name)
# Make geotiff file
Save_as_tiff(name=Save_name, data=datasetTot, geo=geo_out, projection="WGS84")
os.chdir(output_folder)
