def _init(status, conf): # From download.py __this.status = status __this.conf = conf account = conf['account'] folder = conf['folder'] product = conf['product'] # Init supported classes __this.GIS = GIS(status, conf) __this.Dtime = Dtime(status, conf) __this.Log = Log(conf['log']) return account, folder, product
def _init(status, conf): # From download.py __this.status = status __this.conf = conf __this.path = os.path.join( os.getcwd(), os.path.dirname(inspect.getfile(inspect.currentframe()))) account = conf['account'] folder = conf['folder'] product = conf['product'] # Init supported classes __this.GIS = GIS(status, conf) __this.Dtime = Dtime(status, conf) __this.Log = Log(conf['log']) return account, folder, product
def _init(status, conf) -> tuple: # test = sys.modules # print('GIS' in test) # From download.py __this.status = status __this.conf = conf account = conf['account'] folder = conf['folder'] product = conf['product'] # Init supported classes __this.GIS = GIS(status, conf) __this.Dtime = Dtime(status, conf) __this.Log = Log(conf['log']) return account, folder, product
def DownloadData(status, conf) -> int: """This is main interface. Args: status (dict): Status. conf (dict): Configuration. """ __this.account = conf['account'] __this.product = conf['product'] __this.Log = Log(conf['log']) Waitbar = 0 cores = 1 bbox = conf['product']['bbox'] Startdate = conf['product']['period']['s'] Enddate = conf['product']['period']['e'] para_name = conf['product']['parameter'] resolution = conf['product']['resolution'] variable = conf['product']['variable'] TimeFreq = conf['product']['freq'] latlim = conf['product']['data']['lat'] lonlim = conf['product']['data']['lon'] folder = conf['folder'] # Define parameter depedent variables parameter = para_name.lower() unit = conf['product']['data']['units']['l'] latlim = [bbox['s'], bbox['n']] lonlim = [bbox['w'], bbox['e']] # converts the latlim and lonlim into names of the tiles which must be # downloaded if resolution == '3s': name, rangeLon, rangeLat = Find_Document_Names(latlim, lonlim, parameter) # Memory for the map x and y shape (starts with zero) size_X_tot = 0 size_Y_tot = 0 if resolution == '15s' or resolution == '30s': name = Find_Document_names_15s_30s(latlim, lonlim, parameter, resolution) nameResults = [] # Create a temporary folder for processing output_folder = folder['l'] output_folder_trash = folder['t'] # Download, extract, and converts all the files to tiff files for nameFile in name: try: # Download the data from # http://earlywarning.usgs.gov/hydrodata/ output_file, file_name = Download_Data(nameFile, output_folder_trash, parameter, para_name, resolution) # extract zip data Extract_Data_zip(output_file, output_folder_trash) # Converts the data with a adf extention to a tiff extension. # The input is the file name and in which directory the data must be stored file_name_tiff = file_name.split('.')[0] + '_trans_temporary.tif' file_name_extract = file_name.split('_')[0:3] if resolution == '3s': file_name_extract2 = file_name_extract[0] + '_' + file_name_extract[1] if resolution == '15s': file_name_extract2 = file_name_extract[0] + '_' + file_name_extract[1] + '_15s' if resolution == '30s': file_name_extract2 = file_name_extract[0] + '_' + file_name_extract[1] + '_30s' output_tiff = os.path.join(output_folder_trash, file_name_tiff) # convert data from adf to a tiff file if (resolution == "15s" or resolution == "3s"): input_adf = os.path.join(output_folder_trash, file_name_extract2, file_name_extract2, 'hdr.adf') output_tiff = Convert_adf_to_tiff(input_adf, output_tiff) # convert data from adf to a tiff file if resolution == "30s": input_bil = os.path.join(output_folder_trash, '%s.bil' % file_name_extract2) output_tiff = Convert_bil_to_tiff(input_bil, output_tiff) geo_out, proj, size_X, size_Y = Open_array_info(output_tiff) if (resolution == "3s" and ( int(size_X) != int(6000) or int(size_Y) != int(6000))): data = np.ones((6000, 6000)) * -9999 # Create the latitude bound Vfile = str(nameFile)[1:3] SignV = str(nameFile)[0] SignVer = 1 # If the sign before the filename is a south sign than latitude is negative if SignV is "s": SignVer = -1 Bound2 = int(SignVer) * int(Vfile) # Create the longitude bound Hfile = str(nameFile)[4:7] SignH = str(nameFile)[3] SignHor = 1 # If the sign before the filename is a west sign than longitude is negative if SignH is "w": SignHor = -1 Bound1 = int(SignHor) * int(Hfile) Expected_X_min = Bound1 Expected_Y_max = Bound2 + 5 Xid_start = int(np.round((geo_out[0] - Expected_X_min) / geo_out[1])) Xid_end = int(np.round( ((geo_out[0] + size_X * geo_out[1]) - Expected_X_min) / geo_out[1])) Yid_start = int(np.round((Expected_Y_max - geo_out[3]) / (-geo_out[5]))) Yid_end = int(np.round( (Expected_Y_max - (geo_out[3] + (size_Y * geo_out[5]))) / ( -geo_out[5]))) data[Yid_start:Yid_end, Xid_start:Xid_end] = Open_tiff_array( output_tiff) if np.max(data) == 255: data[data == 255] = -9999 data[data < -9999] = -9999 geo_in = [Bound1, 0.00083333333333333, 0.0, int(Bound2 + 5), 0.0, -0.0008333333333333333333] # save chunk as tiff file Save_as_tiff(name=output_tiff, data=data, geo=geo_in, projection="WGS84") except: if resolution == '3s': # If tile not exist create a replacing zero tile (sea tiles) output = nameFile.split('.')[0] + "_trans_temporary.tif" output_tiff = os.path.join(output_folder_trash, output) file_name = nameFile data = np.ones((6000, 6000)) * -9999 data = data.astype(np.float32) # Create the latitude bound Vfile = str(file_name)[1:3] SignV = str(file_name)[0] SignVer = 1 # If the sign before the filename is a south sign than latitude is negative if SignV is "s": SignVer = -1 Bound2 = int(SignVer) * int(Vfile) # Create the longitude bound Hfile = str(file_name)[4:7] SignH = str(file_name)[3] SignHor = 1 # If the sign before the filename is a west sign than longitude is negative if SignH is "w": SignHor = -1 Bound1 = int(SignHor) * int(Hfile) # Geospatial data for the tile geo_in = [Bound1, 0.00083333333333333, 0.0, int(Bound2 + 5), 0.0, -0.0008333333333333333333] # save chunk as tiff file Save_as_tiff(name=output_tiff, data=data, geo=geo_in, projection="WGS84") if resolution == '15s': print('no 15s data is in dataset') if resolution == '3s': # clip data Data, Geo_data = Clip_Data(output_tiff, latlim, lonlim) size_Y_out = int(np.shape(Data)[0]) size_X_out = int(np.shape(Data)[1]) # Total size of the product so far size_Y_tot = int(size_Y_tot + size_Y_out) size_X_tot = int(size_X_tot + size_X_out) if nameFile is name[0]: Geo_x_end = Geo_data[0] Geo_y_end = Geo_data[3] else: Geo_x_end = np.min([Geo_x_end, Geo_data[0]]) Geo_y_end = np.max([Geo_y_end, Geo_data[3]]) # create name for chunk FileNameEnd = "%s_temporary.tif" % (nameFile) nameForEnd = os.path.join(output_folder_trash, FileNameEnd) nameResults.append(str(nameForEnd)) # save chunk as tiff file Save_as_tiff(name=nameForEnd, data=Data, geo=Geo_data, projection="WGS84") if resolution == '3s': # size_X_end = int(size_X_tot) #! # size_Y_end = int(size_Y_tot) #! size_X_end = int(size_X_tot / len(rangeLat)) + 1 # ! size_Y_end = int(size_Y_tot / len(rangeLon)) + 1 # ! # Define the georeference of the end matrix geo_out = [Geo_x_end, Geo_data[1], 0, Geo_y_end, 0, Geo_data[5]] latlim_out = [geo_out[3] + geo_out[5] * size_Y_end, geo_out[3]] lonlim_out = [geo_out[0], geo_out[0] + geo_out[1] * size_X_end] # merge chunk together resulting in 1 tiff map datasetTot = Merge_DEM(latlim_out, lonlim_out, nameResults, size_Y_end, size_X_end) datasetTot[datasetTot < -9999] = -9999 if resolution == '15s': output_file_merged = os.path.join(output_folder_trash, 'merged.tif') datasetTot, geo_out = Merge_DEM_15s_30s(output_folder_trash, output_file_merged, latlim, lonlim, resolution) if resolution == '30s': output_file_merged = os.path.join(output_folder_trash, 'merged.tif') datasetTot, geo_out = Merge_DEM_15s_30s(output_folder_trash, output_file_merged, latlim, lonlim, resolution) # name of the end result output_DEM_name = "%s_HydroShed_%s_%s.tif" % (para_name, unit, resolution) Save_name = os.path.join(output_folder, output_DEM_name) # Make geotiff file Save_as_tiff(name=Save_name, data=datasetTot, geo=geo_out, projection="WGS84") os.chdir(output_folder)