def Download_ASCAT_from_VITO(End_filename, output_folder_temp, Date, yID, xID): """ This function retrieves ALEXI data for a given date from the ftp.wateraccounting.unesco-ihe.org server. Restrictions: The data and this python file may not be distributed to others without permission of the WA+ team due data restriction of the ALEXI developers. Keyword arguments: """ # Define date year_data = Date.year month_data = Date.month day_data = Date.day # filename of ASCAT data on server ASCAT_date = "%d%02d%02d0000" % (year_data, month_data, day_data) ASCAT_name = 'SWI_%s_GLOBE_ASCAT_V3.0' % ASCAT_date ASCAT_filename = "g2_BIOPAR_SWI_%s_GLOBE_ASCAT_V3.0.1.zip" % ASCAT_date # Collect account and FTP information username, password = WebAccounts.Accounts(Type='VITO') URL = "https://land.copernicus.vgt.vito.be/PDF/datapool/Vegetation/Soil_Water/SWI_V3/%s/%s/%s/%s/%s" % ( year_data, month_data, day_data, ASCAT_name, ASCAT_filename) # Output zipfile output_zipfile_ASCAT = os.path.join(output_folder_temp, ASCAT_filename) # Download the ASCAT data try: y = requests.get(URL, auth=HTTPBasicAuth(username, password)) except: from requests.packages.urllib3.exceptions import InsecureRequestWarning requests.packages.urllib3.disable_warnings(InsecureRequestWarning) y = requests.get(URL, auth=(username, password), verify=False) # Write the file in system z = open(output_zipfile_ASCAT, 'wb') z.write(y.content) z.close() # Extract the zipfile DC.Extract_Data(output_zipfile_ASCAT, output_folder_temp) # Open the file f = h5py.File(output_zipfile_ASCAT.replace('.zip', '.h5')) # Open global ASCAT data dataset = np.array((f['SWI']['SWI_010']).value) # Clip extend out of world data data = dataset[yID[0]:yID[1], xID[0]:xID[1]].astype("float") * 0.5 data[data > 100] = -9999 return (data)
def Download_SSEBop_from_Web(output_folder, Filename_only_zip): """ This function retrieves SSEBop data for a given date from the https://edcintl.cr.usgs.gov server. Keyword arguments: local_filename -- name of the temporary file which contains global SSEBop data Filename_dir -- name of the end directory to put in the extracted data """ # Create the total url to the webpage total_URL = "https://edcintl.cr.usgs.gov/downloads/sciweb1/shared/fews/web/global/monthly/eta/downloads/" + str( Filename_only_zip) # Download the data urllib.urlretrieve(total_URL, os.path.join(output_folder, Filename_only_zip)) # unzip the file DC.Extract_Data(os.path.join(output_folder, Filename_only_zip), output_folder) return
def Unzip_ETens_data(output_folder, Lat_tiles, Lon_tiles): """ This function extract the zip files Keyword Arguments: output_folder -- Directory of the outputs Lat_tiles -- [Lat_min, Lat_max] Tile number of the max and min latitude tile number Lon_tiles -- [Lon_min, Lon_max] Tile number of the max and min longitude tile number """ # Unzip the zip files one by one try: for v_tile in range(Lat_tiles[0], Lat_tiles[1]+1): for h_tile in range(Lon_tiles[0], Lon_tiles[1]+1): # Define the file and path to the zip file Tilename = "h%sv%s.zip" %(h_tile, v_tile) input_zip_folder = os.path.join(output_folder, Tilename) if os.path.exists(input_zip_folder): # Extract data DC.Extract_Data(input_zip_folder, output_folder) except: print 'Was not able to unzip %s, data will be replaced by NaN values' %Tilename return()
def DownloadData(output_folder, latlim, lonlim, parameter, resolution): """ This function downloads DEM data from HydroSHED Keyword arguments: output_folder -- directory of the result latlim -- [ymin, ymax] (values must be between -50 and 50) lonlim -- [xmin, xmax] (values must be between -180 and 180) Resample -- 1 = The data will be resampled to 0.001 degree spatial resolution -- 0 = The data will have the same pixel size as the data obtained from the internet """ # Define parameter depedent variables if parameter == "dir_3s": para_name = "DIR" unit = "-" resolution = '3s' parameter = 'dir' if parameter == "dem_3s": para_name = "DEM" unit = "m" resolution = '3s' parameter = 'dem' if parameter == "dir_15s": para_name = "DIR" unit = "-" resolution = '15s' parameter = 'dir' if parameter == "dem_15s": para_name = "DEM" unit = "m" resolution = '15s' parameter = 'dem' # converts the latlim and lonlim into names of the tiles which must be # downloaded if resolution == '3s': name, rangeLon, rangeLat = Find_Document_Names(latlim, lonlim, parameter) # Memory for the map x and y shape (starts with zero) size_X_tot = 0 size_Y_tot = 0 if resolution == '15s': name = Find_Document_names_15s(latlim, lonlim, parameter, resolution) nameResults = [] # Create a temporary folder for processing output_folder_trash = os.path.join(output_folder, "Temp") if not os.path.exists(output_folder_trash): os.makedirs(output_folder_trash) # Download, extract, and converts all the files to tiff files for nameFile in name: try: # Download the data from # http://earlywarning.usgs.gov/hydrodata/ output_file, file_name = Download_Data(nameFile, output_folder_trash, parameter, para_name,resolution) # extract zip data DC.Extract_Data(output_file, output_folder_trash) # Converts the data with a adf extention to a tiff extension. # The input is the file name and in which directory the data must be stored file_name_tiff = file_name.split('.')[0] + '_trans_temporary.tif' file_name_extract = file_name.split('_')[0:3] if resolution == '3s': file_name_extract2 = file_name_extract[0]+'_'+file_name_extract[1] if resolution == '15s': file_name_extract2 = file_name_extract[0]+'_'+file_name_extract[1]+'_15s' input_adf = os.path.join(output_folder_trash, file_name_extract2, file_name_extract2, 'hdr.adf') output_tiff = os.path.join(output_folder_trash, file_name_tiff) # convert data from adf to a tiff file output_tiff = DC.Convert_adf_to_tiff(input_adf, output_tiff) geo_out, proj, size_X, size_Y = RC.Open_array_info(output_tiff) if int(size_X) != int(6000) or int(size_Y) != int(6000): data = np.ones((6000, 6000)) * -9999 # Create the latitude bound Vfile = str(nameFile)[1:3] SignV = str(nameFile)[0] SignVer = 1 # If the sign before the filename is a south sign than latitude is negative if SignV is "s": SignVer = -1 Bound2 = int(SignVer)*int(Vfile) # Create the longitude bound Hfile = str(nameFile)[4:7] SignH = str(nameFile)[3] SignHor = 1 # If the sign before the filename is a west sign than longitude is negative if SignH is "w": SignHor = -1 Bound1 = int(SignHor) * int(Hfile) Expected_X_min = Bound1 Expected_Y_max = Bound2 + 5 Xid_start = int(np.round((geo_out[0] - Expected_X_min)/geo_out[1])) Xid_end = int(np.round(((geo_out[0] + size_X * geo_out[1]) - Expected_X_min)/geo_out[1])) Yid_start = int(np.round((Expected_Y_max - geo_out[3])/(-geo_out[5]))) Yid_end = int(np.round((Expected_Y_max - (geo_out[3] + (size_Y * geo_out[5])))/(-geo_out[5]))) data[Yid_start:Yid_end,Xid_start:Xid_end] = RC.Open_tiff_array(output_tiff) if np.max(data)==255: data[data==255] = -9999 data[data<-9999] = -9999 geo_in = [Bound1, 0.00083333333333333, 0.0, int(Bound2 + 5), 0.0, -0.0008333333333333333333] # save chunk as tiff file DC.Save_as_tiff(name=output_tiff, data=data, geo=geo_in, projection="WGS84") except: if resolution == '3s': # If tile not exist create a replacing zero tile (sea tiles) output = nameFile.split('.')[0] + "_trans_temporary.tif" output_tiff = os.path.join(output_folder_trash, output) file_name = nameFile data = np.ones((6000, 6000)) * -9999 data = data.astype(np.float32) # Create the latitude bound Vfile = str(file_name)[1:3] SignV = str(file_name)[0] SignVer = 1 # If the sign before the filename is a south sign than latitude is negative if SignV is "s": SignVer = -1 Bound2 = int(SignVer)*int(Vfile) # Create the longitude bound Hfile = str(file_name)[4:7] SignH = str(file_name)[3] SignHor = 1 # If the sign before the filename is a west sign than longitude is negative if SignH is "w": SignHor = -1 Bound1 = int(SignHor) * int(Hfile) # Geospatial data for the tile geo_in = [Bound1, 0.00083333333333333, 0.0, int(Bound2 + 5), 0.0, -0.0008333333333333333333] # save chunk as tiff file DC.Save_as_tiff(name=output_tiff, data=data, geo=geo_in, projection="WGS84") if resolution == '15s': print 'no 15s data is in dataset' if resolution =='3s': # clip data Data, Geo_data = RC.clip_data(output_tiff, latlim, lonlim) size_Y_out = int(np.shape(Data)[0]) size_X_out = int(np.shape(Data)[1]) # Total size of the product so far size_Y_tot = int(size_Y_tot + size_Y_out) size_X_tot = int(size_X_tot + size_X_out) if nameFile is name[0]: Geo_x_end = Geo_data[0] Geo_y_end = Geo_data[3] else: Geo_x_end = np.min([Geo_x_end,Geo_data[0]]) Geo_y_end = np.max([Geo_y_end,Geo_data[3]]) # create name for chunk FileNameEnd = "%s_temporary.tif" % (nameFile) nameForEnd = os.path.join(output_folder_trash, FileNameEnd) nameResults.append(str(nameForEnd)) # save chunk as tiff file DC.Save_as_tiff(name=nameForEnd, data=Data, geo=Geo_data, projection="WGS84") if resolution =='3s': #size_X_end = int(size_X_tot) #! #size_Y_end = int(size_Y_tot) #! size_X_end = int(size_X_tot/len(rangeLat)) + 1 #! size_Y_end = int(size_Y_tot/len(rangeLon)) + 1 #! # Define the georeference of the end matrix geo_out = [Geo_x_end, Geo_data[1], 0, Geo_y_end, 0, Geo_data[5]] latlim_out = [geo_out[3] + geo_out[5] * size_Y_end, geo_out[3]] lonlim_out = [geo_out[0], geo_out[0] + geo_out[1] * size_X_end] # merge chunk together resulting in 1 tiff map datasetTot = Merge_DEM(latlim_out, lonlim_out, nameResults, size_Y_end, size_X_end) datasetTot[datasetTot<-9999] = -9999 if resolution =='15s': output_file_merged = os.path.join(output_folder_trash,'merged.tif') datasetTot, geo_out = Merge_DEM_15s(output_folder_trash, output_file_merged,latlim, lonlim) # name of the end result output_DEM_name = "%s_HydroShed_%s_%s.tif" %(para_name,unit,resolution) Save_name = os.path.join(output_folder, output_DEM_name) # Make geotiff file DC.Save_as_tiff(name=Save_name, data=datasetTot, geo=geo_out, projection="WGS84") os.chdir(output_folder) # Delete the temporary folder shutil.rmtree(output_folder_trash)
def RetrieveData(Date, args): """ This function retrieves RFE data for a given date from the ftp://disc2.nascom.nasa.gov server. Keyword arguments: Date -- 'yyyy-mm-dd' args -- A list of parameters defined in the DownloadData function. """ # Argument [output_folder, lonlim, latlim, xID, yID] = args # Create https DirFile = os.path.join(output_folder,'P_RFE.v2.0_mm-day-1_daily_%s.%02s.%02s.tif' %(Date.strftime('%Y'), Date.strftime('%m'), Date.strftime('%d'))) if not os.path.isfile(DirFile): # open ftp server ftp = FTP("ftp.cpc.ncep.noaa.gov", "", "") ftp.login() # Define FTP path to directory pathFTP = 'fews/fewsdata/africa/rfe2/geotiff/' # find the document name in this directory ftp.cwd(pathFTP) listing = [] # read all the file names in the directory ftp.retrlines("LIST", listing.append) # create all the input name (filename) and output (outfilename, filetif, DiFileEnd) names filename = 'africa_rfe.%s%02s%02s.tif.zip' %(Date.strftime('%Y'), Date.strftime('%m'), Date.strftime('%d')) outfilename = os.path.join(output_folder,'africa_rfe.%s%02s%02s.tif' %(Date.strftime('%Y'), Date.strftime('%m'), Date.strftime('%d'))) try: local_filename = os.path.join(output_folder, filename) lf = open(local_filename, "wb") ftp.retrbinary("RETR " + filename, lf.write) lf.close() # unzip the file zip_filename = os.path.join(output_folder, filename) DC.Extract_Data(zip_filename, output_folder) # open tiff file dataset = RC.Open_tiff_array(outfilename) # clip dataset to the given extent data = dataset[yID[0]:yID[1], xID[0]:xID[1]] data[data < 0] = -9999 # save dataset as geotiff file latlim_adj = 40.05 - 0.1 * yID[0] lonlim_adj = -20.05 + 0.1 * xID[0] geo = [lonlim_adj, 0.1, 0, latlim_adj, 0, -0.1] DC.Save_as_tiff(name=DirFile, data=data, geo=geo, projection="WGS84") # delete old tif file os.remove(outfilename) os.remove(zip_filename) except: print "file not exists" return True