Beispiel #1
0
def Download_ETmonitor_from_WA_FTP(local_filename, Filename_in, Type):
    """
    This function retrieves ETmonitor data for a given date from the
    ftp.wateraccounting.unesco-ihe.org server.
				
    Restrictions:
    The data and this python file may not be distributed to others without
    permission of the WA+ team due data restriction of the ETmonitor developers.

    Keyword arguments:
	 local_filename -- name of the temporary file which contains global ETmonitor data			
    Filename_in -- name of the end file with the weekly ETmonitor data	
	 Type = Type of data ("act" or "pot")
    """

    # Collect account and FTP information
    username, password = WebAccounts.Accounts(Type='FTP_WA')
    ftpserver = "ftp.wateraccounting.unesco-ihe.org"

    # Download data from FTP
    ftp = FTP(ftpserver)
    ftp.login(username, password)
    if Type == "act":
        directory = "/WaterAccounting/Data_Satellite/Evaporation/ETmonitor/Global/"
    if Type == "pot":
        directory = "/WaterAccounting/Data_Satellite/Evaporation/ETmonitor/Potential_Evapotranspiration/"
    ftp.cwd(directory)
    lf = open(local_filename, "wb")
    ftp.retrbinary("RETR " + Filename_in, lf.write)
    lf.close()

    return
Beispiel #2
0
def Download_CMRSET_from_WA_FTP(local_filename, Filename_in):
    """
    This function retrieves CMRSET data for a given date from the
    ftp.wateraccounting.unesco-ihe.org server.
				
    Restrictions:
    The data and this python file may not be distributed to others without
    permission of the WA+ team due data restriction of the CMRSET developers.

    Keyword arguments:
	 local_filename -- name of the temporary file which contains global CMRSET data			
    Filename_in -- name of the end file with the monthly CMRSET data
    """

    # Collect account and FTP information
    username, password = WebAccounts.Accounts(Type='FTP_WA')
    ftpserver = "ftp.wateraccounting.unesco-ihe.org"

    # Download data from FTP
    ftp = FTP(ftpserver)
    ftp.login(username, password)
    directory = "/WaterAccounting/Data_Satellite/Evaporation/CMRSET/Global/"
    ftp.cwd(directory)
    lf = open(local_filename, "wb")
    ftp.retrbinary("RETR " + Filename_in, lf.write)
    lf.close()

    return
Beispiel #3
0
def Download_ASCAT_from_VITO(End_filename, output_folder_temp, Date, yID, xID):
    """
    This function retrieves ALEXI data for a given date from the
    ftp.wateraccounting.unesco-ihe.org server.

    Restrictions:
    The data and this python file may not be distributed to others without
    permission of the WA+ team due data restriction of the ALEXI developers.

    Keyword arguments:

    """

    # Define date
    year_data = Date.year
    month_data = Date.month
    day_data = Date.day

    # filename of ASCAT data on server
    ASCAT_date = "%d%02d%02d0000" % (year_data, month_data, day_data)
    ASCAT_name = 'SWI_%s_GLOBE_ASCAT_V3.0' % ASCAT_date
    ASCAT_filename = "g2_BIOPAR_SWI_%s_GLOBE_ASCAT_V3.0.1.zip" % ASCAT_date

    # Collect account and FTP information
    username, password = WebAccounts.Accounts(Type='VITO')
    URL = "https://land.copernicus.vgt.vito.be/PDF/datapool/Vegetation/Soil_Water/SWI_V3/%s/%s/%s/%s/%s" % (
        year_data, month_data, day_data, ASCAT_name, ASCAT_filename)

    # Output zipfile
    output_zipfile_ASCAT = os.path.join(output_folder_temp, ASCAT_filename)

    # Download the ASCAT data
    try:
        y = requests.get(URL, auth=HTTPBasicAuth(username, password))
    except:
        from requests.packages.urllib3.exceptions import InsecureRequestWarning
        requests.packages.urllib3.disable_warnings(InsecureRequestWarning)

        y = requests.get(URL, auth=(username, password), verify=False)

    # Write the file in system
    z = open(output_zipfile_ASCAT, 'wb')
    z.write(y.content)
    z.close()

    # Extract the zipfile
    DC.Extract_Data(output_zipfile_ASCAT, output_folder_temp)

    # Open the file
    f = h5py.File(output_zipfile_ASCAT.replace('.zip', '.h5'))

    # Open global ASCAT data
    dataset = np.array((f['SWI']['SWI_010']).value)

    # Clip extend out of world data
    data = dataset[yID[0]:yID[1], xID[0]:xID[1]].astype("float") * 0.5
    data[data > 100] = -9999

    return (data)
Beispiel #4
0
def Download_ALEXI_from_WA_FTP(local_filename, DirFile, filename, lonlim,
                               latlim, yID, xID, TimeStep):
    """
    This function retrieves ALEXI data for a given date from the
    ftp.wateraccounting.unesco-ihe.org server.

    Restrictions:
    The data and this python file may not be distributed to others without
    permission of the WA+ team due data restriction of the ALEXI developers.

    Keyword arguments:
	local_filename -- name of the temporary file which contains global ALEXI data
    DirFile -- name of the end file with the weekly ALEXI data
    filename -- name of the end file
    lonlim -- [ymin, ymax] (values must be between -60 and 70)
    latlim -- [xmin, xmax] (values must be between -180 and 180)
    """

    # Collect account and FTP information
    username, password = WebAccounts.Accounts(Type='FTP_WA')
    ftpserver = "ftp.wateraccounting.unesco-ihe.org"

    # Download data from FTP
    ftp = FTP(ftpserver)
    ftp.login(username, password)
    if TimeStep is "weekly":
        directory = "/WaterAccounting/Data_Satellite/Evaporation/ALEXI/World/"
    if TimeStep is "daily":
        directory = "/WaterAccounting/Data_Satellite/Evaporation/ALEXI/World_05182018/"
    ftp.cwd(directory)
    lf = open(local_filename, "wb")
    ftp.retrbinary("RETR " + filename, lf.write)
    lf.close()

    if TimeStep is "weekly":

        # Open global ALEXI data
        dataset = RC.Open_tiff_array(local_filename)

        # Clip extend out of world data
        data = dataset[yID[0]:yID[1], xID[0]:xID[1]]
        data[data < 0] = -9999

    if TimeStep is "daily":

        DC.Extract_Data_gz(local_filename, os.path.splitext(local_filename)[0])

        raw_data = np.fromfile(os.path.splitext(local_filename)[0],
                               dtype="<f4")
        dataset = np.flipud(np.resize(raw_data, [3000, 7200]))
        data = dataset[yID[0]:yID[1], xID[0]:xID[1]]
        data[data < 0] = -9999

    # make geotiff file
    geo = [lonlim[0], 0.05, 0, latlim[1], 0, -0.05]
    DC.Save_as_tiff(name=DirFile, data=data, geo=geo, projection="WGS84")
    return
Beispiel #5
0
def Collect_data(FTPprefix, Years, output_folder, Waitbar):
    '''
    This function downloads all the needed GLEAM files from hydras.ugent.be as a nc file.

    Keywords arguments:
    FTPprefix -- FTP path to the GLEAM data
    Date -- 'yyyy-mm-dd' 				
    output_folder -- 'C:/file/to/path/'	
    '''
    # account of the SFTP server (only password is missing)
    server = 'hydras.ugent.be'
    portnumber = 2225

    username, password = WebAccounts.Accounts(Type='GLEAM')

    # Create Waitbar
    print '\nDownload GLEAM data'
    if Waitbar == 1:
        import wa.Functions.Start.WaitbarConsole as WaitbarConsole
        total_amount2 = len(Years)
        amount2 = 0
        WaitbarConsole.printWaitBar(amount2,
                                    total_amount2,
                                    prefix='Progress:',
                                    suffix='Complete',
                                    length=50)

    for year in Years:
        directory = os.path.join(FTPprefix, '%d' % year)
        ssh = paramiko.SSHClient()
        ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
        ssh.connect(server,
                    port=portnumber,
                    username=username,
                    password=password)
        ftp = ssh.open_sftp()
        ftp.chdir(directory)

        filename = 'E_' + str(year) + '_GLEAM_v3.1b.nc'
        local_filename = os.path.join(output_folder, filename)

        if not os.path.exists(local_filename):
            ftp.get(filename, local_filename)

        if Waitbar == 1:
            amount2 += 1
            WaitbarConsole.printWaitBar(amount2,
                                        total_amount2,
                                        prefix='Progress:',
                                        suffix='Complete',
                                        length=50)

    ftp.close()
    ssh.close()

    return ()
Beispiel #6
0
def Download_ALEXI_from_WA_FTP(local_filename, DirFile, filename, lonlim,
                               latlim, yID, xID):
    """
    This function retrieves ALEXI data for a given date from the
    ftp.wateraccounting.unesco-ihe.org server.
				
    Restrictions:
    The data and this python file may not be distributed to others without
    permission of the WA+ team due data restriction of the ALEXI developers.

    Keyword arguments:
	local_filename -- name of the temporary file which contains global ALEXI data			
    DirFile -- name of the end file with the weekly ALEXI data
    filename -- name of the end file
    lonlim -- [ymin, ymax] (values must be between -60 and 70)
    latlim -- [xmin, xmax] (values must be between -180 and 180)			
    """

    try:

        # Collect account and FTP information
        username, password = WebAccounts.Accounts(Type='FTP_WA')
        ftpserver = "ftp.wateraccounting.unesco-ihe.org"

        # Download data from FTP
        ftp = FTP(ftpserver)
        ftp.login(username, password)
        directory = "/WaterAccounting/Data_Satellite/Evaporation/ALEXI/World/"
        ftp.cwd(directory)
        lf = open(local_filename, "wb")
        ftp.retrbinary("RETR " + filename, lf.write)
        lf.close()

        # Open global ALEXI data
        dataset = RC.Open_tiff_array(local_filename)

        # Clip extend out of world data
        data = dataset[yID[0]:yID[1], xID[0]:xID[1]]
        data[data < 0] = -9999

        # make geotiff file
        geo = [lonlim[0], 0.05, 0, latlim[1], 0, -0.05]
        DC.Save_as_tiff(name=DirFile, data=data, geo=geo, projection="WGS84")

        # delete old tif file
        os.remove(local_filename)

    except:
        print "file not exists"

    return
Beispiel #7
0
def Download_GWF_from_WA_FTP(output_folder, filename_Out, lonlim, latlim):
    """
    This function retrieves GWF data for a given date from the
    ftp.wateraccounting.unesco-ihe.org server.

    Keyword arguments:			
    output_folder -- name of the end file with the weekly ALEXI data
    End_filename -- name of the end file
    lonlim -- [ymin, ymax] (values must be between -60 and 70)
    latlim -- [xmin, xmax] (values must be between -180 and 180)			
    """

    try:
        # Collect account and FTP information
        username, password = WebAccounts.Accounts(Type='FTP_WA')
        ftpserver = "ftp.wateraccounting.unesco-ihe.org"

        # Set the file names and directories
        filename = "Gray_Water_Footprint.tif"
        local_filename = os.path.join(output_folder, filename)

        # Download data from FTP
        ftp = FTP(ftpserver)
        ftp.login(username, password)
        directory = "/WaterAccounting_Guest/Static_WA_Datasets/"
        ftp.cwd(directory)
        lf = open(local_filename, "wb")
        ftp.retrbinary("RETR " + filename, lf.write)
        lf.close()

        # Clip extend out of world data
        dataset, Geo_out = RC.clip_data(local_filename, latlim, lonlim)

        # make geotiff file
        DC.Save_as_tiff(name=filename_Out,
                        data=dataset,
                        geo=Geo_out,
                        projection="WGS84")

        # delete old tif file
        os.remove(local_filename)

    except:
        print "file not exists"

    return
Beispiel #8
0
def Download_ETens_from_WA_FTP(output_folder, Lat_tiles, Lon_tiles):           
    """
    This function retrieves ETensV1.0 data for a given date from the
    ftp.wateraccounting.unesco-ihe.org server.
				
    Restrictions:
    The data and this python file may not be distributed to others without
    permission of the WA+ team.

    Keyword arguments:
    output_folder -- Directory of the outputs
    Lat_tiles -- [Lat_min, Lat_max] Tile number of the max and min latitude tile number
    Lon_tiles -- [Lon_min, Lon_max] Tile number of the max and min longitude tile number				
    """      
    for v_tile in range(Lat_tiles[0], Lat_tiles[1]+1):
        for h_tile in range(Lon_tiles[0], Lon_tiles[1]+1):													

            Tilename = "h%sv%s.zip" %(h_tile, v_tile)
            if not os.path.exists(os.path.join(output_folder,Tilename)): 
                try:  
                    # Collect account and FTP information			
                    username, password = WebAccounts.Accounts(Type = 'FTP_WA')
                    FTP_name = "ftp://ftp.wateraccounting.unesco-ihe.org/WaterAccounting/Data_Satellite/Evaporation/ETensV1.0/%s" % Tilename
                    local_filename = os.path.join(output_folder, Tilename)   
			
                    # Download data from FTP 	
                    curl = pycurl.Curl()
                    curl.setopt(pycurl.URL, FTP_name)	
                    curl.setopt(pycurl.USERPWD, '%s:%s' %(username, password))								
                    fp = open(local_filename, "wb")								 
                    curl.setopt(pycurl.WRITEDATA, fp)
                    curl.perform()
                    curl.close()
                    fp.close()	
																
                except:
                    print "tile %s is not found and will be replaced by NaN values"	% Tilename
																
    return()									
Beispiel #9
0
def Collect_data(TilesHorizontal, TilesVertical, Date, output_folder,
                 hdf_library):
    '''
    This function downloads all the needed MODIS tiles from https://n5eil01u.ecs.nsidc.org/MOST/MOD10A2.006/ as a hdf file.

    Keywords arguments:
    TilesHorizontal -- [TileMin,TileMax] max and min horizontal tile number
    TilesVertical -- [TileMin,TileMax] max and min vertical tile number
    Date -- 'yyyy-mm-dd'
    output_folder -- 'C:/file/to/path/'
    '''

    # Make a new tile for the data
    sizeX = int((TilesHorizontal[1] - TilesHorizontal[0] + 1) * 2400)
    sizeY = int((TilesVertical[1] - TilesVertical[0] + 1) * 2400)
    DataTot = np.zeros((sizeY, sizeX))

    # Load accounts
    username, password = WebAccounts.Accounts(Type='NASA')

    # Download the MODIS FPAR data
    url = 'https://n5eil01u.ecs.nsidc.org/MOST/MOD10A2.006/' + Date.strftime(
        '%Y') + '.' + Date.strftime('%m') + '.' + Date.strftime('%d') + '/'

    dataset = requests.get(url, allow_redirects=False, stream=True)
    try:
        get_dataset = requests.get(dataset.headers['location'],
                                   auth=(username, password),
                                   stream=True).content
    except:
        from requests.packages.urllib3.exceptions import InsecureRequestWarning
        requests.packages.urllib3.disable_warnings(InsecureRequestWarning)
        get_dataset = requests.get(dataset.headers['location'],
                                   auth=(username, password),
                                   verify=False).content

    soup = BeautifulSoup(get_dataset, "lxml")

    if len(str(soup)) < 300:
        print 'Download was not succesfull, please check NASA account'
        sys.exit(1)

    # Create the Lat and Long of the MODIS tile in meters
    for Vertical in range(int(TilesVertical[0]), int(TilesVertical[1]) + 1):
        Distance = 231.65635826395834 * 2  # resolution of a MODIS pixel in meter
        countY = (TilesVertical[1] - TilesVertical[0] + 1) - (Vertical -
                                                              TilesVertical[0])

        for Horizontal in range(int(TilesHorizontal[0]),
                                int(TilesHorizontal[1]) + 1):
            countX = Horizontal - TilesHorizontal[0] + 1

            for i in soup.findAll('a',
                                  attrs={'href': re.compile('(?i)(hdf)$')}):

                # Find the file with the wanted tile number
                Vfile = str(i)[30:32]
                Hfile = str(i)[27:29]
                if int(Vfile) is int(Vertical) and int(Hfile) is int(
                        Horizontal):

                    # Define the whole url name
                    full_url = urlparse.urljoin(url, i['href'])

                    # Reset the begin parameters for downloading
                    downloaded = 0
                    N = 0

                    # if not downloaded try to download file
                    while downloaded == 0:

                        try:  # open http and download whole .hdf
                            nameDownload_url = full_url
                            file_name = os.path.join(
                                output_folder,
                                nameDownload_url.split('/')[-1])
                            if os.path.isfile(file_name):
                                downloaded = 1
                            else:
                                x = requests.get(nameDownload_url,
                                                 allow_redirects=False)
                                try:
                                    y = requests.get(x.headers['location'],
                                                     auth=(username, password))
                                except:
                                    from requests.packages.urllib3.exceptions import InsecureRequestWarning
                                    requests.packages.urllib3.disable_warnings(
                                        InsecureRequestWarning)

                                    y = requests.get(x.headers['location'],
                                                     auth=(username, password),
                                                     verify=False)
                                z = open(file_name, 'wb')
                                z.write(y.content)
                                z.close()
                                statinfo = os.stat(file_name)
                                # Say that download was succesfull
                                if int(statinfo.st_size) > 1000:
                                    downloaded = 1

                        # If download was not succesfull
                        except:

                            # Try another time
                            N = N + 1

# Stop trying after 10 times
                        if N == 10:
                            print 'Data from ' + Date.strftime(
                                '%Y-%m-%d') + ' is not available'
                            downloaded = 1

                    try:
                        # Open .hdf only band with SnowFrac and collect all tiles to one array
                        scale_factor = 1
                        dataset = gdal.Open(file_name)
                        sdsdict = dataset.GetMetadata('SUBDATASETS')
                        sdslist = [
                            sdsdict[k] for k in sdsdict.keys()
                            if '_1_NAME' in k
                        ]
                        sds = []

                        for n in sdslist:
                            sds.append(gdal.Open(n))
                            full_layer = [
                                i for i in sdslist if 'MOD_Grid_Snow_500m' in i
                            ]
                            idx = sdslist.index(full_layer[0])
                            if Horizontal == TilesHorizontal[
                                    0] and Vertical == TilesVertical[0]:
                                geo_t = sds[idx].GetGeoTransform()

                                # get the projection value
                                proj = sds[idx].GetProjection()

                            data = sds[idx].ReadAsArray()
                            countYdata = (TilesVertical[1] - TilesVertical[0] +
                                          2) - countY
                            DataTot[int((countYdata - 1) *
                                        2400):int(countYdata * 2400),
                                    int((countX - 1) *
                                        2400):int(countX *
                                                  2400)] = data * scale_factor
                        del data

                    # if the tile not exists or cannot be opened, create a nan array with the right projection
                    except:
                        if Horizontal == TilesHorizontal[
                                0] and Vertical == TilesVertical[0]:
                            x1 = (TilesHorizontal[0] - 19) * 2400 * Distance
                            x4 = (TilesVertical[0] - 9) * 2400 * -1 * Distance
                            geo = [x1, Distance, 0.0, x4, 0.0, -Distance]
                            geo_t = tuple(geo)

                        proj = 'PROJCS["unnamed",GEOGCS["Unknown datum based upon the custom spheroid",DATUM["Not specified (based on custom spheroid)",SPHEROID["Custom spheroid",6371007.181,0]],PRIMEM["Greenwich",0],UNIT["degree",0.0174532925199433]],PROJECTION["Sinusoidal"],PARAMETER["longitude_of_center",0],PARAMETER["false_easting",0],PARAMETER["false_northing",0],UNIT["Meter",1]]'
                        data = np.ones((2400, 2400)) * (-9999)
                        countYdata = (TilesVertical[1] - TilesVertical[0] +
                                      2) - countY
                        DataTot[(countYdata - 1) * 2400:countYdata * 2400,
                                (countX - 1) * 2400:countX *
                                2400] = data * 0.01

    # Make geotiff file
    name2 = os.path.join(output_folder, 'Merged.tif')
    driver = gdal.GetDriverByName("GTiff")
    dst_ds = driver.Create(name2, DataTot.shape[1], DataTot.shape[0], 1,
                           gdal.GDT_Float32, ['COMPRESS=LZW'])
    try:
        dst_ds.SetProjection(proj)
    except:
        proj = 'PROJCS["unnamed",GEOGCS["Unknown datum based upon the custom spheroid",DATUM["Not specified (based on custom spheroid)",SPHEROID["Custom spheroid",6371007.181,0]],PRIMEM["Greenwich",0],UNIT["degree",0.0174532925199433]],PROJECTION["Sinusoidal"],PARAMETER["longitude_of_center",0],PARAMETER["false_easting",0],PARAMETER["false_northing",0],UNIT["Meter",1]]'
        x1 = (TilesHorizontal[0] - 18) * 2400 * Distance
        x4 = (TilesVertical[0] - 9) * 2400 * -1 * Distance
        geo = [x1, Distance, 0.0, x4, 0.0, -Distance]
        geo_t = tuple(geo)
        dst_ds.SetProjection(proj)

    dst_ds.GetRasterBand(1).SetNoDataValue(-9999)
    dst_ds.SetGeoTransform(geo_t)
    dst_ds.GetRasterBand(1).WriteArray(DataTot)
    dst_ds = None
    sds = None
    return ()
Beispiel #10
0
def DownloadData(Dir, Startdate, Enddate, latlim, lonlim, Waitbar, cores, TimeCase):
    """
    This function downloads MSWEP Version 2.1 daily or monthly data

    Keyword arguments:
    Dir -- 'C:/file/to/path/'
    Var -- 'wind_f_inst' : (string) For all variable codes: VariablesInfo('day').descriptions.keys()
    Startdate -- 'yyyy-mm-dd'
    Enddate -- 'yyyy-mm-dd'
    latlim -- [ymin, ymax]
    lonlim -- [xmin, xmax]
    Waitbar -- 0 or 1 (1 is waitbar on)
    cores -- 1....8
    """

    # Load factors / unit / type of variables / accounts
    username, password = WebAccounts.Accounts(Type = 'MSWEP')

    # Set required data for the daily option
    if TimeCase == 'daily':

        # Define output folder and create this one if not exists
        path = os.path.join(Dir, 'Precipitation', 'MSWEP', 'daily')

        if not os.path.exists(path):
            os.makedirs(path)

        # Startdate if not defined
        sd_date = '1979-01-01'

        # Define Time frequency
        TimeFreq = 'D'

        # Define URL by using personal account
        url = 'https://%s:%[email protected]/opendap/MSWEP_V2.1/global_daily_010deg/' %(username,password)

        # Name the definition that will be used to obtain the data
        RetrieveData_fcn = RetrieveData_daily

    # Set required data for the monthly option
    elif TimeCase == 'monthly':

        # Define output folder and create this one if not exists
        path = os.path.join(Dir, 'Precipitation', 'MSWEP', 'monthly')

        if not os.path.exists(path):
            os.makedirs(path)

        # Startdate if not defined
        sd_date = '1979-01-01'

        # Define Time frequency
        TimeFreq = 'MS'

        # Define URL by using personal account
        url = 'https://%s:%[email protected]:443/opendap/MSWEP_V2.1/global_monthly_010deg.nc' %(username,password)

        # Name the definition that will be used to obtain the data
        RetrieveData_fcn = RetrieveData_monthly

    # If none of the possible option are chosen
    else:
        raise KeyError("The input time interval is not supported")

    # Define IDs (latitude/longitude)
    yID = np.int16(np.array([np.ceil((latlim[0] + 90) * 10),
                             np.floor((latlim[1] + 90) * 10)]))
    xID = np.int16(np.array([np.floor((lonlim[0] + 180) * 10),
                             np.ceil((lonlim[1] + 180) * 10)]))

    # Check dates. If no dates are given, the max number of days is used.
    if not Startdate:
        Startdate = pd.Timestamp(sd_date)
    if not Enddate:
        Enddate = pd.Timestamp('Now')  # Should be much than available

    # Create all dates that will be calculated
    Dates = pd.date_range(Startdate, Enddate, freq=TimeFreq)

    # Create Waitbar
    if Waitbar == 1:
        import wa.Functions.Start.WaitbarConsole as WaitbarConsole
        total_amount = len(Dates)
        amount = 0
        WaitbarConsole.printWaitBar(amount, total_amount, prefix = 'Progress:', suffix = 'Complete', length = 50)

    # Create one parameter with all the required arguments
    args = [path, url, TimeCase, xID, yID, lonlim, latlim, username, password]

    # Pass variables to parallel function and run
    if not cores:
        for Date in Dates:
            RetrieveData_fcn(Date, args)
            if Waitbar == 1:
                amount += 1
                WaitbarConsole.printWaitBar(amount, total_amount, prefix = 'Progress:', suffix = 'Complete', length = 50)
        results = True
    else:
        results = Parallel(n_jobs=cores)(delayed(RetrieveData_fcn)(Date, args)
                                         for Date in Dates)
    return results
Beispiel #11
0
def Collect_data(TilesHorizontal, TilesVertical, Date, output_folder,
                 hdf_library):
    '''
    This function downloads all the needed MODIS tiles from http://e4ftl01.cr.usgs.gov/MOLT/MOD17A3H.006/ as a hdf file.

    Keywords arguments:
    TilesHorizontal -- [TileMin,TileMax] max and min horizontal tile number
    TilesVertical -- [TileMin,TileMax] max and min vertical tile number
    Date -- 'yyyy-mm-dd'
    output_folder -- 'C:/file/to/path/'
    '''

    # Make a new tile for the data
    # NPP_SIZE = 2
    NPP_SIZE = 4
    sizeX = int(
        (TilesHorizontal[1] - TilesHorizontal[0] + 1) * 4800 / NPP_SIZE)
    sizeY = int((TilesVertical[1] - TilesVertical[0] + 1) * 4800 / NPP_SIZE)
    DataTot = np.zeros((sizeY, sizeX))

    # Load accounts
    username, password = WebAccounts.Accounts(Type='NASA')

    # Create the Lat and Long of the MODIS tile in meters
    for Vertical in range(int(TilesVertical[0]), int(TilesVertical[1]) + 1):
        Distance = 231.65635826395834 * NPP_SIZE  # resolution of a MODIS pixel in meter
        countY = (TilesVertical[1] - TilesVertical[0] + 1) - (Vertical -
                                                              TilesVertical[0])

        for Horizontal in range(int(TilesHorizontal[0]),
                                int(TilesHorizontal[1]) + 1):
            countX = Horizontal - TilesHorizontal[0] + 1

            # Download the MODIS NPP data
            #url = 'https://e4ftl01.cr.usgs.gov/MOLT/MOD17A3H.006/' + Date.strftime('%Y') + '.' + Date.strftime('%m') + '.' + Date.strftime('%d') + '/'
            url = 'https://e4ftl01.cr.usgs.gov/MOLT/MOD17A3.055/' + Date.strftime(
                '%Y') + '.' + Date.strftime('%m') + '.' + Date.strftime(
                    '%d') + '/'

            # Reset the begin parameters for downloading
            downloaded = 0
            N = 0

            # Check the library given by user
            if hdf_library is not None:
                os.chdir(hdf_library)
                hdf_name = glob.glob("MOD17A3.A%s%03s.h%02dv%02d.*" %
                                     (Date.strftime('%Y'), Date.strftime('%j'),
                                      Horizontal, Vertical))

                if len(hdf_name) == 1:
                    hdf_file = os.path.join(hdf_library, hdf_name[0])

                    if os.path.exists(hdf_file):
                        downloaded = 1
                        file_name = hdf_file

            if not downloaded == 1:
                # Get files on FTP server
                f = urllib2.urlopen(url)

                # Sum all the files on the server
                soup = BeautifulSoup(f, "lxml")
                for i in soup.findAll('a',
                                      attrs={'href':
                                             re.compile('(?i)(hdf)$')}):

                    # Find the file with the wanted tile number
                    #Vfile=str(i)[31:33]
                    # Hfile=str(i)[28:30]

                    Vfile = str(i)[30:32]
                    Hfile = str(i)[27:29]
                    if int(Vfile) is int(Vertical) and int(Hfile) is int(
                            Horizontal):

                        # Define the whole url name
                        full_url = urlparse.urljoin(url, i['href'])

                        # if not downloaded try to download file
                        while downloaded == 0:

                            try:  # open http and download whole .hdf
                                nameDownload = full_url
                                file_name = os.path.join(
                                    output_folder,
                                    nameDownload.split('/')[-1])
                                if os.path.isfile(file_name):
                                    print "file ", file_name, " already exists"
                                    downloaded = 1
                                else:
                                    x = requests.get(nameDownload,
                                                     allow_redirects=False)
                                    try:
                                        y = requests.get(x.headers['location'],
                                                         auth=(username,
                                                               password))
                                    except:
                                        from requests.packages.urllib3.exceptions import InsecureRequestWarning
                                        requests.packages.urllib3.disable_warnings(
                                            InsecureRequestWarning)

                                        y = requests.get(x.headers['location'],
                                                         auth=(username,
                                                               password),
                                                         verify=False)
                                    z = open(file_name, 'wb')
                                    z.write(y.content)
                                    z.close()
                                    statinfo = os.stat(file_name)
                                    # Say that download was succesfull
                                    if int(statinfo.st_size) > 10000:
                                        downloaded = 1

                            # If download was not succesfull
                            except:

                                # Try another time
                                N = N + 1

                                # Stop trying after 10 times
                                if N == 10:
                                    print 'Data from ' + Date.strftime(
                                        '%Y-%m-%d') + ' is not available'
                                    downloaded = 1
            try:
                # Open .hdf only band with NPP and collect all tiles to one array
                dataset = gdal.Open(file_name)
                sdsdict = dataset.GetMetadata('SUBDATASETS')
                #sdslist = [sdsdict[k] for k in sdsdict.keys() if '_2_NAME' in k]
                sdslist = [
                    sdsdict[k] for k in sdsdict.keys() if '_2_NAME' in k
                ]
                sds = []

                for n in sdslist:
                    sds.append(gdal.Open(n))
                    #yfull_layer = [i for i in sdslist if 'Npp_500m' in i]
                    full_layer = [i for i in sdslist if 'Npp_1km' in i]
                    idx = sdslist.index(full_layer[0])
                    if Horizontal == TilesHorizontal[
                            0] and Vertical == TilesVertical[0]:
                        geo_t = sds[idx].GetGeoTransform()

                        # get the projection value
                        proj = sds[idx].GetProjection()

                    data = sds[idx].ReadAsArray()
                    countYdata = (TilesVertical[1] - TilesVertical[0] +
                                  2) - countY
                    DataTot[int((countYdata - 1) * 4800 /
                                NPP_SIZE):int(countYdata * 4800 / NPP_SIZE),
                            int((countX - 1) * 4800 /
                                NPP_SIZE):int(countX * 4800 /
                                              NPP_SIZE)] = data * 0.0001
                del data

            # if the tile not exists or cannot be opened, create a nan array with the right projection
            except:
                if Horizontal == TilesHorizontal[
                        0] and Vertical == TilesVertical[0]:
                    x1 = (TilesHorizontal[0] - 19) * 4800 / NPP_SIZE * Distance
                    x4 = (TilesVertical[0] -
                          9) * 4800 / NPP_SIZE * -1 * Distance
                    geo = [x1, Distance, 0.0, x4, 0.0, -Distance]
                    geo_t = tuple(geo)

                proj = 'PROJCS["unnamed",GEOGCS["Unknown datum based upon the custom spheroid",DATUM["Not specified (based on custom spheroid)",SPHEROID["Custom spheroid",6371007.181,0]],PRIMEM["Greenwich",0],UNIT["degree",0.0174532925199433]],PROJECTION["Sinusoidal"],PARAMETER["longitude_of_center",0],PARAMETER["false_easting",0],PARAMETER["false_northing",0],UNIT["Meter",1]]'
                data = np.ones((4800 / NPP_SIZE, 4800 / NPP_SIZE)) * (0)
                countYdata = (TilesVertical[1] - TilesVertical[0] + 2) - countY
                DataTot[(countYdata - 1) * 4800 / NPP_SIZE:countYdata * 4800 /
                        NPP_SIZE, (countX - 1) * 4800 / NPP_SIZE:countX *
                        4800 / NPP_SIZE] = data * 0.0001

    # Make geotiff file
    DataTot[DataTot > 3.27] = -9999
    name2 = os.path.join(output_folder, 'Merged.tif')
    driver = gdal.GetDriverByName("GTiff")
    dst_ds = driver.Create(name2, DataTot.shape[1], DataTot.shape[0], 1,
                           gdal.GDT_Float32, ['COMPRESS=LZW'])
    try:
        dst_ds.SetProjection(proj)
    except:
        proj = 'PROJCS["unnamed",GEOGCS["Unknown datum based upon the custom spheroid",DATUM["Not specified (based on custom spheroid)",SPHEROID["Custom spheroid",6371007.181,0]],PRIMEM["Greenwich",0],UNIT["degree",0.0174532925199433]],PROJECTION["Sinusoidal"],PARAMETER["longitude_of_center",0],PARAMETER["false_easting",0],PARAMETER["false_northing",0],UNIT["Meter",1]]'
        x1 = (TilesHorizontal[0] - 18) * 4800 / NPP_SIZE * Distance
        x4 = (TilesVertical[0] - 9) * 4800 / NPP_SIZE * -1 * Distance
        geo = [x1, Distance, 0.0, x4, 0.0, -Distance]
        geo_t = tuple(geo)
        dst_ds.SetProjection(proj)

    dst_ds.GetRasterBand(1).SetNoDataValue(-9999)
    dst_ds.SetGeoTransform(geo_t)
    dst_ds.GetRasterBand(1).WriteArray(DataTot)
    dst_ds = None
    sds = None
    return ()
Beispiel #12
0
def RetrieveData(Date, args):
    """
    This function retrieves TRMM data for a given date from the
    ftp://disc2.nascom.nasa.gov server.

    Keyword arguments:
    Date -- 'yyyy-mm-dd'
    args -- A list of parameters defined in the DownloadData function.
    """
    # Argument
    [output_folder, TimeCase, xID, yID, lonlim, latlim] = args

    year = Date.year
    month = Date.month
    day = Date.day

    from wa import WebAccounts
    username, password = WebAccounts.Accounts(Type='NASA')

    # Create https
    if TimeCase == 'daily':
        URL = 'https://disc2.gesdisc.eosdis.nasa.gov/opendap/TRMM_L3/TRMM_3B42_Daily.7/%d/%02d/3B42_Daily.%d%02d%02d.7.nc4.ascii?precipitation[%d:1:%d][%d:1:%d]' % (
            year, month, year, month, day, xID[0], xID[1] - 1, yID[0],
            yID[1] - 1)
        DirFile = os.path.join(
            output_folder, "P_TRMM3B42.V7_mm-day-1_daily_%d.%02d.%02d.tif" %
            (year, month, day))
        Scaling = 1

    if TimeCase == 'monthly':
        if Date >= pd.Timestamp('2010-10-01'):
            URL = 'https://disc2.gesdisc.eosdis.nasa.gov/opendap/TRMM_L3/TRMM_3B43.7/%d/3B43.%d%02d01.7.HDF.ascii?precipitation[%d:1:%d][%d:1:%d]' % (
                year, year, month, xID[0], xID[1] - 1, yID[0], yID[1] - 1)

        else:
            URL = 'https://disc2.gesdisc.eosdis.nasa.gov/opendap/TRMM_L3/TRMM_3B43.7/%d/3B43.%d%02d01.7A.HDF.ascii?precipitation[%d:1:%d][%d:1:%d]' % (
                year, year, month, xID[0], xID[1] - 1, yID[0], yID[1] - 1)

        Scaling = calendar.monthrange(year, month)[1] * 24
        DirFile = os.path.join(
            output_folder,
            "P_TRMM3B43.V7_mm-month-1_monthly_%d.%02d.01.tif" % (year, month))

    if not os.path.isfile(DirFile):
        dataset = requests.get(URL, allow_redirects=False, stream=True)
        try:
            get_dataset = requests.get(dataset.headers['location'],
                                       auth=(username, password),
                                       stream=True)
        except:
            from requests.packages.urllib3.exceptions import InsecureRequestWarning
            requests.packages.urllib3.disable_warnings(InsecureRequestWarning)
            get_dataset = requests.get(dataset.headers['location'],
                                       auth=(username, password),
                                       verify=False)

        # download data (first save as text file)
        pathtext = os.path.join(output_folder, 'temp.txt')
        z = open(pathtext, 'w')
        z.write(get_dataset.content)
        z.close()

        # Open text file and remove header and footer
        data_start = np.genfromtxt(pathtext,
                                   dtype=float,
                                   skip_header=1,
                                   delimiter=',')
        data = data_start[:, 1:] * Scaling
        data[data < 0] = -9999
        data = data.transpose()
        data = np.flipud(data)

        # Delete .txt file
        os.remove(pathtext)

        # Make geotiff file
        geo = [lonlim[0], 0.25, 0, latlim[1], 0, -0.25]
        DC.Save_as_tiff(name=DirFile, data=data, geo=geo, projection="WGS84")

    return True
Beispiel #13
0
def DownloadData(Dir,
                 Var,
                 Startdate,
                 Enddate,
                 latlim,
                 lonlim,
                 Waitbar,
                 cores,
                 TimeCase,
                 CaseParameters,
                 gldas_version='2.1'):
    """
    This function downloads GLDAS Version 2 three-hourly, daily or monthly data

    Keyword arguments:
    Dir -- 'C:/file/to/path/'
    Var -- 'wind_f_inst' : (string) For all variable codes: VariablesInfo('day').descriptions.keys()
    Startdate -- 'yyyy-mm-dd'
    Enddate -- 'yyyy-mm-dd'
    latlim -- [ymin, ymax]
    lonlim -- [xmin, xmax]
    cores -- 1....8
    CaseParameters -- See files: three_hourly.py, daily.py, and monthly.py
    """

    # Load factors / unit / type of variables / accounts
    VarInfo = VariablesInfo(TimeCase)
    username, password = WebAccounts.Accounts(Type='NASA')

    # Set required data for the three hourly option
    if TimeCase == 'three_hourly':

        # Define output folder and create this one if not exists
        path = os.path.join(Dir, 'Weather_Data', 'Model', 'GLDAS', TimeCase,
                            Var)

        if not os.path.exists(path):
            os.makedirs(path)

        # Startdate if not defined
        sd_date = '2000-02-24'

        # Define Time frequency
        TimeFreq = 'D'

        # Define URL by using personal account
        #url = 'http://%s:%[email protected]:80/dods/GLDAS_NOAH025SUBP_3H' %(username,password)
        url = 'https://hydro1.gesdisc.eosdis.nasa.gov/dods/GLDAS_NOAH025_3H.{0}'.format(
            gldas_version)  #%(username,password)

        # Name the definition that will be used to obtain the data
        RetrieveData_fcn = RetrieveData_three_hourly

    # Set required data for the daily option
    elif TimeCase == 'daily':

        # seperate the daily case parameters
        SumMean, Min, Max = CaseParameters

        # Define output folder and create this one if not exists
        path = {
            'mean':
            os.path.join(Dir, 'Weather_Data', 'Model', 'GLDAS', TimeCase, Var,
                         'mean'),
            'min':
            os.path.join(Dir, 'Weather_Data', 'Model', 'GLDAS', TimeCase, Var,
                         'min'),
            'max':
            os.path.join(Dir, 'Weather_Data', 'Model', 'GLDAS', TimeCase, Var,
                         'max')
        }
        selected = np.array([SumMean, Min, Max])
        types = np.array(('mean', 'min', 'max'))[selected == 1]
        CaseParameters = [selected, types]
        for i in range(len(types)):
            if not os.path.exists(path[types[i]]):
                os.makedirs(path[types[i]])

        # Startdate if not defined
        sd_date = '2000-02-24'

        # Define Time frequency
        TimeFreq = 'D'

        # Define URL by using personal account
        #url = 'http://%s:%[email protected]:80/dods/GLDAS_NOAH025SUBP_3H' %(username,password)
        url = 'https://hydro1.gesdisc.eosdis.nasa.gov/dods/GLDAS_NOAH025_3H.{0}'.format(
            gldas_version)  #%(username,password)

        # Name the definition that will be used to obtain the data
        RetrieveData_fcn = RetrieveData_daily

    # Set required data for the monthly option
    elif TimeCase == 'monthly':

        # Define output folder and create this one if not exists
        path = os.path.join(Dir, 'Weather_Data', 'Model', 'GLDAS', TimeCase,
                            Var)
        if not os.path.exists(path):
            os.makedirs(path)
        CaseParameters = []

        # Startdate if not defined
        sd_date = '2000-03-01'

        # Define Time frequency
        TimeFreq = 'MS'

        # Define URL by using personal account
        #url = 'http://%s:%[email protected]:80/dods/GLDAS_NOAH025_M' %(username,password)
        url = 'https://hydro1.gesdisc.eosdis.nasa.gov/dods/GLDAS_NOAH025_M.{0}'.format(
            gldas_version)  #%(username,password)

        # Name the definition that will be used to obtain the data
        RetrieveData_fcn = RetrieveData_monthly

    # If none of the possible option are chosen
    else:
        raise KeyError("The input time interval is not supported")

    # Define IDs (latitude/longitude)
    yID = np.int16(
        np.array(
            [np.ceil((latlim[0] + 60) * 4),
             np.floor((latlim[1] + 60) * 4)]))
    xID = np.int16(
        np.array(
            [np.floor((lonlim[0] + 180) * 4),
             np.ceil((lonlim[1] + 180) * 4)]))

    # Check dates. If no dates are given, the max number of days is used.
    if not Startdate:
        Startdate = pd.Timestamp(sd_date)
    if not Enddate:
        Enddate = pd.Timestamp('Now')  # Should be much than available

    # Create all dates that will be calculated
    Dates = pd.date_range(Startdate, Enddate, freq=TimeFreq)

    # Create Waitbar
    if Waitbar == 1:
        import wa.Functions.Start.WaitbarConsole as WaitbarConsole
        total_amount = len(Dates)
        amount = 0
        WaitbarConsole.printWaitBar(amount,
                                    total_amount,
                                    prefix='Progress:',
                                    suffix='Complete',
                                    length=50)

    # Define the variable string name
    VarStr = VarInfo.names[Var]

    # Create one parameter with all the required arguments
    args = [
        path, url, Var, VarStr, VarInfo, TimeCase, xID, yID, lonlim, latlim,
        CaseParameters, username, password
    ]

    # Pass variables to parallel function and run
    if not cores:
        for Date in Dates:
            RetrieveData_fcn(Date, args)
            if Waitbar == 1:
                amount += 1
                WaitbarConsole.printWaitBar(amount,
                                            total_amount,
                                            prefix='Progress:',
                                            suffix='Complete',
                                            length=50)
        results = True
    else:
        results = Parallel(n_jobs=cores)(delayed(RetrieveData_fcn)(Date, args)
                                         for Date in Dates)
    return results