コード例 #1
0
def Download_ETmonitor_from_WA_FTP(local_filename, Filename_in, Type):
    """
    This function retrieves ETmonitor data for a given date from the
    ftp.wateraccounting.unesco-ihe.org server.

    Restrictions:
    The data and this python file may not be distributed to others without
    permission of the WA+ team due data restriction of the ETmonitor developers.

    Keyword arguments:
	 local_filename -- name of the temporary file which contains global ETmonitor data
    Filename_in -- name of the end file with the weekly ETmonitor data
	 Type = Type of data ("act" or "pot")
    """

    # Collect account and FTP information
    username, password = WebAccounts.Accounts(Type='FTP_WA')
    ftpserver = "ftp.wateraccounting.unesco-ihe.org"

    # Download data from FTP
    ftp = FTP(ftpserver)
    ftp.login(username, password)
    if Type == "pot":
        directory = "/WaterAccounting/Data_Satellite/Evaporation/ETmonitor/Potential_Evapotranspiration/"
    else:
        directory = "/WaterAccounting/Data_Satellite/Evaporation/ETmonitor/Global/"
    ftp.cwd(directory)
    lf = open(local_filename, "wb")
    ftp.retrbinary("RETR " + Filename_in, lf.write)
    lf.close()

    return
コード例 #2
0
ファイル: DataAccess.py プロジェクト: joan-gathu/watertools
def Download_SEBS_from_WA_FTP(local_filename, Filename_in):
    """
    This function retrieves SEBS data for a given date from the
    ftp.wateraccounting.unesco-ihe.org server.

    Restrictions:
    The data and this python file may not be distributed to others without
    permission of the WA+ team due data restriction of the SEBS developers.

    Keyword arguments:
	 local_filename -- name of the temporary file which contains global SEBS data
    Filename_in -- name of the end file with the monthly SEBS data
    """

    # Collect account and FTP information
    username, password = WebAccounts.Accounts(Type = 'FTP_WA')
    ftpserver = "ftp.wateraccounting.unesco-ihe.org"

    # Download data from FTP
    ftp=FTP(ftpserver)
    ftp.login(username,password)
    directory="/WaterAccounting_Guest/SEBS/Global_land_ET_V1/"
    ftp.cwd(directory)
    lf = open(local_filename, "wb")
    ftp.retrbinary("RETR " + Filename_in, lf.write)
    lf.close()

    return
コード例 #3
0
def RetrieveData(Date, args):
    """
    This function retrieves TRMM data for a given date from the
    ftp://disc2.nascom.nasa.gov server.

    Keyword arguments:
    Date -- 'yyyy-mm-dd'
    args -- A list of parameters defined in the DownloadData function.
    """
    # Argument
    [output_folder, TimeCase, xID, yID, lonlim, latlim] = args

    year = Date.year
    month= Date.month
    day = Date.day

    from watertools import WebAccounts
    username, password = WebAccounts.Accounts(Type = 'NASA')

    # Create https
    if TimeCase == 'daily':
        URL = 'https://gpm1.gesdisc.eosdis.nasa.gov/opendap/GPM_L3/GPM_3IMERGDF.05/%d/%02d/3B-DAY.MS.MRG.3IMERG.%d%02d%02d-S000000-E235959.V05.nc4.ascii?precipitationCal[%d:1:%d][%d:1:%d]'  %(year, month, year, month, day, xID[0], xID[1]-1, yID[0], yID[1]-1)
        DirFile = os.path.join(output_folder, "P_TRMM3B42.V7_mm-day-1_daily_%d.%02d.%02d.tif" %(year, month, day))
        Scaling = 1

    if TimeCase == 'monthly':
        URL = 'https://gpm1.gesdisc.eosdis.nasa.gov/opendap/hyrax/GPM_L3/GPM_3IMERGM.05/%d/3B-MO.MS.MRG.3IMERG.%d%02d01-S000000-E235959.%02d.V05B.HDF5.ascii?precipitation[%d:1:%d][%d:1:%d]'  %(year, year, month, month, xID[0], xID[1]-1, yID[0], yID[1]-1)
        Scaling = calendar.monthrange(year,month)[1] * 24
        DirFile = os.path.join(output_folder, "P_GPM.IMERG_mm-month-1_monthly_%d.%02d.01.tif" %(year, month))

    if not os.path.isfile(DirFile):
        dataset = requests.get(URL, allow_redirects=False,stream = True)
        try:
            get_dataset = requests.get(dataset.headers['location'], auth = (username,password),stream = True)
        except:
            from requests.packages.urllib3.exceptions import InsecureRequestWarning
            requests.packages.urllib3.disable_warnings(InsecureRequestWarning)
            get_dataset  = requests.get(dataset.headers['location'], auth = (username, password), verify = False)

        # download data (first save as text file)
        pathtext = os.path.join(output_folder,'temp.txt')
        z = open(pathtext,'wb')
        z.write(get_dataset.content)
        z.close()

        # Open text file and remove header and footer
        data_start = np.genfromtxt(pathtext,dtype = float,skip_header = 1,delimiter=',')
        data = data_start[:,1:] * Scaling
        data[data < 0] = -9999
        data = data.transpose()
        data = np.flipud(data)

        # Delete .txt file
        os.remove(pathtext)

        # Make geotiff file
        geo = [lonlim[0], 0.1, 0, latlim[1], 0, -0.1]
        DC.Save_as_tiff(name=DirFile, data=data, geo=geo, projection="WGS84")

    return True
コード例 #4
0
ファイル: DataAccess.py プロジェクト: joan-gathu/watertools
def Download_ALEXI_from_WA_FTP(local_filename, DirFile, filename, lonlim,
                               latlim, yID, xID, TimeStep):
    """
    This function retrieves ALEXI data for a given date from the
    ftp.wateraccounting.unesco-ihe.org server.

    Restrictions:
    The data and this python file may not be distributed to others without
    permission of the WA+ team due data restriction of the ALEXI developers.

    Keyword arguments:
	local_filename -- name of the temporary file which contains global ALEXI data
    DirFile -- name of the end file with the weekly ALEXI data
    filename -- name of the end file
    lonlim -- [ymin, ymax] (values must be between -60 and 70)
    latlim -- [xmin, xmax] (values must be between -180 and 180)
    """

    # Collect account and FTP information
    username, password = WebAccounts.Accounts(Type='FTP_WA')
    ftpserver = "ftp.wateraccounting.unesco-ihe.org"

    # Download data from FTP
    ftp = FTP(ftpserver)
    ftp.login(username, password)
    if TimeStep is "weekly":
        directory = "/WaterAccounting/Data_Satellite/Evaporation/ALEXI/World/"
    if TimeStep is "daily":
        directory = "/WaterAccounting/Data_Satellite/Evaporation/ALEXI/World_05182018/"
    ftp.cwd(directory)
    lf = open(local_filename, "wb")
    ftp.retrbinary("RETR " + filename, lf.write)
    lf.close()

    if TimeStep is "weekly":

        # Open global ALEXI data
        dataset = RC.Open_tiff_array(local_filename)

        # Clip extend out of world data
        data = dataset[yID[0]:yID[1], xID[0]:xID[1]]
        data[data < 0] = -9999

    if TimeStep is "daily":

        DC.Extract_Data_gz(local_filename, os.path.splitext(local_filename)[0])

        raw_data = np.fromfile(os.path.splitext(local_filename)[0],
                               dtype="<f4")
        dataset = np.flipud(np.resize(raw_data, [3000, 7200]))
        data = dataset[
            yID[0]:yID[1],
            xID[0]:xID[1]] / 2.45  # Values are in MJ/m2d so convert to mm/d
        data[data < 0] = -9999

    # make geotiff file
    geo = [lonlim[0], 0.05, 0, latlim[1], 0, -0.05]
    DC.Save_as_tiff(name=DirFile, data=data, geo=geo, projection="WGS84")
    return
コード例 #5
0
ファイル: DataAccess.py プロジェクト: joan-gathu/watertools
def Collect_data(FTPprefix, Years, output_folder, Waitbar, Product):
    '''
    This function downloads all the needed GLEAM files from hydras.ugent.be as a nc file.

    Keywords arguments:
    FTPprefix -- FTP path to the GLEAM data
    Date -- 'yyyy-mm-dd'
    output_folder -- 'C:/file/to/path/'
    '''
    # account of the SFTP server (only password is missing)
    server = 'hydras.ugent.be'
    portnumber = 2225

    username, password = WebAccounts.Accounts(Type='GLEAM')

    # Create Waitbar
    print('\nDownload GLEAM data')
    if Waitbar == 1:
        import watertools.Functions.Random.WaitbarConsole as WaitbarConsole
        total_amount2 = len(Years)
        amount2 = 0
        WaitbarConsole.printWaitBar(amount2,
                                    total_amount2,
                                    prefix='Progress:',
                                    suffix='Complete',
                                    length=50)

    for year in Years:
        directory = os.path.join(FTPprefix, '%d' % year)
        ssh = paramiko.SSHClient()
        ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
        ssh.connect(server,
                    port=portnumber,
                    username=username,
                    password=password)
        ftp = ssh.open_sftp()
        ftp.chdir(directory)

        if Product == "ET":
            filename = 'E_' + str(year) + '_GLEAM_v3.3b.nc'
        if Product == "ETpot":
            filename = 'Ep_' + str(year) + '_GLEAM_v3.3b.nc'
        local_filename = os.path.join(output_folder, filename)

        if not os.path.exists(local_filename):
            ftp.get(filename, local_filename)

        if Waitbar == 1:
            amount2 += 1
            WaitbarConsole.printWaitBar(amount2,
                                        total_amount2,
                                        prefix='Progress:',
                                        suffix='Complete',
                                        length=50)

    ftp.close()
    ssh.close()

    return ()
コード例 #6
0
ファイル: DataAccess.py プロジェクト: joan-gathu/watertools
def Download_ASCAT_from_VITO(End_filename, output_folder_temp, Date, yID, xID):
    """
    This function retrieves ALEXI data for a given date from the
    ftp.wateraccounting.unesco-ihe.org server.

    Restrictions:
    The data and this python file may not be distributed to others without
    permission of the WA+ team due data restriction of the ALEXI developers.

    Keyword arguments:

    """

    # Define date
    year_data = Date.year
    month_data = Date.month
    day_data = Date.day

    # filename of ASCAT data on server
    ASCAT_date = "%d%02d%02d1200" % (year_data, month_data, day_data)
    ASCAT_name = 'SWI_%s_GLOBE_ASCAT_V3.1.1' % ASCAT_date
    ASCAT_filename = "c_gls_SWI_%s_GLOBE_ASCAT_V3.1.1.nc" % ASCAT_date

    # Collect account and FTP information
    username, password = WebAccounts.Accounts(Type='Copernicus')
    URL = "https://land.copernicus.vgt.vito.be/PDF/datapool/Vegetation/Soil_Water/SWI_V3/%s/%s/%s/%s/%s" % (
        year_data, month_data, day_data, ASCAT_name, ASCAT_filename)

    # Output zipfile
    output_ncfile_ASCAT = os.path.join(output_folder_temp, ASCAT_filename)

    # Download the ASCAT data
    try:
        y = requests.get(URL, auth=HTTPBasicAuth(username, password))
    except:
        from requests.packages.urllib3.exceptions import InsecureRequestWarning
        requests.packages.urllib3.disable_warnings(InsecureRequestWarning)

        y = requests.get(URL, auth=(username, password), verify=False)

    # Write the file in system
    z = open(output_ncfile_ASCAT, 'wb')
    z.write(y.content)
    z.close()

    # Open nc file
    fh = Dataset(output_ncfile_ASCAT)
    dataset = fh.variables['SWI_010'][:, yID[0]:yID[1], xID[0]:xID[1]]
    data = np.squeeze(dataset.data, axis=0)
    data = data * 0.5
    data[data > 100.] = -9999
    fh.close()

    return (data)
コード例 #7
0
def Download_GWF_from_WA_FTP(output_folder, filename_Out, lonlim, latlim):
    """
    This function retrieves GWF data for a given date from the
    ftp.wateraccounting.unesco-ihe.org server.

    Keyword arguments:
    output_folder -- name of the end file with the weekly ALEXI data
    End_filename -- name of the end file
    lonlim -- [ymin, ymax] (values must be between -60 and 70)
    latlim -- [xmin, xmax] (values must be between -180 and 180)
    """

    try:
        # Collect account and FTP information
        username, password = WebAccounts.Accounts(Type='FTP_WA')
        ftpserver = "ftp.wateraccounting.unesco-ihe.org"

        # Set the file names and directories
        filename = "Gray_Water_Footprint.tif"
        local_filename = os.path.join(output_folder, filename)

        # Download data from FTP
        ftp = FTP(ftpserver)
        ftp.login(username, password)
        directory = "/WaterAccounting_Guest/Static_WA_Datasets/"
        ftp.cwd(directory)
        lf = open(local_filename, "wb")
        ftp.retrbinary("RETR " + filename, lf.write)
        lf.close()

        # Clip extend out of world data
        dataset, Geo_out = RC.clip_data(local_filename, latlim, lonlim)

        # make geotiff file
        DC.Save_as_tiff(name=filename_Out,
                        data=dataset,
                        geo=Geo_out,
                        projection="WGS84")

        # delete old tif file
        os.remove(local_filename)

    except:
        print("file not exists")

    return
コード例 #8
0
def Download_ETens_from_WA_FTP(output_folder, Lat_tiles, Lon_tiles):
    """
    This function retrieves ETensV1.0 data for a given date from the
    ftp.wateraccounting.unesco-ihe.org server.

    Restrictions:
    The data and this python file may not be distributed to others without
    permission of the WA+ team.

    Keyword arguments:
    output_folder -- Directory of the outputs
    Lat_tiles -- [Lat_min, Lat_max] Tile number of the max and min latitude tile number
    Lon_tiles -- [Lon_min, Lon_max] Tile number of the max and min longitude tile number
    """
    for v_tile in range(Lat_tiles[0], Lat_tiles[1] + 1):
        for h_tile in range(Lon_tiles[0], Lon_tiles[1] + 1):

            Tilename = "h%sv%s.zip" % (h_tile, v_tile)
            if not os.path.exists(os.path.join(output_folder, Tilename)):
                try:
                    # Collect account and FTP information
                    username, password = WebAccounts.Accounts(Type='FTP_WA')
                    FTP_name = "ftp://ftp.wateraccounting.unesco-ihe.org//WaterAccounting_Guest/ETensV1.0/%s" % Tilename
                    local_filename = os.path.join(output_folder, Tilename)

                    # Download data from FTP
                    curl = pycurl.Curl()
                    curl.setopt(pycurl.URL, FTP_name)
                    curl.setopt(pycurl.USERPWD, '%s:%s' % (username, password))
                    fp = open(local_filename, "wb")
                    curl.setopt(pycurl.WRITEDATA, fp)
                    curl.perform()
                    curl.close()
                    fp.close()

                except:
                    print(
                        "tile %s is not found and will be replaced by NaN values"
                        % Tilename)

    return ()
コード例 #9
0
def Collect_data(TilesHorizontal, TilesVertical, Date, output_folder, band,
                 resolution, hdf_library):
    '''
    This function downloads all the needed MODIS tiles from http://e4ftl01.cr.usgs.gov/MOLT/MOD13Q1.006/ as a hdf file.

    Keywords arguments:
    TilesHorizontal -- [TileMin,TileMax] max and min horizontal tile number
    TilesVertical -- [TileMin,TileMax] max and min vertical tile number
    Date -- 'yyyy-mm-dd'
    output_folder -- 'C:/file/to/path/'
    '''

    if band >= 3 and resolution == "250m":
        sys.exit('Bands higher than 3, are only available in 500m resolution')

    if resolution == "250m":
        size_factor = 1
    else:
        size_factor = 2

    # Make a new tile for the data
    sizeX = int(
        (TilesHorizontal[1] - TilesHorizontal[0] + 1) * 4800 / size_factor)
    sizeY = int((TilesVertical[1] - TilesVertical[0] + 1) * 4800 / size_factor)
    DataTot = np.zeros((sizeY, sizeX))

    # Load accounts
    username, password = WebAccounts.Accounts(Type='NASA')

    # Create the Lat and Long of the MODIS tile in meters
    for Vertical in range(int(TilesVertical[0]), int(TilesVertical[1]) + 1):
        Distance = 231.65635826395834 * size_factor  # resolution of a MODIS pixel in meter
        countY = (TilesVertical[1] - TilesVertical[0] + 1) - (Vertical -
                                                              TilesVertical[0])

        for Horizontal in range(int(TilesHorizontal[0]),
                                int(TilesHorizontal[1]) + 1):
            countX = Horizontal - TilesHorizontal[0] + 1

            # Download the MODIS NDVI data
            if resolution == "250m":
                url = 'https://e4ftl01.cr.usgs.gov/MOLT/MOD09GQ.006/' + Date.strftime(
                    '%Y') + '.' + Date.strftime('%m') + '.' + Date.strftime(
                        '%d') + '/'
            else:
                url = 'https://e4ftl01.cr.usgs.gov/MOLT/MOD09GA.006/' + Date.strftime(
                    '%Y') + '.' + Date.strftime('%m') + '.' + Date.strftime(
                        '%d') + '/'

# Reset the begin parameters for downloading
            downloaded = 0
            N = 0

            # Check the library given by user
            if hdf_library is not None:
                os.chdir(hdf_library)
                hdf_name = glob.glob("MOD09GA.A%s%03s.h%02dv%02d.*" %
                                     (Date.strftime('%Y'), Date.strftime('%j'),
                                      Horizontal, Vertical))

                if len(hdf_name) == 1:
                    hdf_file = os.path.join(hdf_library, hdf_name[0])

                    if os.path.exists(hdf_file):
                        downloaded = 1
                        file_name = hdf_file

            if not downloaded == 1:

                try:
                    # Get files on FTP server
                    if sys.version_info[0] == 3:
                        f = urllib.request.urlopen(url)

                    if sys.version_info[0] == 2:
                        f = urllib2.urlopen(url)
                    # Sum all the files on the server
                    soup = BeautifulSoup(f, "lxml")
                    for i in soup.findAll(
                            'a', attrs={'href': re.compile('(?i)(hdf)$')}):

                        # Find the file with the wanted tile number
                        Vfile = str(i)[30:32]
                        Hfile = str(i)[27:29]
                        if int(Vfile) is int(Vertical) and int(Hfile) is int(
                                Horizontal):

                            # Define the whole url name
                            if sys.version_info[0] == 3:
                                full_url = urllib.parse.urljoin(url, i['href'])

                            if sys.version_info[0] == 2:
                                full_url = urlparse.urljoin(url, i['href'])

                            # if not downloaded try to download file
                            while downloaded == 0:

                                try:  # open http and download whole .hdf
                                    nameDownload = full_url
                                    file_name = os.path.join(
                                        output_folder,
                                        nameDownload.split('/')[-1])
                                    if os.path.isfile(file_name):
                                        downloaded = 1
                                    else:
                                        x = requests.get(nameDownload,
                                                         allow_redirects=False)
                                        try:
                                            y = requests.get(
                                                x.headers['location'],
                                                auth=(username, password))
                                        except:
                                            from requests.packages.urllib3.exceptions import InsecureRequestWarning
                                            requests.packages.urllib3.disable_warnings(
                                                InsecureRequestWarning)
                                            y = requests.get(
                                                x.headers['location'],
                                                auth=(username, password),
                                                verify=False)
                                        z = open(file_name, 'wb')
                                        z.write(y.content)
                                        z.close()
                                        statinfo = os.stat(file_name)
                                        # Say that download was succesfull
                                        if int(statinfo.st_size) > 10000:
                                            downloaded = 1

                                # If download was not succesfull
                                except:

                                    # Try another time
                                    N = N + 1

        # Stop trying after 10 times
                            if N == 10:
                                print('Data from ' +
                                      Date.strftime('%Y-%m-%d') +
                                      ' is not available')
                                downloaded = 1

                except:
                    print("Url not found: %s" % url)

            try:
                # Open .hdf only band with NDVI and collect all tiles to one array
                dataset = gdal.Open(file_name)
                sdsdict = dataset.GetMetadata('SUBDATASETS')
                if resolution == "250m":
                    sdslist = [
                        sdsdict[k] for k in sdsdict.keys()
                        if 'SUBDATASET_%d_NAME' % int(band + 1) in k
                    ]
                else:
                    sdslist = [
                        sdsdict[k] for k in sdsdict.keys()
                        if 'SUBDATASET_%d_NAME' % int(band + 11) in k
                    ]
                sds = []

                for n in sdslist:
                    sds.append(gdal.Open(n))
                    full_layer = [
                        i for i in sdslist if 'sur_refl_b%02d_1' % band in i
                    ]
                    idx = sdslist.index(full_layer[0])
                    if Horizontal == TilesHorizontal[
                            0] and Vertical == TilesVertical[0]:
                        geo_t = sds[idx].GetGeoTransform()

                        # get the projection value
                        proj = sds[idx].GetProjection()

                    data = sds[idx].ReadAsArray()
                    countYdata = (TilesVertical[1] - TilesVertical[0] +
                                  2) - countY
                    DataTot[int((countYdata - 1) * 4800 /
                                size_factor):int(countYdata * 4800 /
                                                 size_factor),
                            int((countX - 1) * 4800 /
                                size_factor):int(countX * 4800 /
                                                 size_factor)] = data
                del data

            # if the tile not exists or cannot be opened, create a nan array with the right projection
            except:
                if Horizontal == TilesHorizontal[
                        0] and Vertical == TilesVertical[0]:
                    x1 = (TilesHorizontal[0] -
                          19) * 4800 / size_factor * Distance
                    x4 = (TilesVertical[0] -
                          9) * 4800 / size_factor * -1 * Distance
                    geo = [x1, Distance, 0.0, x4, 0.0, -Distance]
                    geo_t = tuple(geo)

                proj = 'PROJCS["unnamed",GEOGCS["Unknown datum based upon the custom spheroid",DATUM["Not specified (based on custom spheroid)",SPHEROID["Custom spheroid",6371007.181,0]],PRIMEM["Greenwich",0],UNIT["degree",0.0174532925199433]],PROJECTION["Sinusoidal"],PARAMETER["longitude_of_center",0],PARAMETER["false_easting",0],PARAMETER["false_northing",0],UNIT["Meter",1]]'
                data = np.ones((int(4800 / size_factor), int(
                    4800 / size_factor))) * (-9999)
                countYdata = (TilesVertical[1] - TilesVertical[0] + 2) - countY
                DataTot[int((countYdata - 1) * 4800 /
                            size_factor):int(countYdata * 4800 / size_factor),
                        int((countX - 1) * 4800 /
                            size_factor):int(countX * 4800 /
                                             size_factor)] = data

    # Make geotiff file
    name2 = os.path.join(output_folder, 'Merged.tif')
    driver = gdal.GetDriverByName("GTiff")
    dst_ds = driver.Create(name2, DataTot.shape[1], DataTot.shape[0], 1,
                           gdal.GDT_Float32, ['COMPRESS=LZW'])
    try:
        dst_ds.SetProjection(proj)
    except:
        proj = 'PROJCS["unnamed",GEOGCS["Unknown datum based upon the custom spheroid",DATUM["Not specified (based on custom spheroid)",SPHEROID["Custom spheroid",6371007.181,0]],PRIMEM["Greenwich",0],UNIT["degree",0.0174532925199433]],PROJECTION["Sinusoidal"],PARAMETER["longitude_of_center",0],PARAMETER["false_easting",0],PARAMETER["false_northing",0],UNIT["Meter",1]]'
        x1 = (TilesHorizontal[0] - 18) * 4800 / size_factor * Distance
        x4 = (TilesVertical[0] - 9) * 4800 / size_factor * -1 * Distance
        geo = [x1, Distance, 0.0, x4, 0.0, -Distance]
        geo_t = tuple(geo)
        dst_ds.SetProjection(proj)

    dst_ds.GetRasterBand(1).SetNoDataValue(-9999)
    dst_ds.SetGeoTransform(geo_t)
    dst_ds.GetRasterBand(1).WriteArray(DataTot * 0.0001)
    dst_ds = None
    sds = None
    return ()
コード例 #10
0
ファイル: DataAccess.py プロジェクト: joan-gathu/watertools
def DownloadData(Dir, Startdate, Enddate, latlim, lonlim, Waitbar, cores,
                 TimeCase):
    """
    This function downloads MSWEP Version 2.1 daily or monthly data

    Keyword arguments:
    Dir -- 'C:/file/to/path/'
    Var -- 'wind_f_inst' : (string) For all variable codes: VariablesInfo('day').descriptions.keys()
    Startdate -- 'yyyy-mm-dd'
    Enddate -- 'yyyy-mm-dd'
    latlim -- [ymin, ymax]
    lonlim -- [xmin, xmax]
    Waitbar -- 0 or 1 (1 is waitbar on)
    cores -- 1....8
    """

    # Load factors / unit / type of variables / accounts
    username, password = WebAccounts.Accounts(Type='MSWEP')

    # Set required data for the daily option
    if TimeCase == 'daily':

        # Define output folder and create this one if not exists
        path = os.path.join(Dir, 'Precipitation', 'MSWEP', 'daily')

        if not os.path.exists(path):
            os.makedirs(path)

        # Startdate if not defined
        sd_date = '1979-01-01'

        # Define Time frequency
        TimeFreq = 'D'

        # Define URL by using personal account
        url = 'https://%s:%[email protected]/opendap/MSWEP_V2.1/global_daily_010deg/' % (
            username, password)

        # Name the definition that will be used to obtain the data
        RetrieveData_fcn = RetrieveData_daily

    # Set required data for the monthly option
    elif TimeCase == 'monthly':

        # Define output folder and create this one if not exists
        path = os.path.join(Dir, 'Precipitation', 'MSWEP', 'monthly')

        if not os.path.exists(path):
            os.makedirs(path)

        # Startdate if not defined
        sd_date = '1979-01-01'

        # Define Time frequency
        TimeFreq = 'MS'

        # Define URL by using personal account
        url = 'https://%s:%[email protected]:443/opendap/MSWEP_V2.1/global_monthly_010deg.nc' % (
            username, password)

        # Name the definition that will be used to obtain the data
        RetrieveData_fcn = RetrieveData_monthly

    # If none of the possible option are chosen
    else:
        raise KeyError("The input time interval is not supported")

    # Define IDs (latitude/longitude)
    yID = np.int16(
        np.array(
            [np.ceil((latlim[0] + 90) * 10),
             np.floor((latlim[1] + 90) * 10)]))
    xID = np.int16(
        np.array([
            np.floor((lonlim[0] + 180) * 10),
            np.ceil((lonlim[1] + 180) * 10)
        ]))

    # Check dates. If no dates are given, the max number of days is used.
    if not Startdate:
        Startdate = pd.Timestamp(sd_date)
    if not Enddate:
        Enddate = pd.Timestamp('Now')  # Should be much than available

    # Create all dates that will be calculated
    Dates = pd.date_range(Startdate, Enddate, freq=TimeFreq)

    # Create Waitbar
    if Waitbar == 1:
        import watertools.Functions.Random.WaitbarConsole as WaitbarConsole
        total_amount = len(Dates)
        amount = 0
        WaitbarConsole.printWaitBar(amount,
                                    total_amount,
                                    prefix='Progress:',
                                    suffix='Complete',
                                    length=50)

    # Create one parameter with all the required arguments
    args = [path, url, TimeCase, xID, yID, lonlim, latlim, username, password]

    # Pass variables to parallel function and run
    if not cores:
        for Date in Dates:
            RetrieveData_fcn(Date, args)
            if Waitbar == 1:
                amount += 1
                WaitbarConsole.printWaitBar(amount,
                                            total_amount,
                                            prefix='Progress:',
                                            suffix='Complete',
                                            length=50)
        results = True
    else:
        results = Parallel(n_jobs=cores)(delayed(RetrieveData_fcn)(Date, args)
                                         for Date in Dates)
    return results
コード例 #11
0
def DownloadData(Dir, Var, Startdate, Enddate, latlim, lonlim, Waitbar,
                 CaseParameters, cores, TimeCase):
    """
    This function downloads GLDAS CLSM daily data

    Keyword arguments:
    Dir -- 'C:/file/to/path/'
    Var -- 'wind_f_inst' : (string) For all variable codes: VariablesInfo('day').descriptions.keys()
    Startdate -- 'yyyy-mm-dd'
    Enddate -- 'yyyy-mm-dd'
    latlim -- [ymin, ymax]
    lonlim -- [xmin, xmax]
    cores -- 1....8
    CaseParameters -- See files: three_hourly.py, daily.py, and monthly.py
    """

    # Load factors / unit / type of variables / accounts
    VarInfo = VariablesInfo(TimeCase)
    username, password = WebAccounts.Accounts(Type='NASA')

    # Set required data for the daily option
    # Set required data for the three hourly option
    if TimeCase == 'three_hourly':

        # Define output folder and create this one if not exists
        path = os.path.join(Dir, 'Weather_Data', 'Model', 'GLDAS', TimeCase,
                            Var)

        if not os.path.exists(path):
            os.makedirs(path)

        # Startdate if not defined
        sd_date = '1979-01-02'

        # Define Time frequency
        TimeFreq = 'D'

        # Define URL by using personal account
        #url = 'http://%s:%[email protected]:80/dods/GLDAS_NOAH025SUBP_3H' %(username,password)
        url = 'https://hydro1.gesdisc.eosdis.nasa.gov/dods/GLDAS_CLM10SUBP_3H'  #%(username,password)

        # Name the definition that will be used to obtain the data
        RetrieveData_fcn = RetrieveData_three_hourly

        types = ['mean']

    elif TimeCase == 'daily':

        types = ['mean']

        # Define output folder and create this one if not exists
        path = {
            'mean':
            os.path.join(Dir, 'Weather_Data', 'Model', 'GLDAS_CLSM', TimeCase,
                         Var, 'mean')
        }
        for i in range(len(types)):
            if not os.path.exists(path[types[i]]):
                os.makedirs(path[types[i]])

        # Startdate if not defined
        sd_date = '1948-01-01'

        # Define Time frequency
        TimeFreq = 'D'

        # Define URL by using personal account
        url = 'https://hydro1.gesdisc.eosdis.nasa.gov/dods/GLDAS_CLSM025_D.2.0'

        # Name the definition that will be used to obtain the data
        RetrieveData_fcn = RetrieveData_daily

    # Set required data for the monthly option
    elif TimeCase == 'monthly':

        types = ['mean']

        # Define output folder and create this one if not exists
        path = {
            'mean':
            os.path.join(Dir, 'Weather_Data', 'Model', 'GLDAS_CLSM', TimeCase,
                         Var, 'mean')
        }
        for i in range(len(types)):
            if not os.path.exists(path[types[i]]):
                os.makedirs(path[types[i]])

        # Startdate if not defined
        sd_date = '1979-01-02'

        # Define Time frequency
        TimeFreq = 'MS'

        # Define URL by using personal account
        url = 'https://hydro1.gesdisc.eosdis.nasa.gov/dods/GLDAS_CLSM025_D.2.0'

        # Name the definition that will be used to obtain the data
        RetrieveData_fcn = RetrieveData_monthly
    # If none of the possible option are chosen
    else:
        raise KeyError("The input time interval is not supported")

    if TimeCase == 'three_hourly':

        # Define IDs (latitude/longitude)
        yID = np.int16(
            np.array([np.ceil((latlim[0] + 60)),
                      np.floor((latlim[1] + 60))]))
        xID = np.int16(
            np.array([np.floor((lonlim[0] + 180)),
                      np.ceil((lonlim[1] + 180))]))
    else:

        # Define IDs (latitude/longitude)
        yID = np.int16(
            np.array([
                np.ceil((latlim[0] + 60) * 4),
                np.floor((latlim[1] + 60) * 4)
            ]))
        xID = np.int16(
            np.array([
                np.floor((lonlim[0] + 180) * 4),
                np.ceil((lonlim[1] + 180) * 4)
            ]))

    # Check dates. If no dates are given, the max number of days is used.
    if not Startdate:
        Startdate = pd.Timestamp(sd_date)
    if not Enddate:
        Enddate = pd.Timestamp('Now')  # Should be much than available

    # Create all dates that will be calculated
    Dates = pd.date_range(Startdate, Enddate, freq=TimeFreq)

    # Create Waitbar
    if Waitbar == 1:
        import watertools.Functions.Random.WaitbarConsole as WaitbarConsole
        total_amount = len(Dates)
        amount = 0
        WaitbarConsole.printWaitBar(amount,
                                    total_amount,
                                    prefix='Progress:',
                                    suffix='Complete',
                                    length=50)

    # Define the variable string name
    VarStr = VarInfo.names[Var]

    # Create one parameter with all the required arguments
    args = [
        path, url, Var, VarStr, VarInfo, TimeCase, xID, yID, lonlim, latlim,
        CaseParameters, username, password, types
    ]

    # Pass variables to parallel function and run
    if not cores:
        for Date in Dates:
            RetrieveData_fcn(Date, args)
            if Waitbar == 1:
                amount += 1
                WaitbarConsole.printWaitBar(amount,
                                            total_amount,
                                            prefix='Progress:',
                                            suffix='Complete',
                                            length=50)
        results = True
    else:
        results = Parallel(n_jobs=cores)(delayed(RetrieveData_fcn)(Date, args)
                                         for Date in Dates)
    return results
コード例 #12
0
ファイル: DataAccess.py プロジェクト: joan-gathu/watertools
def RetrieveData(Date, args):
    """
    This function retrieves MYD11 LST data for a given date from the
    https://e4ftl01.cr.usgs.gov/ server.

    Keyword arguments:
    Date -- 'yyyy-mm-dd'
    args -- A list of parameters defined in the DownloadData function.
    """

    # watertools modules
    import watertools.General.raster_conversions as RC
    import watertools.General.data_conversions as DC
    from watertools import WebAccounts

    # Argument
    [
        output_folder, TilesVertical, TilesHorizontal, lonlim, latlim,
        hdf_library
    ] = args

    # Define output names
    LSTfileNamePart = os.path.join(
        output_folder, 'LST_MYD11A1_K_daily_' + Date.strftime('%Y') + '.' +
        Date.strftime('%m') + '.' + Date.strftime('%d') + '*.tif')
    filesMOD = glob.glob(LSTfileNamePart)

    # Load accounts
    username, password = WebAccounts.Accounts(Type='NASA')

    if not len(filesMOD) == 1:

        # Collect the data from the MODIS webpage and returns the data and lat and long in meters of those tiles
        try:
            Collect_data(TilesHorizontal, TilesVertical, Date, username,
                         password, output_folder, hdf_library)
        except:
            print("Was not able to download the file")
        try:
            # Define the output name of the collect data function
            name_collect = os.path.join(output_folder, 'Merged.tif')
            name_collect_time = os.path.join(output_folder, 'Merged_Time.tif')

            # Reproject the MODIS product to epsg_to
            epsg_to = '4326'
            name_reprojected = RC.reproject_MODIS(name_collect, epsg_to)
            name_reprojected_time = RC.reproject_MODIS(name_collect_time,
                                                       epsg_to)

            # Clip the data to the users extend
            data, geo = RC.clip_data(name_reprojected, latlim, lonlim)
            data_time, geo = RC.clip_data(name_reprojected_time, latlim,
                                          lonlim)

            # remove wrong values
            data[data == 0.] = -9999
            data_time[data_time == 25.5] = np.nan
            data_time_ave = np.nanmean(data_time)
            try:
                hour_GMT = int(np.floor(data_time_ave))
                minutes_GMT = int(
                    (data_time_ave - np.floor(data_time_ave)) * 60)
            except:
                hour_GMT = int(12)
                minutes_GMT = int(0)
            LSTfileName = os.path.join(
                output_folder, 'LST_MYD11A1_K_daily_' + Date.strftime('%Y') +
                '.' + Date.strftime('%m') + '.' + Date.strftime('%d') +
                '.%02d%02d.tif' % (hour_GMT, minutes_GMT))

            # Save results as Gtiff
            DC.Save_as_tiff(name=LSTfileName,
                            data=data,
                            geo=geo,
                            projection='WGS84')

            # remove the side products
            os.remove(name_collect)
            os.remove(name_reprojected)
            os.remove(name_collect_time)
            os.remove(name_reprojected_time)
        except:
            print("Failed for date: %s" % Date)

    return True
コード例 #13
0
ファイル: DataAccess.py プロジェクト: joan-gathu/watertools
def Collect_data(TilesHorizontal, TilesVertical, Date, output_folder, timestep,
                 hdf_library, Size_pix):
    '''
    This function downloads all the needed MODIS tiles from ftp.ntsg.umt.edu/pub/MODIS/NTSG_Products/MOD16/MOD16A2_MONTHLY.MERRA_GMAO_1kmALB/ as a hdf file.

    Keywords arguments:
    TilesHorizontal -- [TileMin,TileMax] max and min horizontal tile number
    TilesVertical -- [TileMin,TileMax] max and min vertical tile number
    Date -- 'yyyy-mm-dd'
    output_folder -- 'C:/file/to/path/'
    '''

    # Make a new tile for the data
    sizeX = int(TilesHorizontal[1] - TilesHorizontal[0] + 1) * 1200 * Size_pix
    sizeY = int(TilesVertical[1] - TilesVertical[0] + 1) * 1200 * Size_pix
    DataTot = np.ones((sizeY, sizeX)) * -9999

    # Make a new tile for the lat and long info
    LatMet = np.zeros((sizeY))
    LongMet = np.zeros((sizeX))

    # Create the Lat and Long of the MODIS tile in meters
    for Vertical in range(int(TilesVertical[0]), int(TilesVertical[1]) + 1):
        Distance = 926.625 / Size_pix  # resolution of a MODIS pixel in meter
        countY = int((TilesVertical[1] - TilesVertical[0] + 1) -
                     (Vertical - TilesVertical[0]))
        LatMet[int((countY - 1) * 1200 *
                   Size_pix):int((countY) * 1200 * Size_pix)] = np.linspace(
                       ((8 - Vertical) * 1200 * Size_pix + 0.5) * Distance,
                       ((8 - Vertical) * 1200 * Size_pix + 1200 * Size_pix -
                        0.5) * Distance, 1200 * Size_pix)

        for Horizontal in range(int(TilesHorizontal[0]),
                                int(TilesHorizontal[1]) + 1):
            countX = int(Horizontal - TilesHorizontal[0] + 1)
            LongMet[int((countX - 1) * 1200 * Size_pix):int(
                (countX) * 1200 * Size_pix)] = np.linspace(
                    ((Horizontal - 18) * 1200 * Size_pix + 0.5) * Distance,
                    ((Horizontal - 18) * 1200 + 1200 * Size_pix - 0.5) *
                    Distance, 1200 * Size_pix)

            # Set the download to zero again
            downloaded = 0

            if hdf_library is not None:
                os.chdir(hdf_library)
                hdf_name = glob.glob("MOD16A2.A%s%03s.h%02dv%02d.*" %
                                     (Date.strftime('%Y'), Date.strftime('%j'),
                                      Horizontal, Vertical))

                if len(hdf_name) == 1:
                    hdf_file = os.path.join(hdf_library, hdf_name[0])

                    if os.path.exists(hdf_file):
                        downloaded = 1
                        data = Open_mod16_data(hdf_file)
                        countYdata = (TilesVertical[1] - TilesVertical[0] +
                                      2) - countY
                        DataTot[(int(countYdata) - 1) * 1200 *
                                Size_pix:int(countYdata) * 1200 * Size_pix,
                                (int(countX) - 1) * 1200 *
                                Size_pix:int(countX) * 1200 *
                                Size_pix] = data * 0.1

            while downloaded == 0:

                # Download the MODIS FPAR data
                if timestep == 'monthly':
                    url = 'http://files.ntsg.umt.edu/data/NTSG_Products/MOD16/MOD16A2_MONTHLY.MERRA_GMAO_1kmALB/Y%s/M%02s/' % (
                        Date.strftime('%Y'), Date.strftime('%m'))

                if timestep == '8-daily':
                    url = 'https://e4ftl01.cr.usgs.gov/MOLT/MOD16A2.006/%d.%02d.%02d/' % (
                        Date.year, Date.month, Date.day)

                try:

                    # Get files on FTP server
                    if sys.version_info[0] == 3:
                        f = urllib.request.urlopen(url)

                    if sys.version_info[0] == 2:
                        f = urllib2.urlopen(url)

                    # Sum all the files on the server
                    soup = BeautifulSoup(f, "lxml")

                    for i in soup.findAll(
                            'a', attrs={'href': re.compile('(?i)(hdf)$')}):

                        # Find the file with the wanted tile number
                        nameHDF = str(i)
                        HDF_name = nameHDF.split('>')[-2][:-3]
                        Hfile = HDF_name[18:20]
                        Vfile = HDF_name[21:23]
                        if int(Vfile) is int(Vertical) and int(Hfile) is int(
                                Horizontal):

                            HTTP_name = url + HDF_name
                            output_name = os.path.join(output_folder, HDF_name)

                            if timestep == 'monthly':

                                if not os.path.isfile(output_name):

                                    while downloaded == 0:

                                        if sys.version_info[0] == 2:
                                            urllib.urlretrieve(
                                                HTTP_name, output_name)
                                        if sys.version_info[0] == 3:
                                            urllib.request.urlretrieve(
                                                HTTP_name, output_name)

                                        statinfo = os.stat(output_name)
                                        # Say that download was succesfull
                                        if int(statinfo.st_size) > 1000:
                                            downloaded = 1

                            if timestep == '8-daily':

                                # Reset the begin parameters for downloading
                                N = 0
                                username, password = WebAccounts.Accounts(
                                    Type='NASA')

                                # if not downloaded try to download file
                                while downloaded == 0:

                                    try:  # open http and download whole .hdf
                                        file_name = os.path.join(
                                            output_folder,
                                            HTTP_name.split('/')[-1])
                                        if os.path.isfile(output_name):
                                            downloaded = 1
                                        else:
                                            x = requests.get(
                                                HTTP_name,
                                                allow_redirects=False)
                                            try:
                                                y = requests.get(
                                                    x.headers['location'],
                                                    auth=(username, password))
                                            except:
                                                from requests.packages.urllib3.exceptions import InsecureRequestWarning
                                                requests.packages.urllib3.disable_warnings(
                                                    InsecureRequestWarning)

                                                y = requests.get(
                                                    x.headers['location'],
                                                    auth=(username, password),
                                                    verify=False)

                                            z = open(output_name, 'wb')
                                            z.write(y.content)
                                            z.close()
                                            statinfo = os.stat(file_name)
                                            # Say that download was succesfull
                                            if int(statinfo.st_size) > 1000:
                                                downloaded = 1

                                    # If download was not succesfull
                                    except:

                                        # Try another time
                                        N = N + 1

            # Stop trying after 10 times
                                    if N == 10:
                                        print('Data from ' +
                                              Date.strftime('%Y-%m-%d') +
                                              ' is not available')
                                        downloaded = 1

                            # Open .hdf only band with ET and collect all tiles to one array
                            data = Open_mod16_data(output_name)
                            countYdata = (TilesVertical[1] - TilesVertical[0] +
                                          2) - countY
                            DataTot[(int(countYdata) - 1) * 1200 *
                                    Size_pix:int(countYdata) * 1200 * Size_pix,
                                    (int(countX) - 1) * 1200 *
                                    Size_pix:int(countX) * 1200 *
                                    Size_pix] = data * 0.1
                            DataTot[DataTot > 3000] = -9999
                            downloaded = 1
                            del data

                except:
                    proj = 'PROJCS["unnamed",GEOGCS["Unknown datum based upon the custom spheroid",DATUM["Not specified (based on custom spheroid)",SPHEROID["Custom spheroid",6371007.181,0]],PRIMEM["Greenwich",0],UNIT["degree",0.0174532925199433]],PROJECTION["Sinusoidal"],PARAMETER["longitude_of_center",0],PARAMETER["false_easting",0],PARAMETER["false_northing",0],UNIT["Meter",1]]'
                    data = np.ones(
                        (1200 * Size_pix, 1200 * Size_pix)) * (-9999)
                    countYdata = (TilesVertical[1] - TilesVertical[0] +
                                  2) - countY
                    DataTot[(countYdata - 1) * 1200 * Size_pix:countYdata *
                            1200 * Size_pix, (countX - 1) * 1200 *
                            Size_pix:countX * 1200 * Size_pix] = data * 0.1
                    downloaded = 1

# Make geotiff file
    name2 = os.path.join(output_folder, 'Merged.tif')
    driver = gdal.GetDriverByName("GTiff")
    dst_ds = driver.Create(name2, DataTot.shape[1], DataTot.shape[0], 1,
                           gdal.GDT_Float32, ['COMPRESS=LZW'])
    try:
        dst_ds.SetProjection(proj)
    except:
        proj = 'PROJCS["unnamed",GEOGCS["Unknown datum based upon the custom spheroid",DATUM["Not specified (based on custom spheroid)",SPHEROID["Custom spheroid",6371007.181,0]],PRIMEM["Greenwich",0],UNIT["degree",0.0174532925199433]],PROJECTION["Sinusoidal"],PARAMETER["longitude_of_center",0],PARAMETER["false_easting",0],PARAMETER["false_northing",0],UNIT["Meter",1]]'
        x1 = (TilesHorizontal[0] - 18) * 1200 * Size_pix * Distance
        x4 = (TilesVertical[0] - 9) * 1200 * Size_pix * -1 * Distance
        geo = [x1, Distance, 0.0, x4, 0.0, -Distance]
        geo_t = tuple(geo)
        dst_ds.SetProjection(proj)

    dst_ds.GetRasterBand(1).SetNoDataValue(-9999)
    dst_ds.SetGeoTransform(geo_t)
    dst_ds.GetRasterBand(1).WriteArray(DataTot)
    dst_ds = None

    return (DataTot, LatMet, LongMet)
コード例 #14
0
ファイル: DataAccess.py プロジェクト: joan-gathu/watertools
def Collect_data(TilesHorizontal, TilesVertical, Date, output_folder,
                 hdf_library):
    '''
    This function downloads all the needed MODIS tiles from https://n5eil01u.ecs.nsidc.org/MOST/MOD10A2.006/ as a hdf file.

    Keywords arguments:
    TilesHorizontal -- [TileMin,TileMax] max and min horizontal tile number
    TilesVertical -- [TileMin,TileMax] max and min vertical tile number
    Date -- 'yyyy-mm-dd'
    output_folder -- 'C:/file/to/path/'
    '''

    # Make a new tile for the data
    sizeX = int((TilesHorizontal[1] - TilesHorizontal[0] + 1) * 2400)
    sizeY = int((TilesVertical[1] - TilesVertical[0] + 1) * 2400)
    DataTot = np.zeros((sizeY, sizeX))

    # Load accounts
    username, password = WebAccounts.Accounts(Type='NASA')

    # Download the MODIS FPAR data
    url = 'https://n5eil01u.ecs.nsidc.org/MOST/MOD10A2.006/' + Date.strftime(
        '%Y') + '.' + Date.strftime('%m') + '.' + Date.strftime('%d') + '/'

    dataset = requests.get(url, allow_redirects=False, stream=True)
    try:
        get_dataset = requests.get(dataset.headers['location'],
                                   auth=(username, password),
                                   stream=True).content
    except:
        from requests.packages.urllib3.exceptions import InsecureRequestWarning
        requests.packages.urllib3.disable_warnings(InsecureRequestWarning)
        get_dataset = requests.get(dataset.headers['location'],
                                   auth=(username, password),
                                   verify=False).content

    soup = BeautifulSoup(get_dataset, "lxml")

    if len(str(soup)) < 300:
        print('Download was not succesfull, please check NASA account')
        sys.exit(1)

    # Create the Lat and Long of the MODIS tile in meters
    for Vertical in range(int(TilesVertical[0]), int(TilesVertical[1]) + 1):
        Distance = 231.65635826395834 * 2  # resolution of a MODIS pixel in meter
        countY = (TilesVertical[1] - TilesVertical[0] + 1) - (Vertical -
                                                              TilesVertical[0])

        for Horizontal in range(int(TilesHorizontal[0]),
                                int(TilesHorizontal[1]) + 1):
            countX = Horizontal - TilesHorizontal[0] + 1

            for i in soup.findAll('a',
                                  attrs={'href': re.compile('(?i)(hdf)$')}):

                # Find the file with the wanted tile number
                Vfile = str(i)[30:32]
                Hfile = str(i)[27:29]
                if int(Vfile) is int(Vertical) and int(Hfile) is int(
                        Horizontal):

                    # Define the whole url name
                    if sys.version_info[0] == 3:
                        full_url = urllib.parse.urljoin(url, i['href'])

                    if sys.version_info[0] == 2:
                        full_url = urlparse.urljoin(url, i['href'])

# Reset the begin parameters for downloading
                    downloaded = 0
                    N = 0

                    # if not downloaded try to download file
                    while downloaded == 0:

                        try:  # open http and download whole .hdf
                            nameDownload_url = full_url
                            file_name = os.path.join(
                                output_folder,
                                nameDownload_url.split('/')[-1])
                            if os.path.isfile(file_name):
                                downloaded = 1
                            else:
                                x = requests.get(nameDownload_url,
                                                 allow_redirects=False)
                                try:
                                    y = requests.get(x.headers['location'],
                                                     auth=(username, password))
                                except:
                                    from requests.packages.urllib3.exceptions import InsecureRequestWarning
                                    requests.packages.urllib3.disable_warnings(
                                        InsecureRequestWarning)

                                    y = requests.get(x.headers['location'],
                                                     auth=(username, password),
                                                     verify=False)
                                z = open(file_name, 'wb')
                                z.write(y.content)
                                z.close()
                                statinfo = os.stat(file_name)
                                # Say that download was succesfull
                                if int(statinfo.st_size) > 1000:
                                    downloaded = 1

                        # If download was not succesfull
                        except:

                            # Try another time
                            N = N + 1

# Stop trying after 10 times
                        if N == 10:
                            print('Data from ' + Date.strftime('%Y-%m-%d') +
                                  ' is not available')
                            downloaded = 1

                    try:
                        # Open .hdf only band with SnowFrac and collect all tiles to one array
                        scale_factor = 1
                        dataset = gdal.Open(file_name)
                        sdsdict = dataset.GetMetadata('SUBDATASETS')
                        sdslist = [
                            sdsdict[k] for k in sdsdict.keys()
                            if '_1_NAME' in k
                        ]
                        sds = []

                        for n in sdslist:
                            sds.append(gdal.Open(n))
                            full_layer = [
                                i for i in sdslist if 'MOD_Grid_Snow_500m' in i
                            ]
                            idx = sdslist.index(full_layer[0])
                            if Horizontal == TilesHorizontal[
                                    0] and Vertical == TilesVertical[0]:
                                geo_t = sds[idx].GetGeoTransform()

                                # get the projection value
                                proj = sds[idx].GetProjection()

                            data = sds[idx].ReadAsArray()
                            countYdata = (TilesVertical[1] - TilesVertical[0] +
                                          2) - countY
                            DataTot[int((countYdata - 1) *
                                        2400):int(countYdata * 2400),
                                    int((countX - 1) *
                                        2400):int(countX *
                                                  2400)] = data * scale_factor
                        del data

                    # if the tile not exists or cannot be opened, create a nan array with the right projection
                    except:
                        if Horizontal == TilesHorizontal[
                                0] and Vertical == TilesVertical[0]:
                            x1 = (TilesHorizontal[0] - 19) * 2400 * Distance
                            x4 = (TilesVertical[0] - 9) * 2400 * -1 * Distance
                            geo = [x1, Distance, 0.0, x4, 0.0, -Distance]
                            geo_t = tuple(geo)

                        proj = 'PROJCS["unnamed",GEOGCS["Unknown datum based upon the custom spheroid",DATUM["Not specified (based on custom spheroid)",SPHEROID["Custom spheroid",6371007.181,0]],PRIMEM["Greenwich",0],UNIT["degree",0.0174532925199433]],PROJECTION["Sinusoidal"],PARAMETER["longitude_of_center",0],PARAMETER["false_easting",0],PARAMETER["false_northing",0],UNIT["Meter",1]]'
                        data = np.ones((2400, 2400)) * (-9999)
                        countYdata = (TilesVertical[1] - TilesVertical[0] +
                                      2) - countY
                        DataTot[(countYdata - 1) * 2400:countYdata * 2400,
                                (countX - 1) * 2400:countX *
                                2400] = data * 0.01

    # Make geotiff file
    name2 = os.path.join(output_folder, 'Merged.tif')
    driver = gdal.GetDriverByName("GTiff")
    dst_ds = driver.Create(name2, DataTot.shape[1], DataTot.shape[0], 1,
                           gdal.GDT_Float32, ['COMPRESS=LZW'])
    try:
        dst_ds.SetProjection(proj)
    except:
        proj = 'PROJCS["unnamed",GEOGCS["Unknown datum based upon the custom spheroid",DATUM["Not specified (based on custom spheroid)",SPHEROID["Custom spheroid",6371007.181,0]],PRIMEM["Greenwich",0],UNIT["degree",0.0174532925199433]],PROJECTION["Sinusoidal"],PARAMETER["longitude_of_center",0],PARAMETER["false_easting",0],PARAMETER["false_northing",0],UNIT["Meter",1]]'
        x1 = (TilesHorizontal[0] - 18) * 2400 * Distance
        x4 = (TilesVertical[0] - 9) * 2400 * -1 * Distance
        geo = [x1, Distance, 0.0, x4, 0.0, -Distance]
        geo_t = tuple(geo)
        dst_ds.SetProjection(proj)

    dst_ds.GetRasterBand(1).SetNoDataValue(-9999)
    dst_ds.SetGeoTransform(geo_t)
    dst_ds.GetRasterBand(1).WriteArray(DataTot)
    dst_ds = None
    sds = None
    return ()