def main(args): # vendo lista de tiles if args.tile == None and args.uf == None: print 'É preciso definir uf ou tile' return if args.tile: tiles = str(args.tile) else: UF = eval('estados_v2.' + args.uf) tiles = UF.tiles if args.data_final != None: # usuário esta trocando a data de download modisDown = downmodis.downModis(destinationFolder=args.dest_dir, tiles=tiles, product=args.produto, today=args.data_final, delta=args.delta) else: modisDown = downmodis.downModis(destinationFolder=args.dest_dir, tiles=tiles, product=args.produto, delta=args.delta) modisDown.connect() modisDown.downloadsAllDay() return
def download(self): modisObj = downmodis.downModis(url="https://e4ftl01.cr.usgs.gov", user="******", password='******', path="MOLT", timeout=3600, destinationFolder=self.targetPath, jpg=False, debug=True, tiles=self.tiles, today=self.end, enddate=self.start, product=self.productName) continueLoop = True while (continueLoop): try: print('download') modisObj.connect() modisObj.downloadsAllDay(clean=True, allDays=False) continueLoop = False except: traceback.print_exc() time.sleep(10) print('try again') pass
def getMODISDataFromURL(output_dir, product, tiles, dates, mosaic_dir, tools_dir = "", logger = None): files = None m_files = [] if tiles == '': tiles=None for d in dates: if logger: logger.debug("downloading %s", d) _month = d + '-01' _folder_date = datetime.datetime.strptime(_month, '%Y-%m-%d').strftime('%Y.%m.%d') _new_folder = os.path.join(output_dir, _folder_date) # create folder if it doesn't already exist if not os.path.exists(_new_folder): os.makedirs(_new_folder) _delta = 1 modisDown = downmodis.downModis(destinationFolder=_new_folder, tiles=tiles, product=product, today=_month, delta=_delta) modisDown.connect() _check_files = modisDown.getFilesList(_folder_date) if logger: logger.debug("check files: %s", _check_files) dl_files = modisDown.checkDataExist(_check_files) try: modisDown.downloadsAllDay(clean=True) except Exception, e: if logger: logger.error("Error in pymodis.modisDown.downloadsAllDay") modisDown.removeEmptyFiles() modisDown.closeFilelist() _files = glob.glob(os.path.join(_new_folder, '{product}*.hdf'.format(product=product[:-4]))) if logger: logger.debug("files: %s", files) if logger: logger.debug("dl_files: %s", dl_files) if mosaic_dir: # mosaic files mosaic_file = mosaicTiles(_files, mosaic_dir, tools_dir) m_files.append(mosaic_file)
def getMODISDataFromURL(output_dir, product, tiles, dates, mosaic_dir, tools_dir = "", logger = None): files = None m_files = [] if tiles == '': tiles=None # # new setup to handle authentication # username = '******' # password = '******' # password_manager = urllib2.HTTPPasswordMgrWithDefaultRealm() # password_manager.add_password(None, "https://urs.earthdata.nasa.gov", username, password) # # cookie_jar = CookieJar() # # # Install all the handlers. # # opener = urllib2.build_opener( # urllib2.HTTPBasicAuthHandler(password_manager), # # urllib2.HTTPHandler(debuglevel=1), # Uncomment these two lines to see # # urllib2.HTTPSHandler(debuglevel=1), # details of the requests/responses # urllib2.HTTPCookieProcessor(cookie_jar)) # urllib2.install_opener(opener) # end of new setup to handle authentication for d in dates: if logger: logger.debug("downloading %s", d) _month = d + '-01' _folder_date = datetime.datetime.strptime(_month, '%Y-%m-%d').strftime('%Y.%m.%d') _new_folder = os.path.join(output_dir, _folder_date) # create folder if it doesn't already exist if not os.path.exists(_new_folder): os.makedirs(_new_folder) _delta = 1 modisDown = downmodis.downModis(destinationFolder=_new_folder, tiles=tiles, product=product, today=_month, delta=_delta, user='******', password='******') modisDown.connect() _check_files = modisDown.getFilesList(_folder_date) if logger: logger.debug("check files: %s", _check_files) dl_files = modisDown.checkDataExist(_check_files) try: modisDown.downloadsAllDay(clean=True) except Exception, e: if logger: logger.error("Error in pymodis.modisDown.downloadsAllDay") modisDown.removeEmptyFiles() modisDown.closeFilelist() _files = glob.glob(os.path.join(_new_folder, '{product}*.hdf'.format(product=product[:-4]))) if logger: logger.debug("files: %s", _files) if logger: logger.debug("dl_files: %s", dl_files) if mosaic_dir: # mosaic files mosaic_file = mosaicTiles(_files, mosaic_dir, tools_dir) m_files.append(mosaic_file)
def download(): # 此时,我们可以初始化downmodis对象。我们将在“/tmp”目录下下载MODIS LST产品 modisDown = downmodis.downModis(destinationFolder=dest, password="******", user="******", tiles=tiles, today=day, delta=delta) modisDown.connect() #此时,我们开始下载文件。 modisDown.downloadsAllDay() files = glob.glob(os.path.join(dest, 'MOD11A1.A2014*.hdf')) print(files)
def performDownload(tilelist, dates, i, work_path): ''' Downloads MODIS tiles from USGS ftp server and save them to the appropriate date folder ''' print "" print "Working on {0} to {1}".format(dates[i], dates[i + 1]) print "Files are saved in {}".format(work_path) destination = work_path # make downloader dm = pmod.downModis(destinationFolder=destination, path="MOLT", product="MOD13A3.006", tiles="{}".format(tilelist), today="{}".format(dates[i]), enddate="{}".format(dates[i + 1])) dm.connect() print "Connection Attempts: " + str(dm.nconnection) # Download all the files in that day print "Please Wait....................." dm.downloadsAllDay()
def downloadFiles(day): # Variables for data download dest = "data/" # This directory must already exist BTW # enddate = "2018-10-11" # The download works backward, so that enddate is anterior to day= product = "MOD09GA.006" # Instantiate download class, connect and download modis_down = downmodis.downModis(destinationFolder=dest, tiles=TILES, user="******", password="******", today=day, delta=1, product=product) modis_down.connect() modis_down.downloadsAllDay() # Check that the data has been downloaded MODIS_files = glob.glob(dest + '*.hdf')
def DownLoadData(LastKnownDate, CurrentDate): # Set the destination folder dest = current_dir + '/RawData' # These are four tiles that cover Kenya tiles = 'h21v08,h21v09,h22v08,h22v09' # Using the Pymodis module to download the 4 tiles from the server Modis_Down = downModis(destinationFolder=dest, tiles=tiles, url='https://e4ftl01.cr.usgs.gov', path='MOTA', product='MCD43A4.006', user='******', password='******', today=CurrentDate, enddate=LastKnownDate) Modis_Down.connect() Modis_Down.downloadsAllDay()
def download(self): modisObj = downmodis.downModis( url="http://e4ftl01.cr.usgs.gov", user="******", password=None, path="MOLT", timeout=3600, destinationFolder=self.targetPath, jpg=False, debug=True, tiles=self.tiles, today=self.end, enddate=self.start, product=self.productName) continueLoop = True while(continueLoop): try: print('download') modisObj.connect() modisObj.downloadsAllDay(clean=True, allDays=False) continueLoop = False except: traceback.print_exc() time.sleep(10) print('try again') pass
def download_modis(user, pw, prod, wdir): """ wrapper for downmodis functions. Non-changing vars (e.g., user/pw) left in global scope. """ def list_present_dates(wdir=wdir, tile="1"): """ Compile list of dates already present in MODIS_DIR for particular source/tile. Done to avoid redundant downloads, as this script may have to be applied more than once to download all files. Takes to minimum of what is available for both tiles """ dates_present = [] modis_files = list(wdir.glob("*h1{}v02*.hdf".format(tile))) # keep = [os.stat(file).st_size > 0 for file in modis_files] # files_present = list(itertools.compress(modis_files, keep)) for file in modis_files: """ files of size 0 are assumed to be unsuccessful downloads, need to be removed for .dayDownload() to re-download """ if os.stat(file).st_size == 0: print("Removing empty file: {}".format(str(file).split("/")[-1])) os.remove(file) continue jday = int(str(file).split(".")[1][5:]) year = int(str(file).split(".")[1][1:5]) date = datetime.datetime(year, 1, 1) + datetime.timedelta(jday - 1) dates_present.append(date.strftime("%Y.%m.%d")) return dates_present # choose aqua or terra product = "M{}D11A2.006".format(prod) prod_info = { "MOD11A2.006": ["MOLT", "2000-02-24"], "MYD11A2.006": ["MOLA", "2002-07-24"], } path = prod_info[product][0] start_date = prod_info[product][1] # setup args for downmodis url = "https://e4ftl01.cr.usgs.gov" tiles = "h11v02,h12v02" modis_download = downmodis.downModis( destinationFolder=wdir, password=pw, user=user, url=url, tiles=tiles, product=product, path=path, ) modis_download.connect() """ NOTE: Using .downloadsAllDay() was failing unpredictably after variable numbers of successful downloads. This iterative approach seems promising after testing. """ h11_dates_present = list_present_dates() h12_dates_present = list_present_dates(tile="2") if len(h11_dates_present) < len(h12_dates_present): dates_present = h11_dates_present else: dates_present = h12_dates_present # Download difference of what files have/have not been downloaded or transferred avail_dates = modis_download.getAllDays() get_dates = [date for date in avail_dates if date not in dates_present] if len(get_dates) == 0: print("No new dates to get.") return None n_diff = len(avail_dates) - len(get_dates) print("Downloading {} data, skipping {} files".format(product[:7], n_diff)) for day in get_dates: files_list = modis_download.getFilesList(day) """ AttributeError occurs after some unpredictable number of downloads. This might just be a system-specific thing, but this exception will retry the download 5 times. """ for attempt in range(5): try: modis_download.dayDownload(day, files_list) except AttributeError: print( "Download failed for {} {}. Failure count: {}.".format( product[:7], day, attempt ) ) if attempt < 4: print("Trying again...") else: break else: sys.exit( "Download failed five times for {} {}, download aborted.".format( product[:7], day, attempt ) ) print("done.")
@author: ZDP-10-10 """ import os from pymodis import downmodis import glob outpath = r"E:\temp\LST" # This directory must already exist BTW # tiles = "h25v05,h26v05" tiles = "h20v03, h20v04, h20v05, h20v06,h21v03, h21v04, h21v05, h21v06,h22v03, h22v04, h22v05, h22v06,h23v03," \ " h23v04, h23v05, h23v06,h24v03, h24v04, h24v05, h24v06,h25v03, h25v04, h25v05, h25v06,h26v03," # That's the MODIS tile covering China except the islands in sourth day = "2014.01.01" enddate = "2000.01.01" # The download works backward, so that enddate is anterior to day= product = "MOD11A2.006" modis_down = downmodis.downModis( destinationFolder=outpath, # 数据存放路径 url="https://e4ftl01.cr.usgs.gov", # 下载地址,这个参数不变 tiles=tiles, # 下载行列号 today=day, # 开始日期 user="******", password="******", enddate=enddate, # 结束日期 product=product, # 产品名字 path="MOLT") # 轨道号 modis_down.connect() modis_down.downloadsAllDay() modis_files = glob.glob(outpath + os.path + '*.hdf') print(modis_files) # ,h21v03, h21v04, h21v05, h21v06,h22v03, h22v04, h22v05, h22v06,h23v03, h23v04, h23v05, h23v06,h24v03, h24v04, h24v05, h24v06,h25v03, h25v04, h25v05, h25v06,h26v03, h26v04, h26v05, h26v06
def main(): """Main function""" # usage usage = "usage: %prog [options] destination_folder" if 1 == len(sys.argv) and WXPYTHON: option_parser_class = optparse_gui.OptionParser else: option_parser_class = optparse_required.OptionParser parser = option_parser_class(usage=usage, description='modis_download') # url parser.add_option("-u", "--url", default="http://e4ftl01.cr.usgs.gov", help="http/ftp server url [default=%default]", dest="url") # password parser.add_option("-P", "--password", dest="password", help="password to connect only if ftp server") # username parser.add_option("-U", "--username", dest="user", default="anonymous", help="username to connect only if ftp server " "[default=%default]") # tiles parser.add_option("-t", "--tiles", dest="tiles", default=None, help="string of tiles separated with comma " "[default=%default for all tiles]") # path to add the path in the server parser.add_option("-s", "--source", dest="path", default="MOLT", help="directory on the http/ftp server " "[default=%default]") # path to add the url parser.add_option("-p", "--product", dest="prod", default="MOD11A1.005", help="product name as on the http/ftp server " "[default=%default]") # delta parser.add_option("-D", "--delta", dest="delta", default=10, help="delta of day from the first day " "[default=%default]") # first day parser.add_option("-f", "--firstday", dest="today", default=None, help="the day to start download [default=%default is for" " today]; if you want change data you must use " "this format YYYY-MM-DD", metavar="FIRST_DAY") # last day parser.add_option("-e", "--endday", dest="enday", default=None, metavar="LAST_DAY", help="the day to stop download " "[default=%default]; if you want change" " data you must use this format YYYY-MM-DD") # debug parser.add_option("-x", action="store_true", dest="debug", default=False, help="this is useful for debugging the " "download [default=%default]") # jpg parser.add_option("-j", action="store_true", dest="jpg", default=False, help="download also the jpeg files [default=%default]") # only one day parser.add_option("-O", dest="oneday", action="store_true", default=False, help="download only one day, it set " "delta=1 [default=%default]") # all days parser.add_option("-A", dest="alldays", action="store_true", default=False, help="download all days, it useful for initial download" " of a product. It overwrite the 'firstday' and " "'endday' options [default=%default]") # remove file with size = 0 parser.add_option("-r", dest="empty", action="store_true", default=False, help="remove empty files (size equal to zero) from " "'destination_folder' [default=%default]") #parser.add_option("-A", dest="alldays", action="store_true", default=True, #help="download all days from the first") # set false several options parser.set_defaults(oneday=False) parser.set_defaults(debug=False) parser.set_defaults(jpg=False) # return options and argument (options, args) = parser.parse_args() # test if args[0] it is set if len(args) == 0 and not WXPYTHON: parser.print_help() sys.exit(1) if len(args) == 0: parser.error("You have to define the destination folder for HDF file") # check if oneday option it is set if options.oneday: options.delta = 1 # set modis object modisOgg = downmodis.downModis(url=options.url, user=options.user, password=options.password, destinationFolder=args[0], tiles=options.tiles, path=options.path, product=options.prod, today=options.today, enddate=options.enday, jpg=options.jpg, delta=int(options.delta), debug=options.debug) # connect to ftp modisOgg.connect() if modisOgg.nconnection <= 20: # download data modisOgg.downloadsAllDay(clean=options.empty, allDays=options.alldays) else: parser.error("A problem with the connection occured")
def main(): try: from rmodislib import product except: grass.fatal("i.modis library is not installed") try: from pymodis.downmodis import downModis except: grass.fatal("pymodis library is not installed") # check if you are in GRASS gisbase = os.getenv('GISBASE') if not gisbase: grass.fatal(_('$GISBASE not defined')) return 0 if flags['l']: prod = product() prod.print_prods() return 0 # set username, password and folder if settings are insert by stdin if not options['settings']: user = None passwd = None if check(options['folder']): fold = options['folder'] else: grass.fatal(_("Set folder parameter when using stdin for passing " "the username and password")) elif options['settings'] == '-': if options['folder'] != '': import getpass if check(options['folder']): fold = options['folder'] user = raw_input(_('Insert username: '******'Insert password: '******'settings']): filesett = open(options['settings'], 'r') fileread = filesett.readlines() user = fileread[0].strip() passwd = fileread[1].strip() filesett.close() else: grass.fatal(_("File <%s> not found") % options['settings']) # set the folder by option folder if options['folder'] != '': if check(options['folder']): fold = options['folder'] # set the folder from path where settings file is stored else: path = os.path.split(options['settings'])[0] temp = os.path.split(grass.tempfile())[0] if temp in path: grass.warning(_("You are downloading data into a temporary " "directory. They will be deleted when you " "close this GRASS GIS session")) if check(path): fold = path # check the version version = grass.core.version() # this is would be set automatically if version['version'].find('7.') == -1: grass.fatal(_('GRASS GIS version 7 required')) return 0 # the product products = options['product'].split(',') # first date and delta firstday, finalday, delta = checkdate(options) # set tiles if options['tiles'] == '': tiles = None grass.warning(_("Option 'tiles' not set. Downloading all available tiles")) else: tiles = options['tiles'] # set the debug if flags['d']: debug_opt = True else: debug_opt = False if flags['c']: checkgdal = False else: checkgdal = True for produ in products: prod = product(produ).returned() #start modis class modisOgg = downModis(url=prod['url'], user=user, password=passwd, destinationFolder=fold, tiles=tiles, delta=delta, path=prod['folder'], product=prod['prod'], today=str(firstday), enddate=str(finalday), debug=debug_opt, checkgdal=checkgdal) # connect to ftp modisOgg.connect() if modisOgg.nconnection <= 20: # download tha tiles grass.message(_("Downloading MODIS product <%s>..." % produ)) modisOgg.downloadsAllDay() filesize = int(os.path.getsize(modisOgg.filelist.name)) if flags['g'] and filesize != 0: grass.message("files=%s" % modisOgg.filelist.name) elif filesize == 0: grass.message(_("No data download, probably they have been " "previously downloaded")) elif filesize != 0: grass.message(_("All data have been downloaded, continue " "with i.modis.import with the option " "'files=%s'" % modisOgg.filelist.name)) else: grass.fatal(_("Error during connection"))
TILE_HEIGHT = TILE_WIDTH wgs84_proj = Proj('EPSG:4326') modis_grid = Proj(f'+proj=sinu +R={EARTH_RADIUS} +nagrids=@null ' f'+ellps=WGS84 +wktext') transformer = Transformer.from_proj(wgs84_proj, modis_grid, always_xy=True) def wgs84_to_modis_tile(xx, yy): x, y = transformer.transform(xx, yy) h = (EARTH_WIDTH * .5 + x) / TILE_WIDTH v = (VERTICAL_TILES * TILE_HEIGHT - y - EARTH_WIDTH * .25) / TILE_HEIGHT return f'h{int(h)}v{int(v):02},' modisOgg = downmodis.downModis( user='******', password='******', destinationFolder='/home/artem/Code/ndvi-test/modis_data', tiles=wgs84_to_modis_tile(127.551270, 35.793457), product='MOD09GA.006', delta=0, today='2021-02-09', enddate='2021-02-09', jpg=True) modisOgg.connect() if modisOgg.nconnection <= 20: modisOgg.downloadsAllDay(clean=True, allDays=False)
def main(): """Main function""" #usage usage = "usage: %prog [options] destination_folder" parser = OptionParser(usage=usage) #password parser.add_option("-P", "--password", dest="password", help="password to connect to ftp server", required=True) #username parser.add_option("-U", "--username", dest="user", default="anonymous", help="username to connect to ftp server [default=%default]") #url parser.add_option("-u", "--url", default="e4ftl01.cr.usgs.gov", help="ftp server url [default=%default]", dest="url") #tiles parser.add_option("-t", "--tiles", dest="tiles", default="None", help="string of tiles separated from comma " \ + "[default=%default for all tiles]") #path to add the url parser.add_option("-s", "--source", dest="path", default="MOLT/MOD11A1.005", help="directory on the ftp [default=%default]") #delta parser.add_option("-D", "--delta", dest="delta", default=10, help="delta of day from the first day " \ + "[default=%default]") #first day parser.add_option("-f", "--firstday", dest="today", default=None, metavar="LAST_DAY", help="the day to start download " \ + "[default=%default is for today]; if you want change" \ " data you must use this format YYYY-MM-DD") #first day parser.add_option("-e", "--endday", dest="enday", default=None, metavar="FIRST_DAY", help="the day to start download " \ + "[default=%default]; if you want change" \ " data you must use this format YYYY-MM-DD") #debug parser.add_option("-x", action="store_true", dest="debug", default=False, help="this is useful for debugging the " \ "download [default=%default]") #jpg parser.add_option("-j", action="store_true", dest="jpg", default=False, help="download also the jpeg files [default=%default]") #only one day parser.add_option("-O", dest="oneday", action="store_true", default=False, help="download only one day, it set " \ "delta=1 [default=%default]") #all days parser.add_option("-A", dest="alldays", action="store_true", default=False, help="download all days, it usefull for first download "\ "of a product. It overwrite the 'firstday' and 'endday'"\ " options [default=%default]") #remove file with size = 0 parser.add_option("-r", dest="empty", action="store_true", default=False, help="remove files with size ugual to zero from " \ "'destination_folder' [default=%default]") #parser.add_option("-A", dest="alldays", action="store_true", default=True, #help="download all days from the first") #set false several options parser.set_defaults(oneday=False) parser.set_defaults(debug=False) parser.set_defaults(jpg=False) #return options and argument (options, args) = parser.parse_args() #test if args[0] it is set if len(args) == 0: parser.error("You have to pass the destination folder for HDF file") #check if oneday option it is set if options.oneday: options.delta = 1 #set modis object modisOgg = downmodis.downModis(url=options.url, user=options.user, password=options.password, destinationFolder=args[0], tiles=options.tiles, path=options.path, today=options.today, enddate=options.enday, delta=int(options.delta), jpg=options.jpg, debug=options.debug) #connect to ftp modisOgg.connectFTP() if modisOgg.nconnection <= 20: #download data modisOgg.downloadsAllDay(clean=options.empty, allDays=options.alldays) else: parser.error("Some problem with connection occur")
#导入函数库 import os from pymodis import downmodis dest = r"D:\f_work\QingHai_book\MOD13Q1" # This directory for data download if not os.path.exists(dest): #Test whether a path exists os.makedirs(dest) # Variables for data download # That's the MODIS tile covering China except the islands in sourth #tiles = "h23v04, h23v05, h24v04, h24v05, h25v03, h25v04, h25v05, \ # h25v06, h26v03, h26v04, h26v05, h26v06, \ # h27v04, h27v05, h27v06, h28v05, h28v06, h28v07, h29v06" tiles = "h25v05, h26v05" product = "MOD13Q1.006" year = 2000, 2005 for yr in range(year[0], year[1] + 1): startdate = str(yr) + ".01.01" enddate = str(yr) + ".12.31" print(str(yr) + ' is downloading !') modis_down = downmodis.downModis(destinationFolder=dest, tiles=tiles, url='https://e4ftl01.cr.usgs.gov', today=startdate, user="******", password="******", enddate=enddate, product=product, path="MOLT") #MOTA modis_down.connect() #Connect to the server and fill the dirData variable modis_down.downloadsAllDay() #Download all requested days
def main(): try: from rmodislib import product except: grass.fatal("i.modis library is not installed") try: from pymodis.downmodis import downModis except: grass.fatal("pymodis library is not installed") # check if you are in GRASS gisbase = os.getenv("GISBASE") if not gisbase: grass.fatal(_("$GISBASE not defined")) return 0 if flags["l"]: prod = product() prod.print_prods() return 0 # empty settings and folder would collide if not options["settings"] and not options["folder"]: grass.fatal( _("With empty settings parameter (to use the .netrc file) " "the folder parameter needs to be specified")) # set username, password and folder if settings are insert by stdin if not options["settings"]: user = None passwd = None if check_folder(options["folder"]): fold = options["folder"] else: grass.fatal( _("Set folder parameter when using stdin for passing " "the username and password")) elif options["settings"] == "-": if options["folder"] != "": import getpass if check_folder(options["folder"]): fold = options["folder"] user = raw_input(_("Insert username: "******"Insert password: "******"Set folder parameter when using stdin for passing " "the username and password")) # set username, password and folder by file else: if not os.path.isfile(options["settings"]): grass.fatal( _("The settings parameter <{}> is not a file").format( options["settings"])) # open the file and read the the user and password: # first line is username # second line is password try: with open(options["settings"], "r") as filesett: fileread = filesett.readlines() user = fileread[0].strip() passwd = fileread[1].strip() except (FileNotFoundError, PermissionError) as e: grass.fatal(_("Unable to read settings: {}").format(e)) if options["folder"] != "": if check_folder(options["folder"]): fold = options["folder"] # set the folder from path where settings file is stored else: path = os.path.split(options["settings"])[0] temp = os.path.split(grass.tempfile())[0] if temp in path: grass.warning( _("You are downloading data into a temporary " "directory. They will be deleted when you " "close this GRASS GIS session")) if check_folder(path): fold = path # the product products = options["product"].split(",") # first date and delta firstday, finalday, delta = checkdate(options) # set tiles if options["tiles"] == "": tiles = None grass.warning( _("Option 'tiles' not set. Downloading all available tiles to <{}>" ).format(fold)) else: tiles = options["tiles"] # set the debug if flags["d"]: debug_opt = True else: debug_opt = False if flags["c"]: checkgdal = False else: checkgdal = True for produ in products: prod = product(produ).returned() # start modis class modisOgg = downModis( url=prod["url"], user=user, password=passwd, destinationFolder=fold, tiles=tiles, delta=delta, path=prod["folder"], product=prod["prod"], today=firstday, enddate=finalday, debug=debug_opt, checkgdal=checkgdal, ) # connect to ftp modisOgg.connect() if modisOgg.nconnection <= 20: # download tha tiles grass.message( _("Downloading MODIS product <{}> ({})...".format( produ, prod["prod"]))) modisOgg.downloadsAllDay() filesize = int(os.path.getsize(modisOgg.filelist.name)) if flags["g"] and filesize != 0: grass.message("files=%s" % modisOgg.filelist.name) elif filesize == 0: grass.message( _("No data download, probably they have been " "previously downloaded")) elif filesize != 0: grass.message( _("All data have been downloaded, continue " "with i.modis.import with the option " "'files=%s'" % modisOgg.filelist.name)) else: grass.fatal(_("Error during connection"))
def main(): """Main function""" # usage usage = "usage: %prog [options] destination_folder" if 1 == len(sys.argv): option_parser_class = optparse_gui.OptionParser else: option_parser_class = optparse_required.OptionParser parser = option_parser_class(usage=usage, description="modis_download_from_list") # file parser.add_option("-f", "--file", dest="file", type="file", help="Input file containing data to download") # url parser.add_option( "-u", "--url", default="http://e4ftl01.cr.usgs.gov", help="http/ftp server url [default=%default]", dest="url" ) # password parser.add_option("-P", "--password", dest="password", help="password to connect only if ftp server") # username parser.add_option( "-U", "--username", dest="user", default="anonymous", help="username to connect only if ftp server " "[default=%default]", ) # path to add the path in the server parser.add_option( "-s", "--source", dest="path", default="MOLT", help="directory on the http/ftp server " "[default=%default]" ) # path to add the url parser.add_option( "-p", "--product", dest="prod", default="MOD11A1.005", help="product name as on the http/ftp server " "[default=%default]", ) # debug parser.add_option( "-x", action="store_true", dest="debug", default=False, help="this is useful for debugging the " "download [default=%default]", ) # jpg parser.add_option( "-j", action="store_true", dest="jpg", default=False, help="download also the jpeg overview files [default=%default]", ) # return options and argument (options, args) = parser.parse_args() if len(args) > 1: parser.error("You have to define the destination folder for HDF file") f = open(options.file) lines = [elem for elem in f.readlines()] tiles = [elem.split(".")[2] for elem in lines] tiles = ",".join(sorted(set(tiles))) dates = [elem.split(".")[1].replace("A", "") for elem in lines] dates = sorted(set(dates)) for d in dates: year = int(d[0:4]) doy = int(d[4:7]) fdate = date.fromordinal(date(year, 1, 1).toordinal() + doy - 1).isoformat() modisOgg = downmodis.downModis( url=options.url, user=options.user, password=options.password, destinationFolder=args[0], tiles=tiles, path=options.path, product=options.prod, delta=1, today=fdate, debug=options.debug, jpg=options.jpg, ) modisOgg.connect() day = modisOgg.getListDays()[0] if modisOgg.urltype == "http": listAllFiles = modisOgg.getFilesList(day) else: listAllFiles = modisOgg.getFilesList() listFilesDown = modisOgg.checkDataExist(listAllFiles) modisOgg.dayDownload(day, listFilesDown) if modisOgg.urltype == "http": modisOgg.closeFilelist() else: modisOgg.closeFTP()
def download_last_modis(): today = date.today() start = (today - timedelta(days=1)).strftime("%Y-%m-%d") end = (today - timedelta(days=5)).strftime("%Y-%m-%d") # tiles to download-Greece region tiles = 'h19v05,h20v05,h19v04' # starting day day = start # finishing day enddate = end # username user = '******' # password password = '******' # destination folder dest = '/home/sg/test_modis_download/last' # log delta = 1 product = 'MYD13A1.006' # ,MOD13A1.006' path = 'MOLA' # ,MOLT' # product2 = 'MOD13A1.006' # path2 = 'MOLT' modisDown = downmodis.downModis(destinationFolder=dest, tiles=tiles, today=day, enddate=enddate, delta=delta, user=user, password=password, product=product, path=path, jpg=False, debug=False, timeout=30, checkgdal=True) modisDown.connect() #files_list = modisDown.getFilesList #new_files = modisDown.getAllDays() modisDown.downloadsAllDay() modisDown.removeEmptyFiles() my_path = '/home/sg/test_modis_download/last' with open("/home/sg/test_modis_download/archive/most_recent.json") as f: last_images = json.load(f) i = 0 for image in os.listdir(my_path): if image.endswith('hdf'): i = 1 check = modisDown.checkFile(image) if check == 1: os.rename("/home/sg/test_modis_download/last/" + image, "/home/sg/test_modis_download/archive/" + image) os.rename("/home/sg/test_modis_download/last/" + image + '.xml', "/home/sg/test_modis_download/archive/" + image + '.xml') tile = image[17:23] last_images[tile] = image else: print('Image %s corrupted. Keeping previous image' % image) if i == 0: print('MOLA: No new images found') product = 'MOD13A1.006' path = 'MOLT' modisDown = downmodis.downModis(destinationFolder=dest, tiles=tiles, today=day, enddate=enddate, delta=delta, user=user, password=password, product=product, path=path, jpg=False, debug=False, timeout=30, checkgdal=True) modisDown.connect() modisDown.downloadsAllDay() modisDown.removeEmptyFiles() for image in os.listdir(my_path): if image.endswith('hdf'): i = 1 check = modisDown.checkFile(image) if check == 1: os.rename("/home/sg/test_modis_download/last/" + image, "/home/sg/test_modis_download/archive/" + image) os.rename("/home/sg/test_modis_download/last/" + image + '.xml', "/home/sg/test_modis_download/archive/" + image + '.xml') tile = image[17:23] last_images[tile] = image else: print('Image %s corrupted. Keeping previous image' % image) if i == 0: print('MOLT: No new images found') json_images = json.dumps(last_images) f = open("/home/sg/test_modis_download/archive/most_recent.json", "w") f.write(json_images) f.close()
import os import pymodis from pymodis import downmodis from pymodis import parsemodis import glob dest = "/Users/iza/Desktop/modis_data" tiles = "h18v04,h19v04" day = "2014-08-14" delta = 1 modisDown = downmodis.downModis(destinationFolder=dest, tiles=tiles, today=day, delta=delta) modisDown.connect() modisDown.downloadsAllDay() files = glob.glob(os.path.join(dest, 'MOD11A1.A2014*.hdf')) print(files)
from pymodis import downmodis import glob import numpy as np import matplotlib.pyplot as plt # Variables for data download dest = "/home/dutri001/sandbox/MODIS/" tiles = "h18v03" day = "2015.11.18" enddate = "2015.11.16" product = "MOD13A2.006" # Init download class, connect and download modis_down = downmodis.downModis(destinationFolder=dest, tiles=tiles, today=day, enddate=enddate, product=product) modis_down.connect() modis_down.downloadsAllDay() # List files with the hdf extension MODIS_file = glob.glob(dest + '*.hdf') # Get subdatasets of the first file of the MODIS_file list sds = gdal.Open(MODIS_file[0], gdal.GA_ReadOnly).GetSubDatasets() print sds[0][0] print sds[11][0] # Open vi, to get the metadata (geoTransform, dimensions, projection) vi = gdal.Open(sds[0][0]) ind = [1, 11] # vi_np = np.zeros((vi.RasterYSize, vi.RasterXSize, 2), np.int16)
f.close() # Armazena as informações de usuario e senha usuario = info[0][:-1] senha = info[1][:-1] # Faz o download for year in range(ano_inicio, ano_final + 1): for month in range(mes_inicio, mes_final + 1): for i in range(0, 2): dia_final = "%04d.%02d.%02d" % ( year, month, calendar.monthrange(year, month)[1]) dia_inicial = "%04d.%02d.01" % (year, month) # Cria o diretório para armazenar as imagens destino = os.path.join( BASE_DIR, "data/lst/%s/%04d/%02d" % (satelite[i], year, month)) if not os.path.isdir(destino): os.makedirs(destino) # Cria uma classe de download do MODIS, conecta e faz o download modis_down = downmodis.downModis(password=senha, user=usuario, destinationFolder=destino, path=pasta[i], tiles=tiles, today=dia_final, enddate=dia_inicial, product=produto[i]) modis_down.connect() modis_down.downloadsAllDay()
def main(): """Main function""" # usage usage = "usage: %prog [options] destination_folder" if 1 == len(sys.argv) and WXPYTHON: option_parser_class = optparse_gui.OptionParser else: option_parser_class = optparse_required.OptionParser parser = option_parser_class(usage=usage, description='modis_download_from_list') # file parser.add_option("-f", "--file", dest="file", type='file', help="Input file containing data to download") # url parser.add_option("-u", "--url", default="http://e4ftl01.cr.usgs.gov", help="http/ftp server url [default=%default]", dest="url") # username and password from stdin parser.add_option("-I", "--input", dest="input", action="store_true", help="insert user and password from standard input") # password parser.add_option("-P", "--password", dest="password", help="password to connect only if ftp server") # username parser.add_option("-U", "--username", dest="user", default="anonymous", help="username to connect only if ftp server " "[default=%default]") # path to add the path in the server parser.add_option("-s", "--source", dest="path", default="MOLT", help="directory on the http/ftp server " "[default=%default]") # path to add the url parser.add_option("-p", "--product", dest="prod", default="MOD11A1.005", help="product name as on the http/ftp server " "[default=%default]") # debug parser.add_option("-x", action="store_true", dest="debug", default=False, help="this is useful for debugging the " "download [default=%default]") # jpg parser.add_option("-j", action="store_true", dest="jpg", default=False, help="download also the jpeg overview files " "[default=%default]") # return options and argument (options, args) = parser.parse_args() if len(args) == 0 and not WXPYTHON: parser.print_help() sys.exit(1) if len(args) > 1: parser.error("You have to define the destination folder for HDF file") if not os.path.isdir(args[0]): parser.error("The destination folder is not a dir or not exists") if options.input: if sys.version_info.major == 3: user = input("Username: "******"Username: "******"You have to set user and password") f = open(options.file) lines = [elem for elem in f.readlines()] tiles = [elem.strip().split('.')[2] for elem in lines if elem != '\n'] tiles = ','.join(sorted(set(tiles))) dates = [elem.split('.')[1].replace('A', '') for elem in lines if elem != '\n'] dates = sorted(set(dates)) for d in dates: year = int(d[0:4]) doy = int(d[4:7]) fdate = date.fromordinal(date(year, 1, 1).toordinal() + doy - 1).isoformat() modisOgg = downmodis.downModis(url=options.url, user=options.user, password=options.password, destinationFolder=args[0], tiles=tiles, path=options.path, product=options.prod, delta=1, today=fdate, debug=options.debug, jpg=options.jpg) modisOgg.connect() day = modisOgg.getListDays()[0] if modisOgg.urltype == 'http': listAllFiles = modisOgg.getFilesList(day) else: listAllFiles = modisOgg.getFilesList() listFilesDown = modisOgg.checkDataExist(listAllFiles) modisOgg.dayDownload(day, listFilesDown) if modisOgg.urltype == 'http': modisOgg.closeFilelist() else: modisOgg.closeFTP()
def main(): """Main function""" # usage usage = "usage: %prog [options] destination_folder" if 1 == len(sys.argv) and WXPYTHON: option_parser_class = optparse_gui.OptionParser else: option_parser_class = optparse_required.OptionParser parser = option_parser_class(usage=usage, description='modis_download_from_list') # file parser.add_option("-f", "--file", dest="file", type='file', help="Input file containing data to download") # url parser.add_option("-u", "--url", default="http://e4ftl01.cr.usgs.gov", help="http/ftp server url [default=%default]", dest="url") # password parser.add_option("-P", "--password", dest="password", help="password to connect only if ftp server") # username parser.add_option("-U", "--username", dest="user", default="anonymous", help="username to connect only if ftp server " "[default=%default]") # path to add the path in the server parser.add_option("-s", "--source", dest="path", default="MOLT", help="directory on the http/ftp server " "[default=%default]") # path to add the url parser.add_option("-p", "--product", dest="prod", default="MOD11A1.005", help="product name as on the http/ftp server " "[default=%default]") # debug parser.add_option("-x", action="store_true", dest="debug", default=False, help="this is useful for debugging the " "download [default=%default]") # jpg parser.add_option("-j", action="store_true", dest="jpg", default=False, help="download also the jpeg overview files " "[default=%default]") # return options and argument (options, args) = parser.parse_args() if len(args) == 0 and not WXPYTHON: parser.print_help() sys.exit(1) if len(args) > 1: parser.error("You have to define the destination folder for HDF file") f = open(options.file) lines = [elem for elem in f.readlines()] tiles = [elem.strip().split('.')[2] for elem in lines if elem != '\n'] tiles = ','.join(sorted(set(tiles))) dates = [ elem.split('.')[1].replace('A', '') for elem in lines if elem != '\n' ] dates = sorted(set(dates)) for d in dates: year = int(d[0:4]) doy = int(d[4:7]) fdate = date.fromordinal(date(year, 1, 1).toordinal() + doy - 1).isoformat() modisOgg = downmodis.downModis(url=options.url, user=options.user, password=options.password, destinationFolder=args[0], tiles=tiles, path=options.path, product=options.prod, delta=1, today=fdate, debug=options.debug, jpg=options.jpg) modisOgg.connect() day = modisOgg.getListDays()[0] if modisOgg.urltype == 'http': listAllFiles = modisOgg.getFilesList(day) else: listAllFiles = modisOgg.getFilesList() listFilesDown = modisOgg.checkDataExist(listAllFiles) modisOgg.dayDownload(day, listFilesDown) if modisOgg.urltype == 'http': modisOgg.closeFilelist() else: modisOgg.closeFTP()
def main(): """Main function""" # usage usage = "usage: %prog [options] destination_folder" if 1 == len(sys.argv) and WXPYTHON: option_parser_class = optparse_gui.OptionParser else: option_parser_class = optparse_required.OptionParser parser = option_parser_class(usage=usage, description='modis_download') # url parser.add_option("-u", "--url", default="https://e4ftl01.cr.usgs.gov", help="http/ftp server url [default=%default]", dest="url") # username and password from stdin parser.add_option("-I", "--input", dest="input", action="store_true", help="insert user and password from standard input") # password parser.add_option("-P", "--password", dest="password", help="password to connect to the server") # username parser.add_option("-U", "--username", dest="user", help="username to connect to the server") # tiles parser.add_option("-t", "--tiles", dest="tiles", default=None, help="string of tiles separated with comma " "[default=%default for all tiles]") # path to add the path in the server parser.add_option("-s", "--source", dest="path", default="MOLT", help="directory on the http/ftp server " "[default=%default]") # path to add the url parser.add_option("-p", "--product", dest="prod", default="MOD11A1.005", help="product name as on the http/ftp server " "[default=%default]") # delta parser.add_option("-D", "--delta", dest="delta", default=10, help="delta of day from the first day " "[default=%default]") # first day parser.add_option("-f", "--firstday", dest="today", default=None, help="the day to start download [default=%default is for" " today]; if you want change data you must use " "this format YYYY-MM-DD", metavar="FIRST_DAY") # last day parser.add_option("-e", "--endday", dest="enday", default=None, metavar="LAST_DAY", help="the day to stop download " "[default=%default]; if you want change" " data you must use this format YYYY-MM-DD") # debug parser.add_option("-x", action="store_true", dest="debug", default=False, help="this is useful for debugging the " "download [default=%default]") # jpg parser.add_option("-j", action="store_true", dest="jpg", default=False, help="download also the jpeg files [default=%default]") # only one day parser.add_option("-O", dest="oneday", action="store_true", default=False, help="download only one day, it set " "delta=1 [default=%default]") # all days parser.add_option("-A", dest="alldays", action="store_true", default=False, help="download all days, it useful for initial download" " of a product. It overwrite the 'firstday' and " "'endday' options [default=%default]") # remove file with size = 0 parser.add_option("-r", dest="empty", action="store_true", default=False, help="remove empty files (size equal to zero) from " "'destination_folder' [default=%default]") #parser.add_option("-A", dest="alldays", action="store_true", default=True, #help="download all days from the first") # set false several options parser.set_defaults(oneday=False) parser.set_defaults(debug=False) parser.set_defaults(jpg=False) # return options and argument (options, args) = parser.parse_args() # test if args[0] it is set if len(args) == 0 and not WXPYTHON: parser.print_help() sys.exit(1) if len(args) == 0: parser.error("You have to define the destination folder for HDF file") if not os.path.isdir(args[0]): parser.error("The destination folder is not a dir or not exists") # check if oneday option it is set if options.oneday: options.delta = 1 if options.input: if sys.version_info.major == 3: user = input("Username: "******"Username: "******"A problem with the connection occured")
def main(): """Main function""" # usage usage = "usage: %prog [options] destination_folder" if 1 == len(sys.argv) and WXPYTHON: option_parser_class = optparse_gui.OptionParser else: option_parser_class = optparse_required.OptionParser parser = option_parser_class(usage=usage, description='modis_download_from_list') # file parser.add_option("-f", "--file", dest="file", type='file', help="Input file containing data to download") # url parser.add_option("-u", "--url", default="https://e4ftl01.cr.usgs.gov", help="http/ftp server url [default=%default]", dest="url") # username and password from stdin parser.add_option("-I", "--input", dest="input", action="store_true", help="insert user and password from standard input") # password parser.add_option("-P", "--password", dest="password", help="password to connect to the server") # username parser.add_option("-U", "--username", dest="user", help="username to connect to the server ") # path to add the path in the server parser.add_option("-s", "--source", dest="path", default="MOLT", help="directory on the http/ftp server " "[default=%default]") # path to add the url parser.add_option("-p", "--product", dest="prod", default="MOD11A1.006", help="product name as on the http/ftp server " "[default=%default]") # path to file with server missing tiles parser.add_option("-o", "--outputs", dest="outs", default=None, help="the output where write the missing files in the" " server [default=%default]. Use 'stdout' to write to " " STDOUT") # use netrc file parser.add_option("-n", action="store_true", dest="netrc", default=False, help="use netrc file to read user and password") # debug parser.add_option("-x", action="store_true", dest="debug", default=False, help="this is useful for debugging the " "download [default=%default]") # jpg parser.add_option("-j", action="store_true", dest="jpg", default=False, help="download also the jpeg overview files " "[default=%default]") # return options and argument (options, args) = parser.parse_args() if len(args) == 0 and not WXPYTHON: parser.print_help() sys.exit(1) if len(args) > 1: parser.error("You have to define the destination folder for HDF file") if not os.path.isdir(args[0]): parser.error("The destination folder is not a dir or not exists") if options.netrc: user = None password = None elif options.input: if sys.version_info.major == 3: user = input("Username: "******"Username: "******"stdout": write = sys.stdout else: write = open(options.outs, 'w') for d, tiles in sorted(vals.items(), reverse=True): year = int(d[0:4]) doy = int(d[4:7]) fdate = date.fromordinal(date(year, 1, 1).toordinal() + doy - 1).isoformat() modisOgg = downmodis.downModis(url=options.url, user=user, password=password, destinationFolder=args[0], tiles=','.join(sorted(set(tiles))), path=options.path, product=options.prod, delta=1, today=fdate, debug=options.debug, jpg=options.jpg) modisOgg.connect() day = modisOgg.getListDays()[0] if modisOgg.urltype == 'http': listAllFiles = modisOgg.getFilesList(day) else: listAllFiles = modisOgg.getFilesList() if not listAllFiles: if write: write_out(write, tiles, options, d) continue listFilesDown = modisOgg.checkDataExist(listAllFiles) if listFilesDown: if options.debug: print(listFilesDown) modisOgg.dayDownload(day, listFilesDown) for fi in listFilesDown: if 'xml' not in fi: try: tiles.remove(fi.split('.')[2]) except ValueError: pass if write: write_out(write, tiles, options, d) if modisOgg.urltype == 'http': modisOgg.closeFilelist() else: modisOgg.closeFTP()
def download_data(self, output_dir, product, dates=None, tiles=None, mosaic_dir=None, overwrite=False): logger.info('entering download_data') files = None _m_files = [] _dl_files = [] _delta = 1 _user = self.vp.get('MODIS', 'user') _password = self.vp.get('MODIS', 'password') _dates = [] # if dates is None: # need to get list of dates first if not os.path.exists(output_dir): logger.debug("%s does not exist. Creating directory", output_dir) os.makedirs(output_dir) _modis_connect = downmodis.downModis(destinationFolder=output_dir, user=_user, password=_password, product=product, today=None, delta=_delta, tiles=tiles) _modis_connect.connect() _days_list = _modis_connect.getAllDays() for i in _days_list: if dates is None: _dates.append( i) #('{0}-{1}'.format(i.split('.')[0], i.split('.')[1])) else: # check if in month/year if '{0}-{1}'.format(i.split('.')[0], i.split('.')[1]) in dates: _dates.append(i) # else: # _dates = dates for d in _dates: logger.debug("downloading data for %s", d) _month = d #'{date}-01'.format(date=d) _folder_date = datetime.datetime.strptime( _month, '%Y.%m.%d').strftime('%Y.%m.%d') _new_folder = output_dir #os.path.join(output_dir, _folder_date) # don't need anymore? done above instead? # create folder if it doesn't already exist if not os.path.exists(_new_folder): logger.debug("%s does not exist. Creating directory", _new_folder) os.makedirs(_new_folder) _modis_download = downmodis.downModis( destinationFolder=_new_folder, tiles=tiles, product=product, today=_month, delta=_delta, user=_user, password=_password) _modis_download.connect() _check_files = _modis_download.getFilesList(_folder_date) # logger.debug("check files: %s", _check_files) _dl_files = _modis_download.checkDataExist(_check_files) try: _modis_download.downloadsAllDay(clean=True) except Exception, e: logger.debug( 'Error in pymodis.modisDown.downloadsAllDay downloading %s', _dl_files) raise ValueError('Error in pymodis.modisDown.downloadsAllDay') _modis_download.removeEmptyFiles() _modis_download.closeFilelist() # want list of files downloaded (or files for this day) _files_list = [] for f in _check_files: if os.path.splitext(os.path.join(_new_folder, f))[1] == '.hdf': _files_list.append(os.path.join(_new_folder, f)) _files = glob.glob( os.path.join(_new_folder, '{product}*.hdf'.format(product=product[:-4]))) # logger.debug("files: %s", _files) logger.debug("downloaded files: %s", _files_list) if mosaic_dir: # mosaic files mosaic_file = self.mosaic_tiles(_files_list, mosaic_dir, product) _m_files.append(mosaic_file)