def main(args):
# vendo lista de tiles
if args.tile == None and args.uf == None:
print 'É preciso definir uf ou tile'
return
if args.tile:
tiles = str(args.tile)
else:
UF = eval('estados_v2.' + args.uf)
tiles = UF.tiles
if args.data_final != None:
# usuário esta trocando a data de download
modisDown = downmodis.downModis(destinationFolder=args.dest_dir,
tiles=tiles,
product=args.produto,
today=args.data_final,
delta=args.delta)
else:
modisDown = downmodis.downModis(destinationFolder=args.dest_dir,
tiles=tiles,
product=args.produto,
delta=args.delta)
modisDown.connect()
modisDown.downloadsAllDay()
return
def download(self):
modisObj = downmodis.downModis(url="https://e4ftl01.cr.usgs.gov",
user="******",
password='******',
path="MOLT",
timeout=3600,
destinationFolder=self.targetPath,
jpg=False,
debug=True,
tiles=self.tiles,
today=self.end,
enddate=self.start,
product=self.productName)
continueLoop = True
while (continueLoop):
try:
print('download')
modisObj.connect()
modisObj.downloadsAllDay(clean=True, allDays=False)
continueLoop = False
except:
traceback.print_exc()
time.sleep(10)
print('try again')
pass
def getMODISDataFromURL(output_dir, product, tiles, dates, mosaic_dir,
tools_dir = "", logger = None):
files = None
m_files = []
if tiles == '':
tiles=None
for d in dates:
if logger: logger.debug("downloading %s", d)
_month = d + '-01'
_folder_date = datetime.datetime.strptime(_month, '%Y-%m-%d').strftime('%Y.%m.%d')
_new_folder = os.path.join(output_dir, _folder_date)
# create folder if it doesn't already exist
if not os.path.exists(_new_folder):
os.makedirs(_new_folder)
_delta = 1
modisDown = downmodis.downModis(destinationFolder=_new_folder, tiles=tiles, product=product, today=_month, delta=_delta)
modisDown.connect()
_check_files = modisDown.getFilesList(_folder_date)
if logger: logger.debug("check files: %s", _check_files)
dl_files = modisDown.checkDataExist(_check_files)
try:
modisDown.downloadsAllDay(clean=True)
except Exception, e:
if logger: logger.error("Error in pymodis.modisDown.downloadsAllDay")
modisDown.removeEmptyFiles()
modisDown.closeFilelist()
_files = glob.glob(os.path.join(_new_folder, '{product}*.hdf'.format(product=product[:-4])))
if logger: logger.debug("files: %s", files)
if logger: logger.debug("dl_files: %s", dl_files)
if mosaic_dir:
# mosaic files
mosaic_file = mosaicTiles(_files, mosaic_dir, tools_dir)
m_files.append(mosaic_file)
def getMODISDataFromURL(output_dir, product, tiles, dates, mosaic_dir,
tools_dir = "", logger = None):
files = None
m_files = []
if tiles == '':
tiles=None
# # new setup to handle authentication
# username = '******'
# password = '******'
# password_manager = urllib2.HTTPPasswordMgrWithDefaultRealm()
# password_manager.add_password(None, "https://urs.earthdata.nasa.gov", username, password)
#
# cookie_jar = CookieJar()
#
# # Install all the handlers.
#
# opener = urllib2.build_opener(
# urllib2.HTTPBasicAuthHandler(password_manager),
# # urllib2.HTTPHandler(debuglevel=1), # Uncomment these two lines to see
# # urllib2.HTTPSHandler(debuglevel=1), # details of the requests/responses
# urllib2.HTTPCookieProcessor(cookie_jar))
# urllib2.install_opener(opener)
# end of new setup to handle authentication
for d in dates:
if logger: logger.debug("downloading %s", d)
_month = d + '-01'
_folder_date = datetime.datetime.strptime(_month, '%Y-%m-%d').strftime('%Y.%m.%d')
_new_folder = os.path.join(output_dir, _folder_date)
# create folder if it doesn't already exist
if not os.path.exists(_new_folder):
os.makedirs(_new_folder)
_delta = 1
modisDown = downmodis.downModis(destinationFolder=_new_folder, tiles=tiles,
product=product, today=_month, delta=_delta, user='******', password='******')
modisDown.connect()
_check_files = modisDown.getFilesList(_folder_date)
if logger: logger.debug("check files: %s", _check_files)
dl_files = modisDown.checkDataExist(_check_files)
try:
modisDown.downloadsAllDay(clean=True)
except Exception, e:
if logger: logger.error("Error in pymodis.modisDown.downloadsAllDay")
modisDown.removeEmptyFiles()
modisDown.closeFilelist()
_files = glob.glob(os.path.join(_new_folder, '{product}*.hdf'.format(product=product[:-4])))
if logger: logger.debug("files: %s", _files)
if logger: logger.debug("dl_files: %s", dl_files)
if mosaic_dir:
# mosaic files
mosaic_file = mosaicTiles(_files, mosaic_dir, tools_dir)
m_files.append(mosaic_file)
def download():
# 此时，我们可以初始化downmodis对象。我们将在“/tmp”目录下下载MODIS LST产品
modisDown = downmodis.downModis(destinationFolder=dest,
password="******",
user="******",
tiles=tiles,
today=day,
delta=delta)
modisDown.connect() #此时，我们开始下载文件。
modisDown.downloadsAllDay()
files = glob.glob(os.path.join(dest, 'MOD11A1.A2014*.hdf'))
print(files)
def performDownload(tilelist, dates, i, work_path):
'''
Downloads MODIS tiles from USGS ftp server and save them to the appropriate
date folder
'''
print ""
print "Working on {0} to {1}".format(dates[i], dates[i + 1])
print "Files are saved in {}".format(work_path)
destination = work_path
# make downloader
dm = pmod.downModis(destinationFolder=destination,
path="MOLT", product="MOD13A3.006", tiles="{}".format(tilelist),
today="{}".format(dates[i]), enddate="{}".format(dates[i + 1]))
dm.connect()
print "Connection Attempts: " + str(dm.nconnection)
# Download all the files in that day
print "Please Wait....................."
dm.downloadsAllDay()
def downloadFiles(day):
# Variables for data download
dest = "data/" # This directory must already exist BTW
# enddate = "2018-10-11" # The download works backward, so that enddate is anterior to day=
product = "MOD09GA.006"
# Instantiate download class, connect and download
modis_down = downmodis.downModis(destinationFolder=dest,
tiles=TILES,
user="******",
password="******",
today=day,
delta=1,
product=product)
modis_down.connect()
modis_down.downloadsAllDay()
# Check that the data has been downloaded
MODIS_files = glob.glob(dest + '*.hdf')
def DownLoadData(LastKnownDate, CurrentDate):
# Set the destination folder
dest = current_dir + '/RawData'
# These are four tiles that cover Kenya
tiles = 'h21v08,h21v09,h22v08,h22v09'
# Using the Pymodis module to download the 4 tiles from the server
Modis_Down = downModis(destinationFolder=dest,
tiles=tiles,
url='https://e4ftl01.cr.usgs.gov',
path='MOTA',
product='MCD43A4.006',
user='******',
password='******',
today=CurrentDate,
enddate=LastKnownDate)
Modis_Down.connect()
Modis_Down.downloadsAllDay()
def download(self):
modisObj = downmodis.downModis(
url="http://e4ftl01.cr.usgs.gov",
user="******", password=None, path="MOLT",
timeout=3600, destinationFolder=self.targetPath, jpg=False,
debug=True, tiles=self.tiles,
today=self.end, enddate=self.start,
product=self.productName)
continueLoop = True
while(continueLoop):
try:
print('download')
modisObj.connect()
modisObj.downloadsAllDay(clean=True, allDays=False)
continueLoop = False
except:
traceback.print_exc()
time.sleep(10)
print('try again')
pass
def download_modis(user, pw, prod, wdir):
"""
wrapper for downmodis functions.
Non-changing vars (e.g., user/pw) left in global scope.
"""
def list_present_dates(wdir=wdir, tile="1"):
"""
Compile list of dates already present in MODIS_DIR for particular source/tile.
Done to avoid redundant downloads, as this script may have to be applied
more than once to download all files.
Takes to minimum of what is available for both tiles
"""
dates_present = []
modis_files = list(wdir.glob("*h1{}v02*.hdf".format(tile)))
# keep = [os.stat(file).st_size > 0 for file in modis_files]
# files_present = list(itertools.compress(modis_files, keep))
for file in modis_files:
"""
files of size 0 are assumed to be unsuccessful downloads,
need to be removed for .dayDownload() to re-download
"""
if os.stat(file).st_size == 0:
print("Removing empty file: {}".format(str(file).split("/")[-1]))
os.remove(file)
continue
jday = int(str(file).split(".")[1][5:])
year = int(str(file).split(".")[1][1:5])
date = datetime.datetime(year, 1, 1) + datetime.timedelta(jday - 1)
dates_present.append(date.strftime("%Y.%m.%d"))
return dates_present
# choose aqua or terra
product = "M{}D11A2.006".format(prod)
prod_info = {
"MOD11A2.006": ["MOLT", "2000-02-24"],
"MYD11A2.006": ["MOLA", "2002-07-24"],
}
path = prod_info[product][0]
start_date = prod_info[product][1]
# setup args for downmodis
url = "https://e4ftl01.cr.usgs.gov"
tiles = "h11v02,h12v02"
modis_download = downmodis.downModis(
destinationFolder=wdir,
password=pw,
user=user,
url=url,
tiles=tiles,
product=product,
path=path,
)
modis_download.connect()
"""
NOTE:
Using .downloadsAllDay() was failing unpredictably after variable numbers of
successful downloads. This iterative approach seems promising after testing.
"""
h11_dates_present = list_present_dates()
h12_dates_present = list_present_dates(tile="2")
if len(h11_dates_present) < len(h12_dates_present):
dates_present = h11_dates_present
else:
dates_present = h12_dates_present
# Download difference of what files have/have not been downloaded or transferred
avail_dates = modis_download.getAllDays()
get_dates = [date for date in avail_dates if date not in dates_present]
if len(get_dates) == 0:
print("No new dates to get.")
return None
n_diff = len(avail_dates) - len(get_dates)
print("Downloading {} data, skipping {} files".format(product[:7], n_diff))
for day in get_dates:
files_list = modis_download.getFilesList(day)
"""
AttributeError occurs after some unpredictable number of downloads.
This might just be a system-specific thing, but this exception will
retry the download 5 times.
"""
for attempt in range(5):
try:
modis_download.dayDownload(day, files_list)
except AttributeError:
print(
"Download failed for {} {}. Failure count: {}.".format(
product[:7], day, attempt
)
)
if attempt < 4:
print("Trying again...")
else:
break
else:
sys.exit(
"Download failed five times for {} {}, download aborted.".format(
product[:7], day, attempt
)
)
print("done.")
@author: ZDP-10-10
"""
import os
from pymodis import downmodis
import glob
outpath = r"E:\temp\LST" # This directory must already exist BTW
# tiles = "h25v05,h26v05"
tiles = "h20v03, h20v04, h20v05, h20v06，h21v03, h21v04, h21v05, h21v06,h22v03, h22v04, h22v05, h22v06，h23v03," \
" h23v04, h23v05, h23v06，h24v03, h24v04, h24v05, h24v06，h25v03, h25v04, h25v05, h25v06，h26v03," # That's the MODIS tile covering China except the islands in sourth
day = "2014.01.01"
enddate = "2000.01.01" # The download works backward, so that enddate is anterior to day=
product = "MOD11A2.006"
modis_down = downmodis.downModis(
destinationFolder=outpath, # 数据存放路径
url="https://e4ftl01.cr.usgs.gov", # 下载地址，这个参数不变
tiles=tiles, # 下载行列号
today=day, # 开始日期
user="******",
password="******",
enddate=enddate, # 结束日期
product=product, # 产品名字
path="MOLT") # 轨道号
modis_down.connect()
modis_down.downloadsAllDay()
modis_files = glob.glob(outpath + os.path + '*.hdf')
print(modis_files)
# ,h21v03, h21v04, h21v05, h21v06，h22v03, h22v04, h22v05, h22v06，h23v03, h23v04, h23v05, h23v06，h24v03, h24v04, h24v05, h24v06，h25v03, h25v04, h25v05, h25v06，h26v03, h26v04, h26v05, h26v06
def main():
"""Main function"""
# usage
usage = "usage: %prog [options] destination_folder"
if 1 == len(sys.argv) and WXPYTHON:
option_parser_class = optparse_gui.OptionParser
else:
option_parser_class = optparse_required.OptionParser
parser = option_parser_class(usage=usage, description='modis_download')
# url
parser.add_option("-u", "--url", default="http://e4ftl01.cr.usgs.gov",
help="http/ftp server url [default=%default]",
dest="url")
# password
parser.add_option("-P", "--password", dest="password",
help="password to connect only if ftp server")
# username
parser.add_option("-U", "--username", dest="user", default="anonymous",
help="username to connect only if ftp server "
"[default=%default]")
# tiles
parser.add_option("-t", "--tiles", dest="tiles", default=None,
help="string of tiles separated with comma "
"[default=%default for all tiles]")
# path to add the path in the server
parser.add_option("-s", "--source", dest="path", default="MOLT",
help="directory on the http/ftp server "
"[default=%default]")
# path to add the url
parser.add_option("-p", "--product", dest="prod", default="MOD11A1.005",
help="product name as on the http/ftp server "
"[default=%default]")
# delta
parser.add_option("-D", "--delta", dest="delta", default=10,
help="delta of day from the first day "
"[default=%default]")
# first day
parser.add_option("-f", "--firstday", dest="today", default=None,
help="the day to start download [default=%default is for"
" today]; if you want change data you must use "
"this format YYYY-MM-DD", metavar="FIRST_DAY")
# last day
parser.add_option("-e", "--endday", dest="enday", default=None,
metavar="LAST_DAY", help="the day to stop download "
"[default=%default]; if you want change"
" data you must use this format YYYY-MM-DD")
# debug
parser.add_option("-x", action="store_true", dest="debug", default=False,
help="this is useful for debugging the "
"download [default=%default]")
# jpg
parser.add_option("-j", action="store_true", dest="jpg", default=False,
help="download also the jpeg files [default=%default]")
# only one day
parser.add_option("-O", dest="oneday", action="store_true", default=False,
help="download only one day, it set "
"delta=1 [default=%default]")
# all days
parser.add_option("-A", dest="alldays", action="store_true", default=False,
help="download all days, it useful for initial download"
" of a product. It overwrite the 'firstday' and "
"'endday' options [default=%default]")
# remove file with size = 0
parser.add_option("-r", dest="empty", action="store_true", default=False,
help="remove empty files (size equal to zero) from "
"'destination_folder' [default=%default]")
#parser.add_option("-A", dest="alldays", action="store_true", default=True,
#help="download all days from the first")
# set false several options
parser.set_defaults(oneday=False)
parser.set_defaults(debug=False)
parser.set_defaults(jpg=False)
# return options and argument
(options, args) = parser.parse_args()
# test if args[0] it is set
if len(args) == 0 and not WXPYTHON:
parser.print_help()
sys.exit(1)
if len(args) == 0:
parser.error("You have to define the destination folder for HDF file")
# check if oneday option it is set
if options.oneday:
options.delta = 1
# set modis object
modisOgg = downmodis.downModis(url=options.url, user=options.user,
password=options.password,
destinationFolder=args[0],
tiles=options.tiles, path=options.path,
product=options.prod, today=options.today,
enddate=options.enday, jpg=options.jpg,
delta=int(options.delta),
debug=options.debug)
# connect to ftp
modisOgg.connect()
if modisOgg.nconnection <= 20:
# download data
modisOgg.downloadsAllDay(clean=options.empty, allDays=options.alldays)
else:
parser.error("A problem with the connection occured")
def main():
try:
from rmodislib import product
except:
grass.fatal("i.modis library is not installed")
try:
from pymodis.downmodis import downModis
except:
grass.fatal("pymodis library is not installed")
# check if you are in GRASS
gisbase = os.getenv('GISBASE')
if not gisbase:
grass.fatal(_('$GISBASE not defined'))
return 0
if flags['l']:
prod = product()
prod.print_prods()
return 0
# set username, password and folder if settings are insert by stdin
if not options['settings']:
user = None
passwd = None
if check(options['folder']):
fold = options['folder']
else:
grass.fatal(_("Set folder parameter when using stdin for passing "
"the username and password"))
elif options['settings'] == '-':
if options['folder'] != '':
import getpass
if check(options['folder']):
fold = options['folder']
user = raw_input(_('Insert username: '******'Insert password: '******'settings']):
filesett = open(options['settings'], 'r')
fileread = filesett.readlines()
user = fileread[0].strip()
passwd = fileread[1].strip()
filesett.close()
else:
grass.fatal(_("File <%s> not found") % options['settings'])
# set the folder by option folder
if options['folder'] != '':
if check(options['folder']):
fold = options['folder']
# set the folder from path where settings file is stored
else:
path = os.path.split(options['settings'])[0]
temp = os.path.split(grass.tempfile())[0]
if temp in path:
grass.warning(_("You are downloading data into a temporary "
"directory. They will be deleted when you "
"close this GRASS GIS session"))
if check(path):
fold = path
# check the version
version = grass.core.version()
# this is would be set automatically
if version['version'].find('7.') == -1:
grass.fatal(_('GRASS GIS version 7 required'))
return 0
# the product
products = options['product'].split(',')
# first date and delta
firstday, finalday, delta = checkdate(options)
# set tiles
if options['tiles'] == '':
tiles = None
grass.warning(_("Option 'tiles' not set. Downloading all available tiles"))
else:
tiles = options['tiles']
# set the debug
if flags['d']:
debug_opt = True
else:
debug_opt = False
if flags['c']:
checkgdal = False
else:
checkgdal = True
for produ in products:
prod = product(produ).returned()
#start modis class
modisOgg = downModis(url=prod['url'], user=user, password=passwd,
destinationFolder=fold, tiles=tiles, delta=delta,
path=prod['folder'], product=prod['prod'],
today=str(firstday), enddate=str(finalday),
debug=debug_opt, checkgdal=checkgdal)
# connect to ftp
modisOgg.connect()
if modisOgg.nconnection <= 20:
# download tha tiles
grass.message(_("Downloading MODIS product <%s>..." % produ))
modisOgg.downloadsAllDay()
filesize = int(os.path.getsize(modisOgg.filelist.name))
if flags['g'] and filesize != 0:
grass.message("files=%s" % modisOgg.filelist.name)
elif filesize == 0:
grass.message(_("No data download, probably they have been "
"previously downloaded"))
elif filesize != 0:
grass.message(_("All data have been downloaded, continue "
"with i.modis.import with the option "
"'files=%s'" % modisOgg.filelist.name))
else:
grass.fatal(_("Error during connection"))
TILE_HEIGHT = TILE_WIDTH
wgs84_proj = Proj('EPSG:4326')
modis_grid = Proj(f'+proj=sinu +R={EARTH_RADIUS} +nagrids=@null '
f'+ellps=WGS84 +wktext')
transformer = Transformer.from_proj(wgs84_proj, modis_grid, always_xy=True)
def wgs84_to_modis_tile(xx, yy):
x, y = transformer.transform(xx, yy)
h = (EARTH_WIDTH * .5 + x) / TILE_WIDTH
v = (VERTICAL_TILES * TILE_HEIGHT - y - EARTH_WIDTH * .25) / TILE_HEIGHT
return f'h{int(h)}v{int(v):02},'
modisOgg = downmodis.downModis(
user='******',
password='******',
destinationFolder='/home/artem/Code/ndvi-test/modis_data',
tiles=wgs84_to_modis_tile(127.551270, 35.793457),
product='MOD09GA.006',
delta=0,
today='2021-02-09',
enddate='2021-02-09',
jpg=True)
modisOgg.connect()
if modisOgg.nconnection <= 20:
modisOgg.downloadsAllDay(clean=True, allDays=False)
def main():
"""Main function"""
#usage
usage = "usage: %prog [options] destination_folder"
parser = OptionParser(usage=usage)
#password
parser.add_option("-P", "--password", dest="password",
help="password to connect to ftp server", required=True)
#username
parser.add_option("-U", "--username", dest="user", default="anonymous",
help="username to connect to ftp server [default=%default]")
#url
parser.add_option("-u", "--url", default="e4ftl01.cr.usgs.gov",
help="ftp server url [default=%default]", dest="url")
#tiles
parser.add_option("-t", "--tiles", dest="tiles", default="None",
help="string of tiles separated from comma " \
+ "[default=%default for all tiles]")
#path to add the url
parser.add_option("-s", "--source", dest="path", default="MOLT/MOD11A1.005",
help="directory on the ftp [default=%default]")
#delta
parser.add_option("-D", "--delta", dest="delta", default=10,
help="delta of day from the first day " \
+ "[default=%default]")
#first day
parser.add_option("-f", "--firstday", dest="today", default=None,
metavar="LAST_DAY", help="the day to start download " \
+ "[default=%default is for today]; if you want change" \
" data you must use this format YYYY-MM-DD")
#first day
parser.add_option("-e", "--endday", dest="enday", default=None,
metavar="FIRST_DAY", help="the day to start download " \
+ "[default=%default]; if you want change" \
" data you must use this format YYYY-MM-DD")
#debug
parser.add_option("-x", action="store_true", dest="debug", default=False,
help="this is useful for debugging the " \
"download [default=%default]")
#jpg
parser.add_option("-j", action="store_true", dest="jpg", default=False,
help="download also the jpeg files [default=%default]")
#only one day
parser.add_option("-O", dest="oneday", action="store_true", default=False,
help="download only one day, it set " \
"delta=1 [default=%default]")
#all days
parser.add_option("-A", dest="alldays", action="store_true", default=False,
help="download all days, it usefull for first download "\
"of a product. It overwrite the 'firstday' and 'endday'"\
" options [default=%default]")
#remove file with size = 0
parser.add_option("-r", dest="empty", action="store_true", default=False,
help="remove files with size ugual to zero from " \
"'destination_folder' [default=%default]")
#parser.add_option("-A", dest="alldays", action="store_true", default=True,
#help="download all days from the first")
#set false several options
parser.set_defaults(oneday=False)
parser.set_defaults(debug=False)
parser.set_defaults(jpg=False)
#return options and argument
(options, args) = parser.parse_args()
#test if args[0] it is set
if len(args) == 0:
parser.error("You have to pass the destination folder for HDF file")
#check if oneday option it is set
if options.oneday:
options.delta = 1
#set modis object
modisOgg = downmodis.downModis(url=options.url, user=options.user,
password=options.password, destinationFolder=args[0],
tiles=options.tiles, path=options.path, today=options.today,
enddate=options.enday, delta=int(options.delta), jpg=options.jpg,
debug=options.debug)
#connect to ftp
modisOgg.connectFTP()
if modisOgg.nconnection <= 20:
#download data
modisOgg.downloadsAllDay(clean=options.empty, allDays=options.alldays)
else:
parser.error("Some problem with connection occur")
#导入函数库
import os
from pymodis import downmodis
dest = r"D:\f_work\QingHai_book\MOD13Q1" # This directory for data download
if not os.path.exists(dest): #Test whether a path exists
os.makedirs(dest)
# Variables for data download
# That's the MODIS tile covering China except the islands in sourth
#tiles = "h23v04, h23v05, h24v04, h24v05, h25v03, h25v04, h25v05, \
# h25v06, h26v03, h26v04, h26v05, h26v06, \
# h27v04, h27v05, h27v06, h28v05, h28v06, h28v07, h29v06"
tiles = "h25v05, h26v05"
product = "MOD13Q1.006"
year = 2000, 2005
for yr in range(year[0], year[1] + 1):
startdate = str(yr) + ".01.01"
enddate = str(yr) + ".12.31"
print(str(yr) + ' is downloading !')
modis_down = downmodis.downModis(destinationFolder=dest,
tiles=tiles,
url='https://e4ftl01.cr.usgs.gov',
today=startdate,
user="******",
password="******",
enddate=enddate,
product=product,
path="MOLT") #MOTA
modis_down.connect() #Connect to the server and fill the dirData variable
modis_down.downloadsAllDay() #Download all requested days
def main():
try:
from rmodislib import product
except:
grass.fatal("i.modis library is not installed")
try:
from pymodis.downmodis import downModis
except:
grass.fatal("pymodis library is not installed")
# check if you are in GRASS
gisbase = os.getenv("GISBASE")
if not gisbase:
grass.fatal(_("$GISBASE not defined"))
return 0
if flags["l"]:
prod = product()
prod.print_prods()
return 0
# empty settings and folder would collide
if not options["settings"] and not options["folder"]:
grass.fatal(
_("With empty settings parameter (to use the .netrc file) "
"the folder parameter needs to be specified"))
# set username, password and folder if settings are insert by stdin
if not options["settings"]:
user = None
passwd = None
if check_folder(options["folder"]):
fold = options["folder"]
else:
grass.fatal(
_("Set folder parameter when using stdin for passing "
"the username and password"))
elif options["settings"] == "-":
if options["folder"] != "":
import getpass
if check_folder(options["folder"]):
fold = options["folder"]
user = raw_input(_("Insert username: "******"Insert password: "******"Set folder parameter when using stdin for passing "
"the username and password"))
# set username, password and folder by file
else:
if not os.path.isfile(options["settings"]):
grass.fatal(
_("The settings parameter <{}> is not a file").format(
options["settings"]))
# open the file and read the the user and password:
# first line is username
# second line is password
try:
with open(options["settings"], "r") as filesett:
fileread = filesett.readlines()
user = fileread[0].strip()
passwd = fileread[1].strip()
except (FileNotFoundError, PermissionError) as e:
grass.fatal(_("Unable to read settings: {}").format(e))
if options["folder"] != "":
if check_folder(options["folder"]):
fold = options["folder"]
# set the folder from path where settings file is stored
else:
path = os.path.split(options["settings"])[0]
temp = os.path.split(grass.tempfile())[0]
if temp in path:
grass.warning(
_("You are downloading data into a temporary "
"directory. They will be deleted when you "
"close this GRASS GIS session"))
if check_folder(path):
fold = path
# the product
products = options["product"].split(",")
# first date and delta
firstday, finalday, delta = checkdate(options)
# set tiles
if options["tiles"] == "":
tiles = None
grass.warning(
_("Option 'tiles' not set. Downloading all available tiles to <{}>"
).format(fold))
else:
tiles = options["tiles"]
# set the debug
if flags["d"]:
debug_opt = True
else:
debug_opt = False
if flags["c"]:
checkgdal = False
else:
checkgdal = True
for produ in products:
prod = product(produ).returned()
# start modis class
modisOgg = downModis(
url=prod["url"],
user=user,
password=passwd,
destinationFolder=fold,
tiles=tiles,
delta=delta,
path=prod["folder"],
product=prod["prod"],
today=firstday,
enddate=finalday,
debug=debug_opt,
checkgdal=checkgdal,
)
# connect to ftp
modisOgg.connect()
if modisOgg.nconnection <= 20:
# download tha tiles
grass.message(
_("Downloading MODIS product <{}> ({})...".format(
produ, prod["prod"])))
modisOgg.downloadsAllDay()
filesize = int(os.path.getsize(modisOgg.filelist.name))
if flags["g"] and filesize != 0:
grass.message("files=%s" % modisOgg.filelist.name)
elif filesize == 0:
grass.message(
_("No data download, probably they have been "
"previously downloaded"))
elif filesize != 0:
grass.message(
_("All data have been downloaded, continue "
"with i.modis.import with the option "
"'files=%s'" % modisOgg.filelist.name))
else:
grass.fatal(_("Error during connection"))
def main():
"""Main function"""
# usage
usage = "usage: %prog [options] destination_folder"
if 1 == len(sys.argv):
option_parser_class = optparse_gui.OptionParser
else:
option_parser_class = optparse_required.OptionParser
parser = option_parser_class(usage=usage, description="modis_download_from_list")
# file
parser.add_option("-f", "--file", dest="file", type="file", help="Input file containing data to download")
# url
parser.add_option(
"-u", "--url", default="http://e4ftl01.cr.usgs.gov", help="http/ftp server url [default=%default]", dest="url"
)
# password
parser.add_option("-P", "--password", dest="password", help="password to connect only if ftp server")
# username
parser.add_option(
"-U",
"--username",
dest="user",
default="anonymous",
help="username to connect only if ftp server " "[default=%default]",
)
# path to add the path in the server
parser.add_option(
"-s", "--source", dest="path", default="MOLT", help="directory on the http/ftp server " "[default=%default]"
)
# path to add the url
parser.add_option(
"-p",
"--product",
dest="prod",
default="MOD11A1.005",
help="product name as on the http/ftp server " "[default=%default]",
)
# debug
parser.add_option(
"-x",
action="store_true",
dest="debug",
default=False,
help="this is useful for debugging the " "download [default=%default]",
)
# jpg
parser.add_option(
"-j",
action="store_true",
dest="jpg",
default=False,
help="download also the jpeg overview files [default=%default]",
)
# return options and argument
(options, args) = parser.parse_args()
if len(args) > 1:
parser.error("You have to define the destination folder for HDF file")
f = open(options.file)
lines = [elem for elem in f.readlines()]
tiles = [elem.split(".")[2] for elem in lines]
tiles = ",".join(sorted(set(tiles)))
dates = [elem.split(".")[1].replace("A", "") for elem in lines]
dates = sorted(set(dates))
for d in dates:
year = int(d[0:4])
doy = int(d[4:7])
fdate = date.fromordinal(date(year, 1, 1).toordinal() + doy - 1).isoformat()
modisOgg = downmodis.downModis(
url=options.url,
user=options.user,
password=options.password,
destinationFolder=args[0],
tiles=tiles,
path=options.path,
product=options.prod,
delta=1,
today=fdate,
debug=options.debug,
jpg=options.jpg,
)
modisOgg.connect()
day = modisOgg.getListDays()[0]
if modisOgg.urltype == "http":
listAllFiles = modisOgg.getFilesList(day)
else:
listAllFiles = modisOgg.getFilesList()
listFilesDown = modisOgg.checkDataExist(listAllFiles)
modisOgg.dayDownload(day, listFilesDown)
if modisOgg.urltype == "http":
modisOgg.closeFilelist()
else:
modisOgg.closeFTP()
def download_last_modis():
today = date.today()
start = (today - timedelta(days=1)).strftime("%Y-%m-%d")
end = (today - timedelta(days=5)).strftime("%Y-%m-%d")
# tiles to download-Greece region
tiles = 'h19v05,h20v05,h19v04'
# starting day
day = start
# finishing day
enddate = end
# username
user = '******'
# password
password = '******'
# destination folder
dest = '/home/sg/test_modis_download/last'
# log
delta = 1
product = 'MYD13A1.006' # ,MOD13A1.006'
path = 'MOLA' # ,MOLT'
# product2 = 'MOD13A1.006'
# path2 = 'MOLT'
modisDown = downmodis.downModis(destinationFolder=dest, tiles=tiles, today=day, enddate=enddate,
delta=delta, user=user, password=password, product=product, path=path,
jpg=False, debug=False, timeout=30, checkgdal=True)
modisDown.connect()
#files_list = modisDown.getFilesList
#new_files = modisDown.getAllDays()
modisDown.downloadsAllDay()
modisDown.removeEmptyFiles()
my_path = '/home/sg/test_modis_download/last'
with open("/home/sg/test_modis_download/archive/most_recent.json") as f:
last_images = json.load(f)
i = 0
for image in os.listdir(my_path):
if image.endswith('hdf'):
i = 1
check = modisDown.checkFile(image)
if check == 1:
os.rename("/home/sg/test_modis_download/last/" + image, "/home/sg/test_modis_download/archive/" + image)
os.rename("/home/sg/test_modis_download/last/" + image + '.xml',
"/home/sg/test_modis_download/archive/" + image + '.xml')
tile = image[17:23]
last_images[tile] = image
else:
print('Image %s corrupted. Keeping previous image' % image)
if i == 0:
print('MOLA: No new images found')
product = 'MOD13A1.006'
path = 'MOLT'
modisDown = downmodis.downModis(destinationFolder=dest, tiles=tiles, today=day, enddate=enddate,
delta=delta, user=user, password=password, product=product, path=path,
jpg=False, debug=False, timeout=30, checkgdal=True)
modisDown.connect()
modisDown.downloadsAllDay()
modisDown.removeEmptyFiles()
for image in os.listdir(my_path):
if image.endswith('hdf'):
i = 1
check = modisDown.checkFile(image)
if check == 1:
os.rename("/home/sg/test_modis_download/last/" + image, "/home/sg/test_modis_download/archive/" + image)
os.rename("/home/sg/test_modis_download/last/" + image + '.xml',
"/home/sg/test_modis_download/archive/" + image + '.xml')
tile = image[17:23]
last_images[tile] = image
else:
print('Image %s corrupted. Keeping previous image' % image)
if i == 0:
print('MOLT: No new images found')
json_images = json.dumps(last_images)
f = open("/home/sg/test_modis_download/archive/most_recent.json", "w")
f.write(json_images)
f.close()
import os
import pymodis
from pymodis import downmodis
from pymodis import parsemodis
import glob
dest = "/Users/iza/Desktop/modis_data"
tiles = "h18v04,h19v04"
day = "2014-08-14"
delta = 1
modisDown = downmodis.downModis(destinationFolder=dest,
tiles=tiles,
today=day,
delta=delta)
modisDown.connect()
modisDown.downloadsAllDay()
files = glob.glob(os.path.join(dest, 'MOD11A1.A2014*.hdf'))
print(files)
from pymodis import downmodis import glob import numpy as np import matplotlib.pyplot as plt # Variables for data download dest = "/home/dutri001/sandbox/MODIS/" tiles = "h18v03" day = "2015.11.18" enddate = "2015.11.16" product = "MOD13A2.006" # Init download class, connect and download modis_down = downmodis.downModis(destinationFolder=dest, tiles=tiles, today=day, enddate=enddate, product=product) modis_down.connect() modis_down.downloadsAllDay() # List files with the hdf extension MODIS_file = glob.glob(dest + '*.hdf') # Get subdatasets of the first file of the MODIS_file list sds = gdal.Open(MODIS_file[0], gdal.GA_ReadOnly).GetSubDatasets() print sds[0][0] print sds[11][0] # Open vi, to get the metadata (geoTransform, dimensions, projection) vi = gdal.Open(sds[0][0]) ind = [1, 11] # vi_np = np.zeros((vi.RasterYSize, vi.RasterXSize, 2), np.int16)
f.close()
# Armazena as informações de usuario e senha
usuario = info[0][:-1]
senha = info[1][:-1]
# Faz o download
for year in range(ano_inicio, ano_final + 1):
for month in range(mes_inicio, mes_final + 1):
for i in range(0, 2):
dia_final = "%04d.%02d.%02d" % (
year, month, calendar.monthrange(year, month)[1])
dia_inicial = "%04d.%02d.01" % (year, month)
# Cria o diretório para armazenar as imagens
destino = os.path.join(
BASE_DIR, "data/lst/%s/%04d/%02d" % (satelite[i], year, month))
if not os.path.isdir(destino):
os.makedirs(destino)
# Cria uma classe de download do MODIS, conecta e faz o download
modis_down = downmodis.downModis(password=senha,
user=usuario,
destinationFolder=destino,
path=pasta[i],
tiles=tiles,
today=dia_final,
enddate=dia_inicial,
product=produto[i])
modis_down.connect()
modis_down.downloadsAllDay()
def main():
"""Main function"""
# usage
usage = "usage: %prog [options] destination_folder"
if 1 == len(sys.argv) and WXPYTHON:
option_parser_class = optparse_gui.OptionParser
else:
option_parser_class = optparse_required.OptionParser
parser = option_parser_class(usage=usage,
description='modis_download_from_list')
# file
parser.add_option("-f", "--file", dest="file", type='file',
help="Input file containing data to download")
# url
parser.add_option("-u", "--url", default="http://e4ftl01.cr.usgs.gov",
help="http/ftp server url [default=%default]",
dest="url")
# username and password from stdin
parser.add_option("-I", "--input", dest="input", action="store_true",
help="insert user and password from standard input")
# password
parser.add_option("-P", "--password", dest="password",
help="password to connect only if ftp server")
# username
parser.add_option("-U", "--username", dest="user", default="anonymous",
help="username to connect only if ftp server "
"[default=%default]")
# path to add the path in the server
parser.add_option("-s", "--source", dest="path", default="MOLT",
help="directory on the http/ftp server "
"[default=%default]")
# path to add the url
parser.add_option("-p", "--product", dest="prod", default="MOD11A1.005",
help="product name as on the http/ftp server "
"[default=%default]")
# debug
parser.add_option("-x", action="store_true", dest="debug", default=False,
help="this is useful for debugging the "
"download [default=%default]")
# jpg
parser.add_option("-j", action="store_true", dest="jpg", default=False,
help="download also the jpeg overview files "
"[default=%default]")
# return options and argument
(options, args) = parser.parse_args()
if len(args) == 0 and not WXPYTHON:
parser.print_help()
sys.exit(1)
if len(args) > 1:
parser.error("You have to define the destination folder for HDF file")
if not os.path.isdir(args[0]):
parser.error("The destination folder is not a dir or not exists")
if options.input:
if sys.version_info.major == 3:
user = input("Username: "******"Username: "******"You have to set user and password")
f = open(options.file)
lines = [elem for elem in f.readlines()]
tiles = [elem.strip().split('.')[2] for elem in lines if elem != '\n']
tiles = ','.join(sorted(set(tiles)))
dates = [elem.split('.')[1].replace('A', '') for elem in lines if elem != '\n']
dates = sorted(set(dates))
for d in dates:
year = int(d[0:4])
doy = int(d[4:7])
fdate = date.fromordinal(date(year, 1, 1).toordinal() + doy - 1).isoformat()
modisOgg = downmodis.downModis(url=options.url, user=options.user,
password=options.password,
destinationFolder=args[0],
tiles=tiles, path=options.path,
product=options.prod, delta=1,
today=fdate, debug=options.debug,
jpg=options.jpg)
modisOgg.connect()
day = modisOgg.getListDays()[0]
if modisOgg.urltype == 'http':
listAllFiles = modisOgg.getFilesList(day)
else:
listAllFiles = modisOgg.getFilesList()
listFilesDown = modisOgg.checkDataExist(listAllFiles)
modisOgg.dayDownload(day, listFilesDown)
if modisOgg.urltype == 'http':
modisOgg.closeFilelist()
else:
modisOgg.closeFTP()
def main():
"""Main function"""
# usage
usage = "usage: %prog [options] destination_folder"
if 1 == len(sys.argv) and WXPYTHON:
option_parser_class = optparse_gui.OptionParser
else:
option_parser_class = optparse_required.OptionParser
parser = option_parser_class(usage=usage,
description='modis_download_from_list')
# file
parser.add_option("-f",
"--file",
dest="file",
type='file',
help="Input file containing data to download")
# url
parser.add_option("-u",
"--url",
default="http://e4ftl01.cr.usgs.gov",
help="http/ftp server url [default=%default]",
dest="url")
# password
parser.add_option("-P",
"--password",
dest="password",
help="password to connect only if ftp server")
# username
parser.add_option("-U",
"--username",
dest="user",
default="anonymous",
help="username to connect only if ftp server "
"[default=%default]")
# path to add the path in the server
parser.add_option("-s",
"--source",
dest="path",
default="MOLT",
help="directory on the http/ftp server "
"[default=%default]")
# path to add the url
parser.add_option("-p",
"--product",
dest="prod",
default="MOD11A1.005",
help="product name as on the http/ftp server "
"[default=%default]")
# debug
parser.add_option("-x",
action="store_true",
dest="debug",
default=False,
help="this is useful for debugging the "
"download [default=%default]")
# jpg
parser.add_option("-j",
action="store_true",
dest="jpg",
default=False,
help="download also the jpeg overview files "
"[default=%default]")
# return options and argument
(options, args) = parser.parse_args()
if len(args) == 0 and not WXPYTHON:
parser.print_help()
sys.exit(1)
if len(args) > 1:
parser.error("You have to define the destination folder for HDF file")
f = open(options.file)
lines = [elem for elem in f.readlines()]
tiles = [elem.strip().split('.')[2] for elem in lines if elem != '\n']
tiles = ','.join(sorted(set(tiles)))
dates = [
elem.split('.')[1].replace('A', '') for elem in lines if elem != '\n'
]
dates = sorted(set(dates))
for d in dates:
year = int(d[0:4])
doy = int(d[4:7])
fdate = date.fromordinal(date(year, 1, 1).toordinal() + doy -
1).isoformat()
modisOgg = downmodis.downModis(url=options.url,
user=options.user,
password=options.password,
destinationFolder=args[0],
tiles=tiles,
path=options.path,
product=options.prod,
delta=1,
today=fdate,
debug=options.debug,
jpg=options.jpg)
modisOgg.connect()
day = modisOgg.getListDays()[0]
if modisOgg.urltype == 'http':
listAllFiles = modisOgg.getFilesList(day)
else:
listAllFiles = modisOgg.getFilesList()
listFilesDown = modisOgg.checkDataExist(listAllFiles)
modisOgg.dayDownload(day, listFilesDown)
if modisOgg.urltype == 'http':
modisOgg.closeFilelist()
else:
modisOgg.closeFTP()
def main():
"""Main function"""
# usage
usage = "usage: %prog [options] destination_folder"
if 1 == len(sys.argv) and WXPYTHON:
option_parser_class = optparse_gui.OptionParser
else:
option_parser_class = optparse_required.OptionParser
parser = option_parser_class(usage=usage, description='modis_download')
# url
parser.add_option("-u",
"--url",
default="https://e4ftl01.cr.usgs.gov",
help="http/ftp server url [default=%default]",
dest="url")
# username and password from stdin
parser.add_option("-I",
"--input",
dest="input",
action="store_true",
help="insert user and password from standard input")
# password
parser.add_option("-P",
"--password",
dest="password",
help="password to connect to the server")
# username
parser.add_option("-U",
"--username",
dest="user",
help="username to connect to the server")
# tiles
parser.add_option("-t",
"--tiles",
dest="tiles",
default=None,
help="string of tiles separated with comma "
"[default=%default for all tiles]")
# path to add the path in the server
parser.add_option("-s",
"--source",
dest="path",
default="MOLT",
help="directory on the http/ftp server "
"[default=%default]")
# path to add the url
parser.add_option("-p",
"--product",
dest="prod",
default="MOD11A1.005",
help="product name as on the http/ftp server "
"[default=%default]")
# delta
parser.add_option("-D",
"--delta",
dest="delta",
default=10,
help="delta of day from the first day "
"[default=%default]")
# first day
parser.add_option("-f",
"--firstday",
dest="today",
default=None,
help="the day to start download [default=%default is for"
" today]; if you want change data you must use "
"this format YYYY-MM-DD",
metavar="FIRST_DAY")
# last day
parser.add_option("-e",
"--endday",
dest="enday",
default=None,
metavar="LAST_DAY",
help="the day to stop download "
"[default=%default]; if you want change"
" data you must use this format YYYY-MM-DD")
# debug
parser.add_option("-x",
action="store_true",
dest="debug",
default=False,
help="this is useful for debugging the "
"download [default=%default]")
# jpg
parser.add_option("-j",
action="store_true",
dest="jpg",
default=False,
help="download also the jpeg files [default=%default]")
# only one day
parser.add_option("-O",
dest="oneday",
action="store_true",
default=False,
help="download only one day, it set "
"delta=1 [default=%default]")
# all days
parser.add_option("-A",
dest="alldays",
action="store_true",
default=False,
help="download all days, it useful for initial download"
" of a product. It overwrite the 'firstday' and "
"'endday' options [default=%default]")
# remove file with size = 0
parser.add_option("-r",
dest="empty",
action="store_true",
default=False,
help="remove empty files (size equal to zero) from "
"'destination_folder' [default=%default]")
#parser.add_option("-A", dest="alldays", action="store_true", default=True,
#help="download all days from the first")
# set false several options
parser.set_defaults(oneday=False)
parser.set_defaults(debug=False)
parser.set_defaults(jpg=False)
# return options and argument
(options, args) = parser.parse_args()
# test if args[0] it is set
if len(args) == 0 and not WXPYTHON:
parser.print_help()
sys.exit(1)
if len(args) == 0:
parser.error("You have to define the destination folder for HDF file")
if not os.path.isdir(args[0]):
parser.error("The destination folder is not a dir or not exists")
# check if oneday option it is set
if options.oneday:
options.delta = 1
if options.input:
if sys.version_info.major == 3:
user = input("Username: "******"Username: "******"A problem with the connection occured")
def main():
"""Main function"""
# usage
usage = "usage: %prog [options] destination_folder"
if 1 == len(sys.argv) and WXPYTHON:
option_parser_class = optparse_gui.OptionParser
else:
option_parser_class = optparse_required.OptionParser
parser = option_parser_class(usage=usage,
description='modis_download_from_list')
# file
parser.add_option("-f",
"--file",
dest="file",
type='file',
help="Input file containing data to download")
# url
parser.add_option("-u",
"--url",
default="https://e4ftl01.cr.usgs.gov",
help="http/ftp server url [default=%default]",
dest="url")
# username and password from stdin
parser.add_option("-I",
"--input",
dest="input",
action="store_true",
help="insert user and password from standard input")
# password
parser.add_option("-P",
"--password",
dest="password",
help="password to connect to the server")
# username
parser.add_option("-U",
"--username",
dest="user",
help="username to connect to the server ")
# path to add the path in the server
parser.add_option("-s",
"--source",
dest="path",
default="MOLT",
help="directory on the http/ftp server "
"[default=%default]")
# path to add the url
parser.add_option("-p",
"--product",
dest="prod",
default="MOD11A1.006",
help="product name as on the http/ftp server "
"[default=%default]")
# path to file with server missing tiles
parser.add_option("-o",
"--outputs",
dest="outs",
default=None,
help="the output where write the missing files in the"
" server [default=%default]. Use 'stdout' to write to "
" STDOUT")
# use netrc file
parser.add_option("-n",
action="store_true",
dest="netrc",
default=False,
help="use netrc file to read user and password")
# debug
parser.add_option("-x",
action="store_true",
dest="debug",
default=False,
help="this is useful for debugging the "
"download [default=%default]")
# jpg
parser.add_option("-j",
action="store_true",
dest="jpg",
default=False,
help="download also the jpeg overview files "
"[default=%default]")
# return options and argument
(options, args) = parser.parse_args()
if len(args) == 0 and not WXPYTHON:
parser.print_help()
sys.exit(1)
if len(args) > 1:
parser.error("You have to define the destination folder for HDF file")
if not os.path.isdir(args[0]):
parser.error("The destination folder is not a dir or not exists")
if options.netrc:
user = None
password = None
elif options.input:
if sys.version_info.major == 3:
user = input("Username: "******"Username: "******"stdout":
write = sys.stdout
else:
write = open(options.outs, 'w')
for d, tiles in sorted(vals.items(), reverse=True):
year = int(d[0:4])
doy = int(d[4:7])
fdate = date.fromordinal(date(year, 1, 1).toordinal() + doy -
1).isoformat()
modisOgg = downmodis.downModis(url=options.url,
user=user,
password=password,
destinationFolder=args[0],
tiles=','.join(sorted(set(tiles))),
path=options.path,
product=options.prod,
delta=1,
today=fdate,
debug=options.debug,
jpg=options.jpg)
modisOgg.connect()
day = modisOgg.getListDays()[0]
if modisOgg.urltype == 'http':
listAllFiles = modisOgg.getFilesList(day)
else:
listAllFiles = modisOgg.getFilesList()
if not listAllFiles:
if write:
write_out(write, tiles, options, d)
continue
listFilesDown = modisOgg.checkDataExist(listAllFiles)
if listFilesDown:
if options.debug:
print(listFilesDown)
modisOgg.dayDownload(day, listFilesDown)
for fi in listFilesDown:
if 'xml' not in fi:
try:
tiles.remove(fi.split('.')[2])
except ValueError:
pass
if write:
write_out(write, tiles, options, d)
if modisOgg.urltype == 'http':
modisOgg.closeFilelist()
else:
modisOgg.closeFTP()
def main():
"""Main function"""
# usage
usage = "usage: %prog [options] destination_folder"
if 1 == len(sys.argv) and WXPYTHON:
option_parser_class = optparse_gui.OptionParser
else:
option_parser_class = optparse_required.OptionParser
parser = option_parser_class(usage=usage,
description='modis_download_from_list')
# file
parser.add_option("-f", "--file", dest="file", type='file',
help="Input file containing data to download")
# url
parser.add_option("-u", "--url", default="https://e4ftl01.cr.usgs.gov",
help="http/ftp server url [default=%default]",
dest="url")
# username and password from stdin
parser.add_option("-I", "--input", dest="input", action="store_true",
help="insert user and password from standard input")
# password
parser.add_option("-P", "--password", dest="password",
help="password to connect to the server")
# username
parser.add_option("-U", "--username", dest="user",
help="username to connect to the server ")
# path to add the path in the server
parser.add_option("-s", "--source", dest="path", default="MOLT",
help="directory on the http/ftp server "
"[default=%default]")
# path to add the url
parser.add_option("-p", "--product", dest="prod", default="MOD11A1.006",
help="product name as on the http/ftp server "
"[default=%default]")
# path to file with server missing tiles
parser.add_option("-o", "--outputs", dest="outs", default=None,
help="the output where write the missing files in the"
" server [default=%default]. Use 'stdout' to write to "
" STDOUT")
# use netrc file
parser.add_option("-n", action="store_true", dest="netrc", default=False,
help="use netrc file to read user and password")
# debug
parser.add_option("-x", action="store_true", dest="debug", default=False,
help="this is useful for debugging the "
"download [default=%default]")
# jpg
parser.add_option("-j", action="store_true", dest="jpg", default=False,
help="download also the jpeg overview files "
"[default=%default]")
# return options and argument
(options, args) = parser.parse_args()
if len(args) == 0 and not WXPYTHON:
parser.print_help()
sys.exit(1)
if len(args) > 1:
parser.error("You have to define the destination folder for HDF file")
if not os.path.isdir(args[0]):
parser.error("The destination folder is not a dir or not exists")
if options.netrc:
user = None
password = None
elif options.input:
if sys.version_info.major == 3:
user = input("Username: "******"Username: "******"stdout":
write = sys.stdout
else:
write = open(options.outs, 'w')
for d, tiles in sorted(vals.items(), reverse=True):
year = int(d[0:4])
doy = int(d[4:7])
fdate = date.fromordinal(date(year, 1, 1).toordinal() + doy - 1).isoformat()
modisOgg = downmodis.downModis(url=options.url, user=user,
password=password,
destinationFolder=args[0],
tiles=','.join(sorted(set(tiles))),
path=options.path, product=options.prod,
delta=1, today=fdate,
debug=options.debug, jpg=options.jpg)
modisOgg.connect()
day = modisOgg.getListDays()[0]
if modisOgg.urltype == 'http':
listAllFiles = modisOgg.getFilesList(day)
else:
listAllFiles = modisOgg.getFilesList()
if not listAllFiles:
if write:
write_out(write, tiles, options, d)
continue
listFilesDown = modisOgg.checkDataExist(listAllFiles)
if listFilesDown:
if options.debug:
print(listFilesDown)
modisOgg.dayDownload(day, listFilesDown)
for fi in listFilesDown:
if 'xml' not in fi:
try:
tiles.remove(fi.split('.')[2])
except ValueError:
pass
if write:
write_out(write, tiles, options, d)
if modisOgg.urltype == 'http':
modisOgg.closeFilelist()
else:
modisOgg.closeFTP()
def download_data(self,
output_dir,
product,
dates=None,
tiles=None,
mosaic_dir=None,
overwrite=False):
logger.info('entering download_data')
files = None
_m_files = []
_dl_files = []
_delta = 1
_user = self.vp.get('MODIS', 'user')
_password = self.vp.get('MODIS', 'password')
_dates = []
# if dates is None:
# need to get list of dates first
if not os.path.exists(output_dir):
logger.debug("%s does not exist. Creating directory", output_dir)
os.makedirs(output_dir)
_modis_connect = downmodis.downModis(destinationFolder=output_dir,
user=_user,
password=_password,
product=product,
today=None,
delta=_delta,
tiles=tiles)
_modis_connect.connect()
_days_list = _modis_connect.getAllDays()
for i in _days_list:
if dates is None:
_dates.append(
i) #('{0}-{1}'.format(i.split('.')[0], i.split('.')[1]))
else:
# check if in month/year
if '{0}-{1}'.format(i.split('.')[0], i.split('.')[1]) in dates:
_dates.append(i)
# else:
# _dates = dates
for d in _dates:
logger.debug("downloading data for %s", d)
_month = d #'{date}-01'.format(date=d)
_folder_date = datetime.datetime.strptime(
_month, '%Y.%m.%d').strftime('%Y.%m.%d')
_new_folder = output_dir #os.path.join(output_dir, _folder_date)
# don't need anymore? done above instead?
# create folder if it doesn't already exist
if not os.path.exists(_new_folder):
logger.debug("%s does not exist. Creating directory",
_new_folder)
os.makedirs(_new_folder)
_modis_download = downmodis.downModis(
destinationFolder=_new_folder,
tiles=tiles,
product=product,
today=_month,
delta=_delta,
user=_user,
password=_password)
_modis_download.connect()
_check_files = _modis_download.getFilesList(_folder_date)
# logger.debug("check files: %s", _check_files)
_dl_files = _modis_download.checkDataExist(_check_files)
try:
_modis_download.downloadsAllDay(clean=True)
except Exception, e:
logger.debug(
'Error in pymodis.modisDown.downloadsAllDay downloading %s',
_dl_files)
raise ValueError('Error in pymodis.modisDown.downloadsAllDay')
_modis_download.removeEmptyFiles()
_modis_download.closeFilelist()
# want list of files downloaded (or files for this day)
_files_list = []
for f in _check_files:
if os.path.splitext(os.path.join(_new_folder, f))[1] == '.hdf':
_files_list.append(os.path.join(_new_folder, f))
_files = glob.glob(
os.path.join(_new_folder,
'{product}*.hdf'.format(product=product[:-4])))
# logger.debug("files: %s", _files)
logger.debug("downloaded files: %s", _files_list)
if mosaic_dir:
# mosaic files
mosaic_file = self.mosaic_tiles(_files_list, mosaic_dir,
product)
_m_files.append(mosaic_file)
