def confile(pm, opts, q, mosaik=False): """Create the configuration file for MRT software""" try: # try to import pymodis (modis) and some classes for i.modis.download from rmodislib import resampling, product, projection except ImportError as e: grass.fatal("Unable to load i.modis library: {}".format(e)) # return projection and datum projObj = projection() proj = projObj.returned() dat = projObj.datum() if proj == "UTM": zone = projObj.utmzone() else: zone = None cod = os.path.split(pm.hdfname)[1].split(".")[0] prod = product().fromcode(cod) if mosaik: # if mosaic it remove all the 0 from the subset string to convert all # the right layer spectr = spectral(opts, prod, q, True) else: spectr = spectral(opts, prod, q) # out prefix pref = modis_prefix(pm.hdfname) # resampling resampl = resampling(opts["method"]).returned() # projpar projpar = projObj.return_params() if projpar != "( 0, 0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0 )": dat = "NODATUM" # resolution if proj != "GEO": res = int(prod["res"]) * int(projObj.proj["meters"]) else: res = None try: conf = pm.confResample(spectr, res, pref, dat, resampl, proj, zone, projpar) return conf except IOError as e: grass.fatal(e)
def main(): # check if you are in GRASS gisbase = os.getenv("GISBASE") if not gisbase: grass.fatal(_("$GISBASE not defined")) return 0 if flags["l"]: try: from rmodislib import product except ImportError as e: grass.fatal("Unable to load i.modis library: {}".format(e)) prod = product() prod.print_prods() return 0 # return an error if q and spectral are set if not flags["q"] and options["spectral"] != "": grass.warning( _('If no QA layer chosen in the "spectral" option' " the command will report an error")) # return an error if both input and files option are set or not if options["input"] == "" and options["files"] == "": grass.fatal(_('Choose one of "input" or "files" options')) return 0 elif options["input"] != "" and options["files"] != "": grass.fatal( _('It is not possible set "input" and "files"' " options together")) return 0 # check if remove the files or not if flags["t"]: remove = False else: remove = True if grass.overwrite(): over = True else: over = False # check if do check quality, rescaling and setting of colors if flags["q"]: analyze = False else: analyze = True # return the number of select layer from HDF files if options["spectral"]: count = options["spectral"].strip("(").strip(")").split().count("1") else: count = 0 outfile = None # check if file for t.register has to been created if options["outfile"]: if flags["a"]: outfile = open(options["outfile"], "a") else: outfile = open(options["outfile"], "w") if count > 1: grass.warning("The spectral subsets are more than one so the " " output file will be renamed") elif flags["w"] and not options["outfile"]: outfile = tempfile.NamedTemporaryFile(delete=False) # check if import simple file or mosaic if flags["m"] and options["input"] != "": grass.fatal( _("It is not possible to create a mosaic with a single" " HDF file")) return 0 elif flags["m"]: mosaic(options, remove, analyze, over, outfile) else: single(options, remove, analyze, over, outfile) # if t.register file is create if outfile: outfile.close() tempdir = grass.tempdir() # one layer only if count == 1: if flags["g"]: grass.message(_("file={name}".format(name=outfile.name))) else: grass.message( _("You can use temporal framework, registering" " the maps using t.register input=your_strds " "'file={name}'".format(name=outfile.name))) # for more layer create several files with only a subset for each layer elif count > 1: tfile = open(outfile.name) outfiles = {} lines = tfile.readlines() # get the codes from only one HDF for line in lines[:count]: if flags["m"]: code = "_".join(line.split("|")[0].split("_")[2:]) else: code = line.split("|")[0].split(".")[-1] outfiles[code] = open( os.path.join(tempdir, "{co}.txt".format(co=code)), "w") # split the lines for each code for line in lines: if flags["m"]: code = "_".join(line.split("|")[0].split("_")[2:]) else: code = line.split("|")[0].split(".")[-1] outfiles[code].write(line) for k, v in outfiles.items(): v.close() if flags["g"]: message = "" else: message = ("You can use temporal framework, registering the " "maps in different temporal datasets using " "t.register and \n") tfile.close() for fil in outfiles.values(): message += "'file={name}'\n".format(name=fil.name) grass.message(_(message))
def mosaic(options, remove, an, ow, fil): """Create a daily mosaic of HDF files convert to TIF and import it""" try: # try to import pymodis (modis) and some classes for i.modis.download from rmodislib import product, projection, get_proj except ImportError as e: grass.fatal("Unable to load i.modis library: {}".format(e)) try: from pymodis.convertmodis import convertModis, createMosaic from pymodis.convertmodis_gdal import createMosaicGDAL, convertModisGDAL from pymodis.parsemodis import parseModis except ImportError as e: grass.fatal("Unable to import pymodis library: {}".format(e)) dictfile, targetdir = list_files(options, True) pid = str(os.getpid()) # for each day count = len(dictfile.keys()) idx = 1 for dat, listfiles in dictfile.items(): grass.message( _("Processing <{d}> ({i}/{c})...").format(d=dat, i=idx, c=count)) grass.percent(idx, count, 5) idx += 1 pref = listfiles[0].split(os.path.sep)[-1] prod = product().fromcode(pref.split(".")[0]) spectr = spectral(options, prod, an) spectr = spectr.lstrip("( ").rstrip(" )") outname = "%s.%s_mosaic" % (pref.split(".")[0], pref.split(".")[1]) outname = outname.replace(" ", "_") # create mosaic if options["mrtpath"]: # create the file with the list of name tempfile = open(os.path.join(targetdir, pid), "w") tempfile.writelines(listfiles) tempfile.close() # basedir of tempfile, where hdf files are write basedir = os.path.split(tempfile.name)[0] # return the spectral subset in according mrtmosaic tool format cm = createMosaic(tempfile.name, outname, options["mrtpath"], spectr) cm.run() hdfiles = glob.glob1(basedir, outname + "*.hdf") else: basedir = targetdir listfiles = [os.path.join(basedir, i) for i in listfiles] cm = createMosaicGDAL(listfiles, spectr) try: cm.write_vrt(os.path.join(basedir, outname), quiet=True) except: cm.write_vrt(os.path.join(basedir, outname)) hdfiles = glob.glob1(basedir, outname + "*.vrt") for i in hdfiles: # the full path to hdf file hdf = os.path.join(basedir, i) try: pm = parseModis(hdf) except: out = i.replace(".vrt", "") data = doy2date(dat[1:]) pm = grassParseModis(out, data) # create convertModis class and convert it in tif file if options["mrtpath"]: # create conf file fro mrt tools confname = confile(pm, options, an, True) execmodis = convertModis(hdf, confname, options["mrtpath"]) else: confname = None projwkt = get_proj("w") projObj = projection() if projObj.returned() != "GEO": res = int(prod["res"]) * int(projObj.proj["meters"]) else: res = None execmodis = convertModisGDAL(str(hdf), out, spectr, res, wkt=str(projwkt), vrt=True) # produce temporary files in input folder os.chdir(basedir) try: execmodis.run(quiet=True) except: execmodis.run() # remove hdf if remove: # import tif files import_tif( basedir=basedir, rem=remove, write=ow, pm=pm, listfile=fil, prod=prod, ) try: os.remove(hdf) os.remove(hdf + ".xml") except OSError: pass # move the hdf and hdf.xml to the dir where are the original files else: # import tif files import_tif( basedir=basedir, rem=remove, write=ow, pm=pm, target=targetdir, listfile=fil, prod=prod, ) if i not in os.listdir(targetdir): try: shutil.move(hdf, targetdir) shutil.move(hdf + ".xml", targetdir) except OSError: pass # remove the conf file try: os.remove(confname) except (OSError, TypeError) as e: pass if options["mrtpath"]: grass.try_remove(tempfile.name) grass.try_remove(os.path.join(targetdir, "mosaic", pid))
def single(options, remove, an, ow, fil): """Convert the HDF file to TIF and import it""" try: # try to import pymodis (modis) and some classes for i.modis.download from rmodislib import product, projection, get_proj except ImportError as e: grass.fatal("Unable to load i.modis library: {}".format(e)) try: from pymodis.convertmodis import convertModis from pymodis.convertmodis_gdal import convertModisGDAL from pymodis.parsemodis import parseModis except ImportError as e: grass.fatal("Unable to import pymodis library: {}".format(e)) listfile, basedir = list_files(options) if not listfile: grass.warning(_("No HDF files found")) return # for each file count = len(listfile) idx = 1 for i in listfile: if os.path.exists(i): hdf = i else: # the full path to hdf file hdf = os.path.join(basedir, i) if not os.path.exists(hdf): grass.warning(_("%s not found" % i)) continue grass.message( _("Proccessing <{f}> ({i}/{c})...").format(f=os.path.basename(hdf), i=idx, c=count)) grass.percent(idx, count, 5) idx += 1 try: pm = parseModis(hdf) except OSError: grass.fatal(_("<{}> is not a HDF file").format(hdf)) continue if options["mrtpath"]: # create conf file fro mrt tools confname = confile(pm, options, an) # create convertModis class and convert it in tif file execmodis = convertModis(hdf, confname, options["mrtpath"]) else: projwkt = get_proj("w") projObj = projection() pref = i.split(os.path.sep)[-1] prod = product().fromcode(pref.split(".")[0]) spectr = spectral(options, prod, an) if projObj.returned() != "GEO": res = int(prod["res"]) * int(projObj.proj["meters"]) else: res = None outname = "%s.%s.%s.single" % ( pref.split(".")[0], pref.split(".")[1], pref.split(".")[2], ) outname = outname.replace(" ", "_") execmodis = convertModisGDAL(str(hdf), outname, spectr, res, wkt=str(projwkt)) # produce temporary files in input folder os.chdir(basedir) try: execmodis.run(quiet=True) except: execmodis.run() import_tif(basedir=basedir, rem=remove, write=ow, pm=pm, listfile=fil, prod=prod) if options["mrtpath"]: os.remove(confname)
def main(): # check if you are in GRASS gisbase = os.getenv('GISBASE') if not gisbase: grass.fatal(_('$GISBASE not defined')) return 0 if flags['l']: try: from rmodislib import product except ImportError as e: grass.fatal("Unable to load i.modis library: {}".format(e)) prod = product() prod.print_prods() return 0 # return an error if q and spectral are set if not flags['q'] and options['spectral'] != '': grass.warning( _('If no QA layer chosen in the "spectral" option' ' the command will report an error')) # return an error if both input and files option are set or not if options['input'] == '' and options['files'] == '': grass.fatal(_('Choose one of "input" or "files" options')) return 0 elif options['input'] != '' and options['files'] != '': grass.fatal( _('It is not possible set "input" and "files"' ' options together')) return 0 # check the version version = grass.core.version() # this is would be set automatically if version['version'].find('7.') == -1: grass.fatal(_('GRASS GIS version 7 required')) return 0 # check if remove the files or not if flags['t']: remove = False else: remove = True if grass.overwrite(): over = True else: over = False # check if do check quality, rescaling and setting of colors if flags['q']: analyze = False else: analyze = True # return the number of select layer from HDF files if options['spectral']: count = options['spectral'].strip('(').strip(')').split().count('1') else: count = 0 outfile = None # check if file for t.register has to been created if options['outfile']: if flags['a']: outfile = open(options['outfile'], 'a') else: outfile = open(options['outfile'], 'w') if count > 1: grass.warning("The spectral subsets are more than one so the " " output file will be renamed") elif flags['w'] and not options['outfile']: outfile = tempfile.NamedTemporaryFile(delete=False) # check if import simple file or mosaic if flags['m'] and options['input'] != '': grass.fatal( _('It is not possible to create a mosaic with a single' ' HDF file')) return 0 elif flags['m']: mosaic(options, remove, analyze, over, outfile) else: single(options, remove, analyze, over, outfile) # if t.register file is create if outfile: outfile.close() tempdir = grass.tempdir() # one layer only if count == 1: if flags['g']: grass.message(_("file={name}".format(name=outfile.name))) else: grass.message( _("You can use temporal framework, registering" " the maps using t.register input=your_strds " "'file={name}'".format(name=outfile.name))) # for more layer create several files with only a subset for each layer elif count > 1: tfile = open(outfile.name) outfiles = {} lines = tfile.readlines() # get the codes from only one HDF for line in lines[:count]: if flags['m']: code = '_'.join(line.split('|')[0].split('_')[2:]) else: code = line.split('|')[0].split('.')[-1] outfiles[code] = open( os.path.join(tempdir, "{co}.txt".format(co=code)), 'w') # split the lines for each code for line in lines: if flags['m']: code = '_'.join(line.split('|')[0].split('_')[2:]) else: code = line.split('|')[0].split('.')[-1] outfiles[code].write(line) for k, v in outfiles.items(): v.close() if flags['g']: message = "" else: message = "You can use temporal framework, registering the " \ "maps in different temporal datasets using " \ "t.register and \n" tfile.close() for fil in outfiles.values(): message += "'file={name}'\n".format(name=fil.name) grass.message(_(message))
def main(): try: from rmodislib import product except: grass.fatal("i.modis library is not installed") try: from pymodis.downmodis import downModis except: grass.fatal("pymodis library is not installed") # check if you are in GRASS gisbase = os.getenv('GISBASE') if not gisbase: grass.fatal(_('$GISBASE not defined')) return 0 if flags['l']: prod = product() prod.print_prods() return 0 # set username, password and folder if settings are insert by stdin if not options['settings']: user = None passwd = None if check(options['folder']): fold = options['folder'] else: grass.fatal(_("Set folder parameter when using stdin for passing " "the username and password")) elif options['settings'] == '-': if options['folder'] != '': import getpass if check(options['folder']): fold = options['folder'] user = raw_input(_('Insert username: '******'Insert password: '******'settings']): filesett = open(options['settings'], 'r') fileread = filesett.readlines() user = fileread[0].strip() passwd = fileread[1].strip() filesett.close() else: grass.fatal(_("File <%s> not found") % options['settings']) # set the folder by option folder if options['folder'] != '': if check(options['folder']): fold = options['folder'] # set the folder from path where settings file is stored else: path = os.path.split(options['settings'])[0] temp = os.path.split(grass.tempfile())[0] if temp in path: grass.warning(_("You are downloading data into a temporary " "directory. They will be deleted when you " "close this GRASS GIS session")) if check(path): fold = path # check the version version = grass.core.version() # this is would be set automatically if version['version'].find('7.') == -1: grass.fatal(_('GRASS GIS version 7 required')) return 0 # the product products = options['product'].split(',') # first date and delta firstday, finalday, delta = checkdate(options) # set tiles if options['tiles'] == '': tiles = None grass.warning(_("Option 'tiles' not set. Downloading all available tiles")) else: tiles = options['tiles'] # set the debug if flags['d']: debug_opt = True else: debug_opt = False if flags['c']: checkgdal = False else: checkgdal = True for produ in products: prod = product(produ).returned() #start modis class modisOgg = downModis(url=prod['url'], user=user, password=passwd, destinationFolder=fold, tiles=tiles, delta=delta, path=prod['folder'], product=prod['prod'], today=str(firstday), enddate=str(finalday), debug=debug_opt, checkgdal=checkgdal) # connect to ftp modisOgg.connect() if modisOgg.nconnection <= 20: # download tha tiles grass.message(_("Downloading MODIS product <%s>..." % produ)) modisOgg.downloadsAllDay() filesize = int(os.path.getsize(modisOgg.filelist.name)) if flags['g'] and filesize != 0: grass.message("files=%s" % modisOgg.filelist.name) elif filesize == 0: grass.message(_("No data download, probably they have been " "previously downloaded")) elif filesize != 0: grass.message(_("All data have been downloaded, continue " "with i.modis.import with the option " "'files=%s'" % modisOgg.filelist.name)) else: grass.fatal(_("Error during connection"))
def main(): try: from rmodislib import product except: grass.fatal("i.modis library is not installed") try: from pymodis.downmodis import downModis except: grass.fatal("pymodis library is not installed") # check if you are in GRASS gisbase = os.getenv("GISBASE") if not gisbase: grass.fatal(_("$GISBASE not defined")) return 0 if flags["l"]: prod = product() prod.print_prods() return 0 # empty settings and folder would collide if not options["settings"] and not options["folder"]: grass.fatal( _("With empty settings parameter (to use the .netrc file) " "the folder parameter needs to be specified")) # set username, password and folder if settings are insert by stdin if not options["settings"]: user = None passwd = None if check_folder(options["folder"]): fold = options["folder"] else: grass.fatal( _("Set folder parameter when using stdin for passing " "the username and password")) elif options["settings"] == "-": if options["folder"] != "": import getpass if check_folder(options["folder"]): fold = options["folder"] user = raw_input(_("Insert username: "******"Insert password: "******"Set folder parameter when using stdin for passing " "the username and password")) # set username, password and folder by file else: if not os.path.isfile(options["settings"]): grass.fatal( _("The settings parameter <{}> is not a file").format( options["settings"])) # open the file and read the the user and password: # first line is username # second line is password try: with open(options["settings"], "r") as filesett: fileread = filesett.readlines() user = fileread[0].strip() passwd = fileread[1].strip() except (FileNotFoundError, PermissionError) as e: grass.fatal(_("Unable to read settings: {}").format(e)) if options["folder"] != "": if check_folder(options["folder"]): fold = options["folder"] # set the folder from path where settings file is stored else: path = os.path.split(options["settings"])[0] temp = os.path.split(grass.tempfile())[0] if temp in path: grass.warning( _("You are downloading data into a temporary " "directory. They will be deleted when you " "close this GRASS GIS session")) if check_folder(path): fold = path # the product products = options["product"].split(",") # first date and delta firstday, finalday, delta = checkdate(options) # set tiles if options["tiles"] == "": tiles = None grass.warning( _("Option 'tiles' not set. Downloading all available tiles to <{}>" ).format(fold)) else: tiles = options["tiles"] # set the debug if flags["d"]: debug_opt = True else: debug_opt = False if flags["c"]: checkgdal = False else: checkgdal = True for produ in products: prod = product(produ).returned() # start modis class modisOgg = downModis( url=prod["url"], user=user, password=passwd, destinationFolder=fold, tiles=tiles, delta=delta, path=prod["folder"], product=prod["prod"], today=firstday, enddate=finalday, debug=debug_opt, checkgdal=checkgdal, ) # connect to ftp modisOgg.connect() if modisOgg.nconnection <= 20: # download tha tiles grass.message( _("Downloading MODIS product <{}> ({})...".format( produ, prod["prod"]))) modisOgg.downloadsAllDay() filesize = int(os.path.getsize(modisOgg.filelist.name)) if flags["g"] and filesize != 0: grass.message("files=%s" % modisOgg.filelist.name) elif filesize == 0: grass.message( _("No data download, probably they have been " "previously downloaded")) elif filesize != 0: grass.message( _("All data have been downloaded, continue " "with i.modis.import with the option " "'files=%s'" % modisOgg.filelist.name)) else: grass.fatal(_("Error during connection"))
def mosaic(options, remove, an, ow, fil): """Create a daily mosaic of HDF files convert to TIF and import it """ try: # try to import pymodis (modis) and some classes for i.modis.download from rmodislib import product, projection, get_proj except: grass.fatal("i.modis library is not installed") try: from pymodis.convertmodis import convertModis, createMosaic from pymodis.convertmodis_gdal import createMosaicGDAL, convertModisGDAL from pymodis.parsemodis import parseModis except: grass.fatal("pymodis library is not installed") dictfile, targetdir = list_files(options, True) pid = str(os.getpid()) # for each day for dat, listfiles in dictfile.items(): pref = listfiles[0].split('/')[-1] prod = product().fromcode(pref.split('.')[0]) spectr = spectral(options, prod, an) spectr = spectr.lstrip('( ').rstrip(' )') outname = "%s.%s_mosaic" % (pref.split('.')[0], pref.split('.')[1]) outname = outname.replace(' ', '_') # create mosaic if options['mrtpath']: # create the file with the list of name tempfile = open(os.path.join(targetdir, pid), 'w') tempfile.writelines(listfiles) tempfile.close() # basedir of tempfile, where hdf files are write basedir = os.path.split(tempfile.name)[0] # return the spectral subset in according mrtmosaic tool format cm = createMosaic(tempfile.name, outname, options['mrtpath'], spectr) cm.run() hdfiles = glob.glob1(basedir, outname + "*.hdf") else: basedir = targetdir listfiles = [os.path.join(basedir, i) for i in listfiles] cm = createMosaicGDAL(listfiles, spectr) try: cm.write_vrt(os.path.join(basedir, outname), quiet=True) except: cm.write_vrt(os.path.join(basedir, outname)) hdfiles = glob.glob1(basedir, outname + "*.vrt") for i in hdfiles: # the full path to hdf file hdf = os.path.join(basedir, i) try: pm = parseModis(hdf) except: out = i.replace('.vrt', '') data = doy2date(dat[1:]) pm = grassParseModis(out, data) # create convertModis class and convert it in tif file if options['mrtpath']: # create conf file fro mrt tools confname = confile(pm, options, an, True) execmodis = convertModis(hdf, confname, options['mrtpath']) else: confname = None projwkt = get_proj('w') projObj = projection() if projObj.returned() != 'GEO': res = int(prod['res']) * int(projObj.proj['meters']) else: res = None execmodis = convertModisGDAL(str(hdf), out, spectr, res, wkt=str(projwkt), vrt=True) try: execmodis.run(quiet=True) except: execmodis.run() # remove hdf if remove: # import tif files import_tif(basedir=basedir, rem=remove, write=ow, pm=pm, listfile=fil, prod=prod) try: os.remove(hdf) os.remove(hdf + '.xml') except OSError: pass # move the hdf and hdf.xml to the dir where are the original files else: # import tif files import_tif(basedir=basedir, rem=remove, write=ow, pm=pm, target=targetdir, listfile=fil, prod=prod) if i not in os.listdir(targetdir): try: shutil.move(hdf, targetdir) shutil.move(hdf + '.xml', targetdir) except OSError: pass # remove the conf file try: os.remove(confname) except (OSError, TypeError) as e: pass if options['mrtpath']: grass.try_remove(tempfile.name) grass.try_remove(os.path.join(targetdir, 'mosaic', pid))
def single(options, remove, an, ow, fil): """Convert the HDF file to TIF and import it """ try: # try to import pymodis (modis) and some classes for i.modis.download from rmodislib import product, projection, get_proj except: grass.fatal("i.modis library is not installed") try: from pymodis.convertmodis import convertModis from pymodis.convertmodis_gdal import convertModisGDAL from pymodis.parsemodis import parseModis except: grass.fatal("pymodis library is not installed") listfile, basedir = list_files(options) # for each file for i in listfile: if os.path.exists(i): hdf = i else: # the full path to hdf file hdf = os.path.join(basedir, i) if not os.path.exists(hdf): grass.warning(_("%s not found" % i)) continue pm = parseModis(hdf) if options['mrtpath']: # create conf file fro mrt tools confname = confile(pm, options, an) # create convertModis class and convert it in tif file execmodis = convertModis(hdf, confname, options['mrtpath']) else: projwkt = get_proj('w') projObj = projection() pref = i.split('/')[-1] prod = product().fromcode(pref.split('.')[0]) spectr = spectral(options, prod, an) if projObj.returned() != 'GEO': res = int(prod['res']) * int(projObj.proj['meters']) else: res = None outname = "%s.%s.%s.single" % ( pref.split('.')[0], pref.split('.')[1], pref.split('.')[2]) outname = outname.replace(' ', '_') execmodis = convertModisGDAL(str(hdf), outname, spectr, res, wkt=str(projwkt)) try: execmodis.run(quiet=True) except: execmodis.run() import_tif(basedir=basedir, rem=remove, write=ow, pm=pm, listfile=fil, prod=prod) if options['mrtpath']: os.remove(confname)