def calcAverageOfDayNight_filter(dayDir, nightDir, outDir, patterns, logger=None):
# find file from pattern
day_pattern = patterns[0]
night_pattern = patterns[1]
average_pattern = patterns[2]
day_list = directoryUtils.getMatchingFiles(base_dir=dayDir, filter=day_pattern)
night_list = directoryUtils.getMatchingFiles(base_dir=nightDir, filter=night_pattern)
if logger:
logger.debug("Day files: %s", day_list)
logger.debug("Night files: %s", night_list)
for fl in day_list:
# find matching night file
# find date stamp in day file
_r_in = re.compile(day_pattern)
_r_in_night = re.compile(night_pattern)
f = os.path.basename(fl)
_m = _r_in.match(f)
year = 0
dayofyear = 0
if 'year' in _m.groupdict():
year = _m.groupdict()['year']
if 'dayofyear' in _m.groupdict():
dayofyear = _m.groupdict()['dayofyear']
for nfl in night_list:
_mn = _r_in_night.match(os.path.basename(nfl))
if 'year' in _mn.groupdict() and _mn.groupdict()['year'] == year:
# matched year
if 'dayofyear' in _mn.groupdict() and _mn.groupdict()['dayofyear'] == dayofyear:
# matched day of year
avg_fl = filenameUtils.generateOutputFilename(os.path.basename(fl), day_pattern, average_pattern)
calcAverageOfDayNight(fl, nfl, os.path.join(outDir, avg_fl))
break
return 0
def mosaicTiles(files, output_dir, tools_dir="", overwrite = False, subset=[1,1,0,0,0,0,0,0,0,0,0], ofmt='HDF4Image',
gdal=False, logger = None):
# use MRTools
if tools_dir:
mrtpath = tools_dir
else:
mrtpath = "c:\\MODIS\\bin\\"
filelist = os.path.join(output_dir, "file_list.txt")
writeMosaicList(filelist, files)
new_filename = filenameUtils.generateOutputFilename(os.path.basename(files[0]),
modis_patterns['mosaic_in'],
modis_patterns['mosaic_out'], False)
if not os.path.exists(os.path.normpath(os.path.join(output_dir, new_filename))) or overwrite == True:
mosaicFiles(os.path.normpath(filelist), os.path.normpath(os.path.join(output_dir, new_filename)), os.path.normpath(mrtpath))
# reproject mosaic to GEO projection
param_file = os.path.join(output_dir, os.path.basename(new_filename) + ".prm")
outputname = os.path.join(output_dir, '{0}_r{1}'.format(os.path.basename(new_filename), os.path.splitext(new_filename)[1]))
generateParamFile(output_dir, param_file, os.path.normpath(os.path.join(output_dir, new_filename)), outputname)
reprojectMosaic(param_file, os.path.normpath(mrtpath))
# remove temp file
os.remove(os.path.normpath(os.path.join(output_dir, new_filename)))
os.rename(outputname, os.path.normpath(os.path.join(output_dir, new_filename)))
os.remove(param_file)
os.remove(filelist)
if logger: logger.debug("finished mosaic")
return new_filename
def extractSubset(base_path, output_path, tools_path,
patterns, subset, subset_name, overwrite = False, nodata=None, logger = None):
_all_files = directoryUtils.getMatchingFiles(base_path, patterns[0])
if not _all_files:
print 'No files found in ' + base_path + ', please check directory and try again'
return -1
# _pfl = os.path.join(base_path, 'fileslist' + ".prm")
new_files = []
# _params = [subset]
for _ifl in _all_files:
# generate parameter file
_nfl = filenameUtils.generateOutputFilename(os.path.basename(_ifl), patterns[0], patterns[1])
_ofl = os.path.join(output_path, _nfl)
_checkfl = "{0}.{1}{2}".format(os.path.splitext(_ofl)[0], subset_name, os.path.splitext(_ofl)[1])
if not os.path.exists(_checkfl) or overwrite == True:
try:
src_ds = gdal.Open(_ifl)
except RuntimeError, e:
if logger: logger.debug('Unable to open file')
return None
sds = src_ds.GetSubDatasets()
if logger: logger.debug("Number of bands: %s",src_ds.RasterCount)
# generateParamFile(base_path, _pfl, _ifl, _ofl, _params)
# reprojectMosaic(_pfl, tools_path)
convertToTiff(_ifl, _ofl, tools_path, nodata)
# ss = [1]
for idx, sbs in enumerate(sds): #range(src_ds.RasterCount):
if logger: logger.debug("Subdataset: %s", sbs[0])
# get subset name (without spaces)
_n = (sbs[0].rsplit(':', 1)[1]).replace(' ', '_')
# _n = _n.replace(' ', '_')
_rf = "{0}.{1}{2}".format(os.path.splitext(os.path.basename(_ofl))[0], _n, os.path.splitext(_ofl)[1])
# _cf = "{0}_{1}{2}".format(os.path.splitext(os.path.basename(_ofl))[0], str(idx+1).zfill(2), os.path.splitext(_ofl)[1])
# GDAL 1.10
_cf = "{0}{1}{2}".format(os.path.splitext(os.path.basename(_ofl))[0], os.path.splitext(_ofl)[1], str(idx+1).zfill(1))
if not os.path.exists(os.path.join(output_path, _cf)):
_cf = "{0}_{1}{2}".format(os.path.splitext(os.path.basename(_ofl))[0], str(idx+1), os.path.splitext(_ofl)[1])
if idx+1 not in subset:
# remove un-needed files (including .aux & .aux.xml)
os.remove(os.path.join(output_path, _cf))
_aux_f = os.path.join(output_path,"{0}.aux.xml".format(_cf))
if os.path.exists(_aux_f):
os.remove(_aux_f)
# for i in range(2,18):
# rf = "{0}_{1}{2}".format(os.path.splitext(os.path.basename(_ofl))[0], str(i).zfill(2), os.path.splitext(_ofl)[1])
# os.remove(os.path.join(output_path, rf))
else:
# keep this file - rename with subset name
os.rename(os.path.join(output_path, _cf), os.path.join(output_path, _rf))
_aux_f = os.path.join(output_path,"{0}.aux.xml".format(_cf))
if os.path.exists(_aux_f):
os.rename(_aux_f, os.path.join(output_path, "{0}.aux.xml".format(_rf)))
new_files.append(_rf)
def mosaicTiles(files, output_dir, tools_dir="", overwrite = False, subset=[1,1,0,0,0,0,0,0,0,0,0], ofmt='HDF4Image',
gdal=False, logger = None):
# from pymodis.convertmodis_gdal import createMosaicGDAL
# from pymodis.convertmodis import createMosaic
# # NDVI, EVI
# if gdal:
# mosaic = createMosaicGDAL(hdfnames=files, subset=subset, outformat=ofmt)
# ofn = os.path.basename(output_prefix + 'mosaic.hdf')
# output_file = os.path.join(output_dir, ofn)
# print "mosaic ", files, ", output: ", output_file
# mosaic.run(output_file)
# else:
# use MRTools
if tools_dir:
mrtpath = tools_dir
else:
mrtpath = "c:\\MODIS\\bin\\"
filelist = os.path.join(output_dir, "file_list.txt")
writeMosaicList(filelist, files)
# try:
# pfile = open(filelist, 'w')
# except IOError as e:
# if e.errno == errno.EACCES:
# return "Couldn't write list of files"
# # Not a permission error.
# raise
# else:
# with pfile:
# for f in files:
# pfile.write('"' + f + '"\n')
# pfile.close()
new_filename = filenameUtils.generateOutputFilename(os.path.basename(files[0]), modis_patterns['mosaic_in'], modis_patterns['mosaic_out'])
if not os.path.exists(os.path.normpath(os.path.join(output_dir, new_filename))) or overwrite == True:
mosaicFiles(os.path.normpath(filelist), os.path.normpath(os.path.join(output_dir, new_filename)), os.path.normpath(mrtpath))
# mosaicFiles(os.path.normpath(filelist), os.path.normpath(os.path.join(output_dir, output_prefix + 'mosaic.hdf')), os.path.normpath(mrtpath))
# mosaic = createMosaic(filelist, output_prefix + 'mosaic', mrtpath, [1,1,0,0,0,0,0,0,0,0,0])
# orig_dir = os.getcwd()
# print "original directory: ", orig_dir
# os.chdir(output_dir)
# mosaic.run()
os.remove(filelist)
# os.chdir(orig_dir)
if logger: logger.debug("finished mosaic")
return new_filename