def calcAverageOfDayNight_filter(dayDir, nightDir, outDir, patterns, logger=None):
# find file from pattern
day_pattern = patterns[0]
night_pattern = patterns[1]
average_pattern = patterns[2]
day_list = directoryUtils.getMatchingFiles(base_dir=dayDir, filter=day_pattern)
night_list = directoryUtils.getMatchingFiles(base_dir=nightDir, filter=night_pattern)
if logger:
logger.debug("Day files: %s", day_list)
logger.debug("Night files: %s", night_list)
for fl in day_list:
# find matching night file
# find date stamp in day file
_r_in = re.compile(day_pattern)
_r_in_night = re.compile(night_pattern)
f = os.path.basename(fl)
_m = _r_in.match(f)
year = 0
dayofyear = 0
if 'year' in _m.groupdict():
year = _m.groupdict()['year']
if 'dayofyear' in _m.groupdict():
dayofyear = _m.groupdict()['dayofyear']
for nfl in night_list:
_mn = _r_in_night.match(os.path.basename(nfl))
if 'year' in _mn.groupdict() and _mn.groupdict()['year'] == year:
# matched year
if 'dayofyear' in _mn.groupdict() and _mn.groupdict()['dayofyear'] == dayofyear:
# matched day of year
avg_fl = filenameUtils.generateOutputFilename(os.path.basename(fl), day_pattern, average_pattern)
calcAverageOfDayNight(fl, nfl, os.path.join(outDir, avg_fl))
break
return 0
def matchDayNightFiles(dayPath, nightPath, outPath, patterns = (None, None), open_src = False, gdal_path=None):
# dayFiles = list(os.listdir(dayPath))
nightFiles = set(os.listdir(nightPath))
if patterns[0]:
if not os.path.isdir(dayPath):
dayFiles = directoryUtils.getMatchingFiles(os.path.dirname(dayPath), patterns[0])
else:
dayFiles = directoryUtils.getMatchingFiles(dayPath, patterns[0])
else:
dayFiles = list(os.listdir(dayPath))
print "Day files: ", dayFiles
print "Night files: ", nightFiles
for fl in dayFiles:
# find matching night file
d_fl, ext = os.path.splitext(os.path.basename(os.path.normpath(fl)))
if (ext == '.tif'):
d_t = d_fl.rpartition('.')
n_fl = d_t[0] + d_t[1] + 'LST_Night_CMG' + ext
if (n_fl) in nightFiles:
avg_fl = os.path.join(outPath, d_t[0] + d_t[1] + 'avg' + ext)
dp = os.path.join(dayPath, d_fl+ext)
np = os.path.join(nightPath, n_fl)
if open_src:
calcAverageOfDayNight_os(dp, np, avg_fl, gdal_path)
else:
calcAverageOfDayNight(dp, np, avg_fl)
return 0
def calcAverageOfDayNight_dir(output_dir, dayDir, nightDir, patterns = (None, None)):
print "calcAverage of Day & Night for directory: ", dayDir, nightDir
#an empty array/vector in which to store the different bands
if patterns[0]:
all_dayfiles = directoryUtils.getMatchingFiles(os.path.dirname(dayDir), patterns[0])
else:
all_dayfiles = directoryUtils.buildFileList(os.path.dirname(dayDir), '.tif')
if patterns[0]:
all_nightfiles = directoryUtils.getMatchingFiles(os.path.dirname(nightDir), patterns[0])
else:
all_nightfiles = directoryUtils.buildFileList(os.path.dirname(nightDir), '.tif')
# for df in all_dayfiles:
return 0
def extractSubset(base_path, output_path, tools_path,
patterns, subset, subset_name, overwrite = False, nodata=None, logger = None):
_all_files = directoryUtils.getMatchingFiles(base_path, patterns[0])
if not _all_files:
print 'No files found in ' + base_path + ', please check directory and try again'
return -1
# _pfl = os.path.join(base_path, 'fileslist' + ".prm")
new_files = []
# _params = [subset]
for _ifl in _all_files:
# generate parameter file
_nfl = filenameUtils.generateOutputFilename(os.path.basename(_ifl), patterns[0], patterns[1])
_ofl = os.path.join(output_path, _nfl)
_checkfl = "{0}.{1}{2}".format(os.path.splitext(_ofl)[0], subset_name, os.path.splitext(_ofl)[1])
if not os.path.exists(_checkfl) or overwrite == True:
try:
src_ds = gdal.Open(_ifl)
except RuntimeError, e:
if logger: logger.debug('Unable to open file')
return None
sds = src_ds.GetSubDatasets()
if logger: logger.debug("Number of bands: %s",src_ds.RasterCount)
# generateParamFile(base_path, _pfl, _ifl, _ofl, _params)
# reprojectMosaic(_pfl, tools_path)
convertToTiff(_ifl, _ofl, tools_path, nodata)
# ss = [1]
for idx, sbs in enumerate(sds): #range(src_ds.RasterCount):
if logger: logger.debug("Subdataset: %s", sbs[0])
# get subset name (without spaces)
_n = (sbs[0].rsplit(':', 1)[1]).replace(' ', '_')
# _n = _n.replace(' ', '_')
_rf = "{0}.{1}{2}".format(os.path.splitext(os.path.basename(_ofl))[0], _n, os.path.splitext(_ofl)[1])
# _cf = "{0}_{1}{2}".format(os.path.splitext(os.path.basename(_ofl))[0], str(idx+1).zfill(2), os.path.splitext(_ofl)[1])
# GDAL 1.10
_cf = "{0}{1}{2}".format(os.path.splitext(os.path.basename(_ofl))[0], os.path.splitext(_ofl)[1], str(idx+1).zfill(1))
if not os.path.exists(os.path.join(output_path, _cf)):
_cf = "{0}_{1}{2}".format(os.path.splitext(os.path.basename(_ofl))[0], str(idx+1), os.path.splitext(_ofl)[1])
if idx+1 not in subset:
# remove un-needed files (including .aux & .aux.xml)
os.remove(os.path.join(output_path, _cf))
_aux_f = os.path.join(output_path,"{0}.aux.xml".format(_cf))
if os.path.exists(_aux_f):
os.remove(_aux_f)
# for i in range(2,18):
# rf = "{0}_{1}{2}".format(os.path.splitext(os.path.basename(_ofl))[0], str(i).zfill(2), os.path.splitext(_ofl)[1])
# os.remove(os.path.join(output_path, rf))
else:
# keep this file - rename with subset name
os.rename(os.path.join(output_path, _cf), os.path.join(output_path, _rf))
_aux_f = os.path.join(output_path,"{0}.aux.xml".format(_cf))
if os.path.exists(_aux_f):
os.rename(_aux_f, os.path.join(output_path, "{0}.aux.xml".format(_rf)))
new_files.append(_rf)
def reprojectMODISFile(base_path, outWks, patterns, toolDir):
# reproject mosaic to GEO projection
if not patterns:
patterns = (modis_patterns['reproject_in'], modis_patterns['reproject_out'])
_all_files = directoryUtils.getMatchingFiles(base_path, patterns[0])
if not _all_files:
print 'No files found in ' + base_path + ', please check directory and try again'
return -1
for f in _all_files:
param_file = os.path.join(outWks, os.path.basename(f) + ".prm")
outputname = os.path.join(outWks, '{0}_r{1}'.format(os.path.basename(f), os.path.splitext(f)[1]))
generateParamFile(outWks, param_file, f, outputname)
reprojectMosaic(param_file, os.path.normpath(toolDir))
# remove temp file
os.remove(os.path.normpath(os.path.join(outWks, f)))
os.rename(outputname, os.path.normpath(os.path.join(outWks, f)))
os.remove(param_file)
def calcLongTermAverageTemp(base_path, output_path, functionList = [], filenames = ('idn_cli_MOD11C3', '.tif'), pattern = None):
# png_cli_MOD11C3.A2000061.005.2007177231646.avg
ext = filenames[1]
all_files = directoryUtils.getMatchingFiles(base_path, pattern)
# all_files = directoryUtils.buildFileList(base_path, ext)
# do all - get all files, work out what years are included
yrs = []
days = []
daysOfYear = defaultdict(list)
for fl in all_files:
# fl includes directory, get just filename
fn = os.path.basename(fl)
ydoy = getMODISYrAndDoY(fn)
y = int(ydoy[1:5])
d = int(ydoy[-3:])
if y%4 == 0 and d>60: # leap year and day after Feb 29, subtract one from day
d = d-1
daysOfYear[str("{0:03d}".format(d))].append(fl)
yrs.append(y)
years = set(yrs)
syr = min(years) #1981
eyr = max(years)
numyrs = eyr - syr
for dd in daysOfYear.keys():
# newfilename = '{0}.{1}-{2}.{3}.{4}yrs'.format(filenames[0], syr, eyr, dd, str(numyrs))
mth = getMonthFromDayOfYear(int(dd), 2001)
newfilename = '{0}.{1}-{2}.{3}.{4}yrs'.format(filenames[0], syr, eyr, mth, str(numyrs))
if not functionList:
# default is to calculate the minimum and maximum
functionList.append('MIN')
functionList.append('MAX')
performCalculations(daysOfYear[dd], newfilename, output_path, functionList)
return 0
