def calcAverageOfDayNight_filter(dayDir, nightDir, outDir, patterns, logger=None): # find file from pattern day_pattern = patterns[0] night_pattern = patterns[1] average_pattern = patterns[2] day_list = directoryUtils.getMatchingFiles(base_dir=dayDir, filter=day_pattern) night_list = directoryUtils.getMatchingFiles(base_dir=nightDir, filter=night_pattern) if logger: logger.debug("Day files: %s", day_list) logger.debug("Night files: %s", night_list) for fl in day_list: # find matching night file # find date stamp in day file _r_in = re.compile(day_pattern) _r_in_night = re.compile(night_pattern) f = os.path.basename(fl) _m = _r_in.match(f) year = 0 dayofyear = 0 if 'year' in _m.groupdict(): year = _m.groupdict()['year'] if 'dayofyear' in _m.groupdict(): dayofyear = _m.groupdict()['dayofyear'] for nfl in night_list: _mn = _r_in_night.match(os.path.basename(nfl)) if 'year' in _mn.groupdict() and _mn.groupdict()['year'] == year: # matched year if 'dayofyear' in _mn.groupdict() and _mn.groupdict()['dayofyear'] == dayofyear: # matched day of year avg_fl = filenameUtils.generateOutputFilename(os.path.basename(fl), day_pattern, average_pattern) calcAverageOfDayNight(fl, nfl, os.path.join(outDir, avg_fl)) break return 0
def matchDayNightFiles(dayPath, nightPath, outPath, patterns = (None, None), open_src = False, gdal_path=None): # dayFiles = list(os.listdir(dayPath)) nightFiles = set(os.listdir(nightPath)) if patterns[0]: if not os.path.isdir(dayPath): dayFiles = directoryUtils.getMatchingFiles(os.path.dirname(dayPath), patterns[0]) else: dayFiles = directoryUtils.getMatchingFiles(dayPath, patterns[0]) else: dayFiles = list(os.listdir(dayPath)) print "Day files: ", dayFiles print "Night files: ", nightFiles for fl in dayFiles: # find matching night file d_fl, ext = os.path.splitext(os.path.basename(os.path.normpath(fl))) if (ext == '.tif'): d_t = d_fl.rpartition('.') n_fl = d_t[0] + d_t[1] + 'LST_Night_CMG' + ext if (n_fl) in nightFiles: avg_fl = os.path.join(outPath, d_t[0] + d_t[1] + 'avg' + ext) dp = os.path.join(dayPath, d_fl+ext) np = os.path.join(nightPath, n_fl) if open_src: calcAverageOfDayNight_os(dp, np, avg_fl, gdal_path) else: calcAverageOfDayNight(dp, np, avg_fl) return 0
def calcAverageOfDayNight_dir(output_dir, dayDir, nightDir, patterns = (None, None)): print "calcAverage of Day & Night for directory: ", dayDir, nightDir #an empty array/vector in which to store the different bands if patterns[0]: all_dayfiles = directoryUtils.getMatchingFiles(os.path.dirname(dayDir), patterns[0]) else: all_dayfiles = directoryUtils.buildFileList(os.path.dirname(dayDir), '.tif') if patterns[0]: all_nightfiles = directoryUtils.getMatchingFiles(os.path.dirname(nightDir), patterns[0]) else: all_nightfiles = directoryUtils.buildFileList(os.path.dirname(nightDir), '.tif') # for df in all_dayfiles: return 0
def extractSubset(base_path, output_path, tools_path, patterns, subset, subset_name, overwrite = False, nodata=None, logger = None): _all_files = directoryUtils.getMatchingFiles(base_path, patterns[0]) if not _all_files: print 'No files found in ' + base_path + ', please check directory and try again' return -1 # _pfl = os.path.join(base_path, 'fileslist' + ".prm") new_files = [] # _params = [subset] for _ifl in _all_files: # generate parameter file _nfl = filenameUtils.generateOutputFilename(os.path.basename(_ifl), patterns[0], patterns[1]) _ofl = os.path.join(output_path, _nfl) _checkfl = "{0}.{1}{2}".format(os.path.splitext(_ofl)[0], subset_name, os.path.splitext(_ofl)[1]) if not os.path.exists(_checkfl) or overwrite == True: try: src_ds = gdal.Open(_ifl) except RuntimeError, e: if logger: logger.debug('Unable to open file') return None sds = src_ds.GetSubDatasets() if logger: logger.debug("Number of bands: %s",src_ds.RasterCount) # generateParamFile(base_path, _pfl, _ifl, _ofl, _params) # reprojectMosaic(_pfl, tools_path) convertToTiff(_ifl, _ofl, tools_path, nodata) # ss = [1] for idx, sbs in enumerate(sds): #range(src_ds.RasterCount): if logger: logger.debug("Subdataset: %s", sbs[0]) # get subset name (without spaces) _n = (sbs[0].rsplit(':', 1)[1]).replace(' ', '_') # _n = _n.replace(' ', '_') _rf = "{0}.{1}{2}".format(os.path.splitext(os.path.basename(_ofl))[0], _n, os.path.splitext(_ofl)[1]) # _cf = "{0}_{1}{2}".format(os.path.splitext(os.path.basename(_ofl))[0], str(idx+1).zfill(2), os.path.splitext(_ofl)[1]) # GDAL 1.10 _cf = "{0}{1}{2}".format(os.path.splitext(os.path.basename(_ofl))[0], os.path.splitext(_ofl)[1], str(idx+1).zfill(1)) if not os.path.exists(os.path.join(output_path, _cf)): _cf = "{0}_{1}{2}".format(os.path.splitext(os.path.basename(_ofl))[0], str(idx+1), os.path.splitext(_ofl)[1]) if idx+1 not in subset: # remove un-needed files (including .aux & .aux.xml) os.remove(os.path.join(output_path, _cf)) _aux_f = os.path.join(output_path,"{0}.aux.xml".format(_cf)) if os.path.exists(_aux_f): os.remove(_aux_f) # for i in range(2,18): # rf = "{0}_{1}{2}".format(os.path.splitext(os.path.basename(_ofl))[0], str(i).zfill(2), os.path.splitext(_ofl)[1]) # os.remove(os.path.join(output_path, rf)) else: # keep this file - rename with subset name os.rename(os.path.join(output_path, _cf), os.path.join(output_path, _rf)) _aux_f = os.path.join(output_path,"{0}.aux.xml".format(_cf)) if os.path.exists(_aux_f): os.rename(_aux_f, os.path.join(output_path, "{0}.aux.xml".format(_rf))) new_files.append(_rf)
def reprojectMODISFile(base_path, outWks, patterns, toolDir): # reproject mosaic to GEO projection if not patterns: patterns = (modis_patterns['reproject_in'], modis_patterns['reproject_out']) _all_files = directoryUtils.getMatchingFiles(base_path, patterns[0]) if not _all_files: print 'No files found in ' + base_path + ', please check directory and try again' return -1 for f in _all_files: param_file = os.path.join(outWks, os.path.basename(f) + ".prm") outputname = os.path.join(outWks, '{0}_r{1}'.format(os.path.basename(f), os.path.splitext(f)[1])) generateParamFile(outWks, param_file, f, outputname) reprojectMosaic(param_file, os.path.normpath(toolDir)) # remove temp file os.remove(os.path.normpath(os.path.join(outWks, f))) os.rename(outputname, os.path.normpath(os.path.join(outWks, f))) os.remove(param_file)
def calcLongTermAverageTemp(base_path, output_path, functionList = [], filenames = ('idn_cli_MOD11C3', '.tif'), pattern = None): # png_cli_MOD11C3.A2000061.005.2007177231646.avg ext = filenames[1] all_files = directoryUtils.getMatchingFiles(base_path, pattern) # all_files = directoryUtils.buildFileList(base_path, ext) # do all - get all files, work out what years are included yrs = [] days = [] daysOfYear = defaultdict(list) for fl in all_files: # fl includes directory, get just filename fn = os.path.basename(fl) ydoy = getMODISYrAndDoY(fn) y = int(ydoy[1:5]) d = int(ydoy[-3:]) if y%4 == 0 and d>60: # leap year and day after Feb 29, subtract one from day d = d-1 daysOfYear[str("{0:03d}".format(d))].append(fl) yrs.append(y) years = set(yrs) syr = min(years) #1981 eyr = max(years) numyrs = eyr - syr for dd in daysOfYear.keys(): # newfilename = '{0}.{1}-{2}.{3}.{4}yrs'.format(filenames[0], syr, eyr, dd, str(numyrs)) mth = getMonthFromDayOfYear(int(dd), 2001) newfilename = '{0}.{1}-{2}.{3}.{4}yrs'.format(filenames[0], syr, eyr, mth, str(numyrs)) if not functionList: # default is to calculate the minimum and maximum functionList.append('MIN') functionList.append('MAX') performCalculations(daysOfYear[dd], newfilename, output_path, functionList) return 0