def calcAverageOfDayNight_dir(output_dir, dayDir, nightDir, patterns = (None, None)):
print "calcAverage of Day & Night for directory: ", dayDir, nightDir
#an empty array/vector in which to store the different bands
if patterns[0]:
all_dayfiles = directoryUtils.getMatchingFiles(os.path.dirname(dayDir), patterns[0])
else:
all_dayfiles = directoryUtils.buildFileList(os.path.dirname(dayDir), '.tif')
if patterns[0]:
all_nightfiles = directoryUtils.getMatchingFiles(os.path.dirname(nightDir), patterns[0])
else:
all_nightfiles = directoryUtils.buildFileList(os.path.dirname(nightDir), '.tif')
# for df in all_dayfiles:
return 0
def selectSeasonalFiles(base_path, seasonal, years, filenames):
f_base = filenames[0]
ext = filenames[1]
all_files = buildFileList(base_path, ext)
seasons = []
yrs = []
for fl in all_files:
yrs.append(getCHIRPSYear(fl))
seasons.append(getCHIRPSSeason(fl))
if not years:
# do all - get all files, work out what years are included
years = set(yrs)
if not seasonal:
# do all
seasonal = set(seasons) #['010203', '020304', '030405', '040506', '050607', '060708', '070809', '080910', '091011', '101112', '111201', '120102']
files = set(all_files)
fileList = []
for s in seasonal:
yrList = []
for y in years:
# create file
fn = path.join(base_path, "{0}.{1}.{2}{3}".format(f_base, y, s, ext))
if fn in files:
if path.isfile(fn):
yrList.append(fn)
if yrList:
yrList.sort()
t = s, yrList
fileList.append(t)
fileList.sort()
return fileList
def selectDekadFiles_Dates(base_path, start_date, end_date, filenames):
fileList = []
f_base = filenames[0]
ext = filenames[1]
all_files = buildFileList(base_path, ext)
# start_date is a list of [dekad, month, year]
# end_date is a list of [dekad, month, year]
start_m = start_date[1]
end_m = 12
# loop through range of years
for y in range(start_date[2], end_date[2]+1): #range() goes up to, but not including, the stop value
# loop from start month to end month for current year
if y == end_date[2]:
end_m = end_date[1]
for m in range(start_m, end_m+1):
start_d = 1
end_d = 3
if m == start_m and y == start_date[2]:
# start at start dekad for first year
start_d = start_date[0]
# if on last month, stop at end dekad
if m == end_m and y == end_date[2]:
end_d = end_date[0]
for d in range(start_d, end_d+1):
# create file
fn = path.join(base_path, "{0}.{1}.{2:02d}.{3}{4}".format(f_base, y, m, d, ext))
if fn in all_files:
if path.isfile(fn):
fileList.append(fn)
start_m = 1
return fileList, True
def calcMonthlyAverages(base_path, output_path, functionList = [], filenames = ('idn_cli_chirps-v2.0', '.tif')):
ext = filenames[1]
all_files = buildFileList(base_path, ext)
# do all - get all files, work out what years are included
yrs = []
for fl in all_files:
yrs.append(getCHIRPSYear(fl))
years = set(yrs)
# do all
months = ['01', '02', '03', '04', '05', '06', '07', '08', '09', '10', '11', '12']
filesList = selectMonthlyFiles(base_path, months, years, filenames)
syr = min(years) #1981
eyr = max(years)
numyrs = int(eyr) - int(syr)
for m in range(0,12):
# for each month, calculate long term average
fl = (filesList[m])[1]
# mth = fl[0].split('.')[2]
# print "Calculating long term average for ", fl
newfilename = '{0}.{1}-{2}.{3}.monthly.{4}yrs'.format(filenames[0], syr, eyr, months[m], str(numyrs))
if not functionList:
# default is to calculate the average
functionList.append('AVG')
performCalculations(fl, newfilename, output_path, functionList)
return 0
def selectMonthlyFiles(base_path, months, years, filenames):
f_base = filenames[0]
ext = filenames[1]
all_files = buildFileList(base_path, ext)
if not years:
# do all - get all files, work out what years are included
yrs = []
for fl in all_files:
yrs.append(getCHIRPSYear(fl))
years = set(yrs)
if not months:
# do all
months = ['01', '02', '03', '04', '05', '06', '07', '08', '09', '10', '11', '12']
files = set(all_files)
fileList = []
for m in months:
yrList = []
for y in years:
# create file
fn = path.join(base_path, "{0}.{1}.{2}{3}".format(f_base, y, m, ext))
if fn in files:
if path.isfile(fn):
yrList.append(fn)
yrList.sort()
mt = m, yrList
fileList.append(mt)
fileList.sort()
return fileList
def mosaicMODIS(base_path, output_path, tools_path, work_path, filenames = ('MOD13Q1', '.hdf')):
f_base = filenames[0] #'MOD13Q1'
ext = filenames[1] #'.hdf'
all_files = directoryUtils.buildFileList(base_path, ext)
if not all_files:
print 'No files found in ' + base_path + ', please check directory and try again'
return -1
param_file = os.path.join(work_path, f_base + ".prm")
files_dict = defaultdict(list)
for fl in all_files:
if fl.endswith(ext):
# add file to appropriate place in the dictionary
yr_doy = getMODISYrAndDoY(fl)
files_dict[yr_doy].append(fl)
for i,v in files_dict.iteritems():
# create a list file for mosaic for each day
file_list = os.path.join(work_path, f_base + "_" + i + ".lst")
writeMosaicList(file_list, v)
mosaicname = os.path.join(output_path, '{0}.{1}.005_m{2}'.format(f_base, i, ext))
# mosaic files for this day
if mosaicFiles(file_list, mosaicname, tools_path) == -1:
print 'Error generating mosaic ' + mosaicname
else:
outputname = os.path.join(output_path, '{0}.{1}.005{2}'.format(f_base, i, ext))
generateParamFile(output_path, param_file, mosaicname, outputname)
reprojectMosaic(param_file, tools_path)
# remove temp file
os.remove(mosaicname)
return 0
def tranformToWGS84(base_path, output_path, tools_path, filenames = ('MOD13Q1', '.tif'), out_proj = 'EPSG:4326', projection_text = 'GEOGCS["WGS 84",DATUM["WGS_1984",SPHEROID["WGS 84",6378137,298.257223563,AUTHORITY["EPSG","7030"]],AUTHORITY["EPSG","6326"]],PRIMEM["Greenwich",0],UNIT["degree",0.0174532925199433],AUTHORITY["EPSG","4326"]]'):
f_base = filenames[0] #'MOD13Q1'
ext = filenames[1] #'.tif'
all_files = directoryUtils.buildFileList(base_path, ext)
if not all_files:
print 'No files found in ' + base_path + ', please check directory and try again'
return -1
for ifl in all_files:
print "checking transform of {0}".format(ifl)
raster = gdal.Open(ifl)
projection = raster.GetProjection()
print projection
if projection != projection_text:
# transform
try:
ofl = os.path.join(output_path, ifl)
pf = platform.system()
if 'Windows' in pf:
check_call([tools_path + 'gdalwarp.exe', '-t_srs', out_proj, ifl, ofl])
elif 'Linux' in pf:
check_call([tools_path + 'gdalwarp', '-t_srs', out_proj, ifl, ofl])
except CalledProcessError as e:
print("Error in change projection")
print(e.output)
raise
return 0
def getYearsList(base_path, ext, yrs = []):
all_files = buildFileList(base_path, ext)
# do all - get all files, work out what years are included
y = []
for fl in all_files:
fl_yr = getCHIRPSYear(fl)
if yrs:
if int(fl_yr) >= yrs[0] and int(fl_yr) <= yrs[1]:
y.append(fl_yr)
else:
y.append(fl_yr)
years = set(y)
return years
def selectDailyFiles(base_path, start_date, numdays, filenames):
f_base = filenames[0]
ext = filenames[1]
all_files = buildFileList(base_path, ext)
base = start_date
date_list = [base - datetime.timedelta(days=x) for x in range(0, numdays)]
files = set(all_files)
fileList = []
for d in date_list:
# create filename
fn = path.join(base_path, "{0}.{1}{2}".format(f_base, d.strftime("%Y.%m.%d"), ext))
if fn in files:
if path.isfile(fn):
fileList.append(fn)
return fileList
def selectPentadFiles(base_path, pentad, months, years, filenames):
f_base = filenames[0]
ext = filenames[1]
all_files = buildFileList(base_path, ext)
if not years:
# do all - get all files, work out what years are included
yrs = []
for fl in all_files:
yrs.append(getCHIRPSYear(fl))
years = set(yrs)
if not months:
# do all
months = ['01', '02', '03', '04', '05', '06', '07', '08', '09', '10', '11', '12']
if not pentad:
# do all pentads
pentad = [1, 2, 3, 4, 5, 6]
# f_base = 'idn_cli_chirps-v2.0.'
files = set(all_files)
fileList = []
for m in months:
mList = []
for p in pentad:
yrList = []
for y in years:
# create file
fn = "{0}{1}.{2}.{3}.{4}{5}".format(base_path, f_base, y, m, p, ext)
if fn in files:
if path.isfile(fn):
yrList.append(fn)
yrList.sort()
t = p, yrList
mList.append(t)
mList.sort()
mt = m, mList
fileList.append(mt)
fileList.sort()
return fileList
