def readMap(fileName, fileFormat,logger,unzipcmd='pigz -d -k'):
"""
Read PCRaster geographical file into memory
"""
unzipped = 0
if not os.path.exists(fileName):
# try and unzip
if os.path.exists(fileName + ".gz"):
os.system(unzipcmd + ' ' + fileName + ".gz")
logger.info("unzipping: " + fileName + ".gz")
unzipped = 1
pcrdata = _pcrut.readmap(fileName)
x = _pcrut.pcr2numpy(_pcrut.xcoordinate(_pcrut.boolean(_pcrut.cover(1.0))),NaN)[0,:]
y = _pcrut.pcr2numpy(_pcrut.ycoordinate(_pcrut.boolean(_pcrut.cover(1.0))),NaN)[:,0]
FillVal = float(1E31)
data = _pcrut.pcr2numpy(pcrdata,FillVal)
if unzipped:
#Delete uncompressed file if compressed exsists
if os.path.exists(fileName + ".gz"):
logger.info("Removing: " + fileName)
os.remove(fileName)
return x, y, data, FillVal
def readMap(fileName, fileFormat, logger, unzipcmd='pigz -d -k'):
"""
Read PCRaster geographical file into memory
"""
unzipped = 0
if not os.path.exists(fileName):
# try and unzip
if os.path.exists(fileName + ".gz"):
os.system(unzipcmd + ' ' + fileName + ".gz")
logger.info("unzipping: " + fileName + ".gz")
unzipped = 1
pcrdata = _pcrut.readmap(fileName)
x = _pcrut.pcr2numpy(_pcrut.xcoordinate(_pcrut.boolean(_pcrut.cover(1.0))),
NaN)[0, :]
y = _pcrut.pcr2numpy(_pcrut.ycoordinate(_pcrut.boolean(_pcrut.cover(1.0))),
NaN)[:, 0]
FillVal = float(1E31)
data = _pcrut.pcr2numpy(pcrdata, FillVal)
if unzipped:
#Delete uncompressed file if compressed exsists
if os.path.exists(fileName + ".gz"):
logger.info("Removing: " + fileName)
os.remove(fileName)
return x, y, data, FillVal
def readMap(fileName, fileFormat,logger):
"""
Read geographical file into memory
"""
# Open file for binary-reading
#pcrdata = _pcrut.readmap(fileName)
# mapFormat = gdal.GetDriverByName(fileFormat)
# mapFormat.Register()
# ds = gdal.Open(fileName)
# if ds is None:
# logger.error('Could not open ' + fileName + '. Something went wrong!! Shutting down')
# sys.exit(1)
# # Retrieve geoTransform info
# geotrans = ds.GetGeoTransform()
# originX = geotrans[0]
# originY = geotrans[3]
# resX = geotrans[1]
# resY = geotrans[5]
# cols = ds.RasterXSize
# rows = ds.RasterYSize
# x = linspace(originX+resX/2,originX+resX/2+resX*(cols-1),cols)
# y = linspace(originY+resY/2,originY+resY/2+resY*(rows-1),rows)
# # Retrieve raster
# RasterBand = ds.GetRasterBand(1) # there's only 1 band, starting from 1
# data = RasterBand.ReadAsArray(0,0,cols,rows)
# FillVal = RasterBand.GetNoDataValue()
# RasterBand = None
# del ds
# #ds = None
pcrdata = _pcrut.readmap(fileName)
x = _pcrut.pcr2numpy(_pcrut.xcoordinate(_pcrut.boolean(_pcrut.cover(1.0))),NaN)[0,:]
y = _pcrut.pcr2numpy(_pcrut.ycoordinate(_pcrut.boolean(_pcrut.cover(1.0))),NaN)[:,0]
FillVal = float(1E31)
data = _pcrut.pcr2numpy(pcrdata,FillVal)
return x, y, data, FillVal
def main(argv=None):
"""
Perform command line execution of the model.
"""
# initiate metadata entries
metadata = {}
metadata["title"] = "wflow input mapstack"
metadata["institution"] = "Deltares"
metadata["source"] = "pcr2netcdf"
metadata["history"] = time.ctime()
metadata["references"] = "http://wflow.googlecode.com"
metadata["Conventions"] = "CF-1.4"
ncoutfile = "inmaps.nc"
mapstackfolder = "inmaps"
inifile = "not set"
mapstackname = []
var = []
varname = []
unit = "mm"
startstr = "1-1-1990 00:00:00"
endstr = "2-2-1990 00:00:00"
mbuf = 600
timestepsecs = 86400
outputFillVal = 1e31
clonemap = None
OFormat = "NETCDF4"
IFormat = "PCRaster"
EPSG = "EPSG:4326"
Singlemap = False
zlib = True
least_significant_digit = None
clonemapname = "None"
startstep = 1
perYear = False
if argv is None:
argv = sys.argv[1:]
if len(argv) == 0:
usage()
return
## Main model starts here
########################################################################
try:
opts, args = getopt.getopt(argv, "c:S:E:N:I:O:b:t:F:zs:d:YP:Mi:C:")
except getopt.error as msg:
usage(msg)
for o, a in opts:
if o == "-S":
startstr = a
if o == "-s":
startstep = int(a)
if o == "-E":
endstr = a
if o == "-i":
IFormat = a
if o == "-O":
ncoutfile = a
if o == "-c":
inifile = a
if o == "-I":
mapstackfolder = a
if o == "-b":
mbuf = int(a)
if o == "-Y":
perYear = True
if o == "-z":
zlib = True
if o == "-P":
EPSG = a
if o == "-M":
Singlemap = True
if o == "-F":
OFormat = a
if o == "-d":
least_significant_digit = int(a)
if o == "-C":
clonemapname = a
if o == "-t":
timestepsecs = int(a)
if o == "-N":
flst = glob.glob(a)
if len(flst) == 0:
mapstackname.append(a)
var.append(a)
varname.append(a)
else:
mapstackname = flst
var = flst
varname = flst
# Use first timestep as clone-map
logger = setlogger("pcr2netcdf.log", "pcr2netcdf", thelevel=logging.DEBUG)
count = 1
below_thousand = count % 1000
above_thousand = count / 1000
if clonemapname == "None":
clonemapname = str(
mapstackname[0] + "%0" + str(8 - len(mapstackname[0])) + ".f.%03.f"
) % (above_thousand, below_thousand)
clonemap = os.path.join(mapstackfolder, clonemapname)
if Singlemap:
clonemap = mapstackname[0]
if IFormat == "PCRaster":
_pcrut.setclone(clonemap)
x, y, clone, FillVal = _readMap(clonemap, IFormat, logger)
start = dt.datetime.strptime(startstr, "%d-%m-%Y %H:%M:%S")
if Singlemap:
end = start
else:
end = dt.datetime.strptime(endstr, "%d-%m-%Y %H:%M:%S")
if timestepsecs == 86400:
if perYear:
timeList = date_range_peryear(start, end, tdelta="days")
else:
timeList = date_range(start, end, timestepsecs)
else:
if perYear:
timeList = date_range_peryear(start, end, tdelta="hours")
else:
timeList = date_range(start, end, timestepsecs)
if os.path.exists(inifile):
inimetadata = getnetcdfmetafromini(inifile)
metadata.update(inimetadata)
# break up into separate years
if not Singlemap:
varmeta = {}
startmapstack = startstep
if perYear:
for yr_timelist in timeList:
ncoutfile_yr = (
os.path.splitext(ncoutfile)[0]
+ "_"
+ str(yr_timelist[0].year)
+ os.path.splitext(ncoutfile)[1]
)
if os.path.exists(ncoutfile_yr):
logger.info("Skipping file: " + ncoutfile_yr)
else:
ncdf.prepare_nc(
ncoutfile_yr,
yr_timelist,
x,
y,
metadata,
logger,
Format=OFormat,
zlib=zlib,
EPSG=EPSG,
FillValue=outputFillVal,
)
idx = 0
for mname in mapstackname:
logger.info(
"Converting mapstack: " + mname + " to " + ncoutfile
)
# get variable attributes from ini file here
if os.path.exists(inifile):
varmeta = getvarmetadatafromini(inifile, var[idx])
write_netcdf_timeseries(
mapstackfolder,
mname,
ncoutfile_yr,
var[idx],
unit,
varname[idx],
yr_timelist,
varmeta,
logger,
clone,
maxbuf=mbuf,
Format=OFormat,
zlib=zlib,
least_significant_digit=least_significant_digit,
startidx=startmapstack,
EPSG=EPSG,
FillVal=outputFillVal,
)
idx = idx + 1
logger.info(
"Old stack: "
+ str(startmapstack)
+ " new startpoint "
+ str(startmapstack + len(yr_timelist) - 1)
)
startmapstack = startmapstack + len(yr_timelist)
else:
# ncoutfile_yr = os.path.splitext(ncoutfile)[0] + "_" + str(yr_timelist[0].year) + os.path.splitext(ncoutfile)[1]
ncdf.prepare_nc(
ncoutfile,
timeList,
x,
y,
metadata,
logger,
Format=OFormat,
zlib=zlib,
EPSG=EPSG,
)
idx = 0
for mname in mapstackname:
logger.info("Converting mapstack: " + mname + " to " + ncoutfile)
# get variable attributes from ini file here
if os.path.exists(inifile):
varmeta = getvarmetadatafromini(inifile, var[idx])
write_netcdf_timeseries(
mapstackfolder,
mname,
ncoutfile,
var[idx],
unit,
varname[idx],
timeList,
varmeta,
logger,
clone,
maxbuf=mbuf,
Format=OFormat,
zlib=zlib,
least_significant_digit=least_significant_digit,
startidx=startmapstack,
EPSG=EPSG,
FillVal=outputFillVal,
)
idx = idx + 1
else:
NcOutput = ncdf.netcdfoutputstatic(
ncoutfile,
logger,
timeList[0],
1,
timestepsecs=timestepsecs,
maxbuf=1,
metadata=metadata,
EPSG=EPSG,
Format=OFormat,
zlib=zlib,
)
for file in mapstackname:
pcrdata = _pcrut.readmap(file)
thevar = os.path.basename(file)
NcOutput.savetimestep(1, pcrdata, unit="mm", var=thevar, name=file)
ncdf.prepare_nc(ncoutfile, timeList, x, y, metadata, logger,Format=OFormat,zlib=zlib,EPSG=EPSG)
idx = 0
for mname in mapstackname:
logger.info("Converting mapstack: " + mname + " to " + ncoutfile)
# get variable attributes from ini file here
if os.path.exists(inifile):
varmeta = getvarmetadatafromini(inifile,var[idx])
write_netcdf_timeseries(mapstackfolder, mname, ncoutfile, var[idx], unit, varname[idx], timeList, varmeta,\
logger,clone,maxbuf=mbuf,Format=OFormat,zlib=zlib,least_significant_digit=least_significant_digit,\
startidx=startmapstack,EPSG=EPSG,FillVal=outputFillVal)
idx = idx + 1
else:
NcOutput = ncdf.netcdfoutputstatic(ncoutfile, logger, timeList[0],1,timestepsecs=timestepsecs,
maxbuf=1, metadata=metadata, EPSG=EPSG,Format=OFormat,
zlib=zlib)
for file in mapstackname:
pcrdata = _pcrut.readmap(file)
thevar = os.path.basename(file)
NcOutput.savetimestep(1, pcrdata, unit="mm", var=thevar, name=file)
if __name__ == "__main__":
main()
logger.info("Converting mapstack: " + mname + " to " +
ncoutfile)
# get variable attributes from ini file here
if os.path.exists(inifile):
varmeta = getvarmetadatafromini(inifile, var[idx])
write_netcdf_timeseries(mapstackfolder, mname, ncoutfile, var[idx], unit, varname[idx], timeList, varmeta,\
logger,clone,maxbuf=mbuf,Format=OFormat,zlib=zlib,least_significant_digit=least_significant_digit,\
startidx=startmapstack,EPSG=EPSG,FillVal=outputFillVal)
idx = idx + 1
else:
NcOutput = ncdf.netcdfoutputstatic(ncoutfile,
logger,
timeList[0],
1,
timestepsecs=timestepsecs,
maxbuf=1,
metadata=metadata,
EPSG=EPSG,
Format=OFormat,
zlib=zlib)
for file in mapstackname:
pcrdata = _pcrut.readmap(file)
thevar = os.path.basename(file)
NcOutput.savetimestep(1, pcrdata, unit="mm", var=thevar, name=file)
if __name__ == "__main__":
main()
def main(argv=None):
"""
Perform command line execution of the model.
"""
# initiate metadata entries
metadata = {}
metadata["title"] = "wflow input mapstack"
metadata["institution"] = "Deltares"
metadata["source"] = "pcr2netcdf"
metadata["history"] = time.ctime()
metadata["references"] = "http://wflow.googlecode.com"
metadata["Conventions"] = "CF-1.4"
ncoutfile = "inmaps.nc"
mapstackfolder = "inmaps"
inifile = "not set"
mapstackname = []
var = []
varname = []
unit = "mm"
startstr = "1-1-1990 00:00:00"
endstr = "2-2-1990 00:00:00"
mbuf = 600
timestepsecs = 86400
outputFillVal = 1e31
clonemap = None
OFormat = "NETCDF4"
IFormat = "PCRaster"
EPSG = "EPSG:4326"
Singlemap = False
zlib = True
least_significant_digit = None
clonemapname = "None"
startstep = 1
perYear = False
if argv is None:
argv = sys.argv[1:]
if len(argv) == 0:
usage()
return
## Main model starts here
########################################################################
try:
opts, args = getopt.getopt(argv, "c:S:E:N:I:O:b:t:F:zs:d:YP:Mi:C:")
except getopt.error as msg:
usage(msg)
for o, a in opts:
if o == "-S":
startstr = a
if o == "-s":
startstep = int(a)
if o == "-E":
endstr = a
if o == "-i":
IFormat = a
if o == "-O":
ncoutfile = a
if o == "-c":
inifile = a
if o == "-I":
mapstackfolder = a
if o == "-b":
mbuf = int(a)
if o == "-Y":
perYear = True
if o == "-z":
zlib = True
if o == "-P":
EPSG = a
if o == "-M":
Singlemap = True
if o == "-F":
OFormat = a
if o == "-d":
least_significant_digit = int(a)
if o == "-C":
clonemapname = a
if o == "-t":
timestepsecs = int(a)
if o == "-N":
flst = glob.glob(a)
if len(flst) == 0:
mapstackname.append(a)
var.append(a)
varname.append(a)
else:
mapstackname = flst
var = flst
varname = flst
# Use first timestep as clone-map
logger = setlogger("pcr2netcdf.log", "pcr2netcdf", thelevel=logging.DEBUG)
count = 1
below_thousand = count % 1000
above_thousand = count / 1000
if clonemapname == "None":
clonemapname = str(
mapstackname[0] + "%0" + str(8 - len(mapstackname[0])) + ".f.%03.f"
) % (above_thousand, below_thousand)
clonemap = os.path.join(mapstackfolder, clonemapname)
if Singlemap:
clonemap = mapstackname[0]
if IFormat == "PCRaster":
_pcrut.setclone(clonemap)
x, y, clone, FillVal = _readMap(clonemap, IFormat, logger)
start = dt.datetime.strptime(startstr, "%d-%m-%Y %H:%M:%S")
if Singlemap:
end = start
else:
end = dt.datetime.strptime(endstr, "%d-%m-%Y %H:%M:%S")
if timestepsecs == 86400:
if perYear:
timeList = date_range_peryear(start, end, tdelta="days")
else:
timeList = date_range(start, end, timestepsecs)
else:
if perYear:
timeList = date_range_peryear(start, end, tdelta="hours")
else:
timeList = date_range(start, end, timestepsecs)
if os.path.exists(inifile):
inimetadata = getnetcdfmetafromini(inifile)
metadata.update(inimetadata)
# break up into separate years
if not Singlemap:
varmeta = {}
startmapstack = startstep
if perYear:
for yr_timelist in timeList:
ncoutfile_yr = (
os.path.splitext(ncoutfile)[0]
+ "_"
+ str(yr_timelist[0].year)
+ os.path.splitext(ncoutfile)[1]
)
if os.path.exists(ncoutfile_yr):
logger.info("Skipping file: " + ncoutfile_yr)
else:
ncdf.prepare_nc(
ncoutfile_yr,
yr_timelist,
x,
y,
metadata,
logger,
Format=OFormat,
zlib=zlib,
EPSG=EPSG,
FillValue=outputFillVal,
)
idx = 0
for mname in mapstackname:
logger.info(
"Converting mapstack: " + mname + " to " + ncoutfile
)
# get variable attributes from ini file here
if os.path.exists(inifile):
varmeta = getvarmetadatafromini(inifile, var[idx])
write_netcdf_timeseries(
mapstackfolder,
mname,
ncoutfile_yr,
var[idx],
unit,
varname[idx],
yr_timelist,
varmeta,
logger,
clone,
maxbuf=mbuf,
Format=OFormat,
zlib=zlib,
least_significant_digit=least_significant_digit,
startidx=startmapstack,
EPSG=EPSG,
FillVal=outputFillVal,
)
idx = idx + 1
logger.info(
"Old stack: "
+ str(startmapstack)
+ " new startpoint "
+ str(startmapstack + len(yr_timelist) - 1)
)
startmapstack = startmapstack + len(yr_timelist)
else:
# ncoutfile_yr = os.path.splitext(ncoutfile)[0] + "_" + str(yr_timelist[0].year) + os.path.splitext(ncoutfile)[1]
ncdf.prepare_nc(
ncoutfile,
timeList,
x,
y,
metadata,
logger,
Format=OFormat,
zlib=zlib,
EPSG=EPSG,
)
idx = 0
for mname in mapstackname:
logger.info("Converting mapstack: " + mname + " to " + ncoutfile)
# get variable attributes from ini file here
if os.path.exists(inifile):
varmeta = getvarmetadatafromini(inifile, var[idx])
write_netcdf_timeseries(
mapstackfolder,
mname,
ncoutfile,
var[idx],
unit,
varname[idx],
timeList,
varmeta,
logger,
clone,
maxbuf=mbuf,
Format=OFormat,
zlib=zlib,
least_significant_digit=least_significant_digit,
startidx=startmapstack,
EPSG=EPSG,
FillVal=outputFillVal,
)
idx = idx + 1
else:
NcOutput = ncdf.netcdfoutputstatic(
ncoutfile,
logger,
timeList[0],
1,
timestepsecs=timestepsecs,
maxbuf=1,
metadata=metadata,
EPSG=EPSG,
Format=OFormat,
zlib=zlib,
)
for file in mapstackname:
pcrdata = _pcrut.readmap(file)
thevar = os.path.basename(file)
NcOutput.savetimestep(1, pcrdata, unit="mm", var=thevar, name=file)