def do_test():
environ = {
'wsgi.url_scheme': 'http',
'REQUEST_METHOD': 'GET', 'PATH_INFO': '/', 'SERVER_PROTOCOL': 'HTTP/1.1', 'HTTP_HOST': 'localhost:8081',
'QUERY_STRING':
'layers=ecmwf_EUR_LL015.PLDiv01&styles=&elevation=200&crs=EPSG%3A4326&format=image%2Fpng&'
'request=GetMap&bgcolor=0xFFFFFF&height=376&dim_init_time=2012-10-17T12%3A00%3A00Z&width=479&'
'version=1.3.0&bbox=20.0%2C-50.0%2C75.0%2C20.0&time=2012-10-17T12%3A00%3A00Z&'
'exceptions=XML&transparent=FALSE'}
pl_file = next(file for file in os.listdir(DATA_DIR) if ".pl" in file)
self.client = mswms.application.test_client()
result = self.client.get('/?{}'.format(environ["QUERY_STRING"]))
# Assert modified file was reloaded and now looks different
nco = Nco()
nco.ncap2(input=os.path.join(DATA_DIR, pl_file), output=os.path.join(DATA_DIR, pl_file),
options=["-s \"geopotential_height*=2\""])
result2 = self.client.get('/?{}'.format(environ["QUERY_STRING"]))
nco.ncap2(input=os.path.join(DATA_DIR, pl_file), output=os.path.join(DATA_DIR, pl_file),
options=["-s \"geopotential_height/=2\""])
assert result.data != result2.data
# Assert moved file was reloaded and now looks like the first image
move(os.path.join(DATA_DIR, pl_file), os.path.join(DATA_DIR, pl_file + "2"))
result3 = self.client.get('/?{}'.format(environ["QUERY_STRING"]))
move(os.path.join(DATA_DIR, pl_file + "2"), os.path.join(DATA_DIR, pl_file))
assert result.data == result3.data
def narr_enhance(output_dir):
outfile = output_dir.parent / 'narr.nc'
outtmp = output_dir.parent / 'narr_tmp.nc'
# remove unecessary vars
nco = Nco()
options = ['-O', '-C', '-x', '-v climatology_bounds,valid_yr_count']
nco.ncks(input=str(outfile), output=str(outfile), options=options)
# change lon from [0, 360) to [-180, 180)
options = [
'-O',
'--msa',
'-d lon,181.,360.',
'-d lon,0.,180.0',
]
nco.ncks(input=str(outfile), output=str(outtmp), options=options)
options = ['-O', "-s 'where(lon > 180) lon=lon-360'"]
nco.ncap2(input=str(outtmp), output=str(outfile), options=options)
# times don't work with xarray, so the year is changed to the middle of climatology
# climatology: 1981-01-01 - 2010-12-31
# midyear is 1996
time_range = xr.cftime_range(start='1996', end='1996-12-31', freq='MS')
new_units = 'days since 1970-01-01 00:00:00'
new_times = nc.date2num(time_range, units=new_units)
with nc.Dataset(outfile, 'a') as ncd:
time = ncd.variables['time']
time[:] = new_times
time.units = new_units
def run(self, latidx, lonidx):
try:
inputfile_dir = self.config.get_dict(self.translator_type,
'inputfile_dir',
default=os.path.join(
'..', '..'))
inputfiles = self.config.get_dict(self.translator_type,
'inputfile',
default='1.soil.tile.nc4')
inputfiles = param_to_list(inputfiles)
latdelta, londelta = [
double(d) / 60 for d in self.config.get('delta').split(',')
] # convert to degrees
nco = Nco()
outputfiles = self.config.get_dict(
self.translator_type,
'outputfile',
default=inputs_to_outputs(inputfiles))
outputfiles = param_to_list(outputfiles)
inputfiles = apply_prefix(inputfiles, inputfile_dir)
for i in range(len(inputfiles)):
inputfile = inputfiles[i]
outputfile = outputfiles[i]
with nc(inputfile) as f:
variables = f.variables.keys()
soil_id = f.getncattr('soil_id')
# get latitude, longitude limits
minlat = 90 - latdelta * latidx
maxlat = minlat + latdelta
minlon = -180 + londelta * (lonidx - 1)
maxlon = minlon + londelta
# additional options
options = '-h -a lat,lon -d lat,%f,%f -d lon,%f,%f --rd' % (
minlat, maxlat, minlon, maxlon)
if 'cropland' in variables:
options += ' -w cropland'
# perform aggregation
nco.ncwa(input=inputfile, output=outputfile, options=options)
# add degenerate profile dimension
nco.ncecat(input=outputfile,
output=outputfile,
options='-O -h -u profile')
nco.ncap2(input=outputfile,
output=outputfile,
options='-O -h -s profile[profile]=1')
# add soil_id variable
nco.ncap2(input=outputfile,
output=outputfile,
options='-O -h -s soil_id[profile,lat,lon]=1')
nco.ncatted(input=outputfile,
output=outputfile,
options='-O -h -a units,soil_id,c,c,"mapping"')
nco.ncatted(input=outputfile,
output=outputfile,
options='-O -h -a long_name,soil_id,c,c,"%s"' %
str(soil_id))
nco.ncatted(input=outputfile,
output=outputfile,
options='-O -h -a soil_id,global,d,c,""')
# change latitude, longitude to simulated point
with nc(outputfile, 'a') as f:
latv = f.variables['lat']
latv[:] = 90 - latdelta * (latidx - 0.5)
lonv = f.variables['lon']
lonv[:] = -180 + londelta * (lonidx - 0.5)
return True
except:
print "[%s]: %s" % (os.path.basename(__file__),
traceback.format_exc())
return False
def combinelon(prefix, inputdir, fill_value='1e20', daily=False, year=None):
if daily:
files = [
inputdir + sep + f
for f in filter(listdir(inputdir), '%s*.%d.psims.nc' %
(prefix, year))
]
else:
files = [
inputdir + sep + f
for f in filter(listdir(inputdir), '%s*.psims.nc' % prefix)
]
# tile latitude and longitude indices
tlatidx = basename(files[0]).split('_')[1]
lonidx = [int(basename(f).split('_')[2][:4]) for f in files]
# get file information
with nc(files[0]) as f:
vars = setdiff1d(f.variables.keys(),
['time', 'scen', 'irr', 'lat', 'lon'])
nscen = f.variables['scen'].size
ldim = f.variables[vars[0]].dimensions[0]
vunits = [0] * len(vars)
vlnames = [0] * len(vars)
for i in range(len(vars)):
var = f.variables[vars[i]]
vunits[i] = var.units if 'units' in var.ncattrs() else ''
vlnames[i] = var.long_name if 'long_name' in var.ncattrs() else ''
# fill longitude gaps
for idx in setdiff1d(fulllonidx, lonidx):
if daily:
lonfile = inputdir + sep + '%s_%s_%04d.%d.psims.nc' % (
prefix, tlatidx, idx, year)
else:
lonfile = inputdir + sep + '%s_%s_%04d.psims.nc' % (prefix,
tlatidx, idx)
copyfile(files[0], lonfile)
lons = arange(-180 + tlond * (idx - 1) + lond / 2., -180 + tlond * idx,
lond)
with nc(lonfile, 'a') as f:
lonvar = f.variables['lon']
lonvar[:] = lons
for i in range(len(vars)):
var = f.variables[vars[i]]
var[:] = masked_array(zeros(var.shape), mask=ones(var.shape))
files.append(lonfile)
# output file
if daily:
outputfile = outputdir + sep + '%s_%s.%d.psims.nc' % (prefix, tlatidx,
year)
else:
outputfile = outputdir + sep + '%s_%s.psims.nc' % (prefix, tlatidx)
nco = Nco()
# make longitude lead dimension
for i in range(len(files)):
nco.ncpdq(input=files[i],
output=files[i],
options='-O -h -a lon,%s' % str(ldim))
# concatenate all files
if daily:
inputfiles = ' '.join([
inputdir + sep + '%s_%s_%04d.%d.psims.nc' %
(prefix, tlatidx, idx, year) for idx in fulllonidx
])
else:
inputfiles = ' '.join([
inputdir + sep + '%s_%s_%04d.psims.nc' % (prefix, tlatidx, idx)
for idx in fulllonidx
])
nco.ncrcat(input=inputfiles, output=outputfile, options='-h')
# make latitude lead dimension
nco.ncpdq(input=outputfile, output=outputfile, options='-O -h -a lat,lon')
# add new scenario dimension
nscennew = nscen / (1 + irrflag)
scen_range = ','.join([str(s) for s in range(1, nscennew + 1)])
scenopt = '-O -h -s \'defdim("scen_new",%d)\' -s "scen_new[scen_new]={%s}"' % (
nscennew, scen_range)
nco.ncap2(input=outputfile, output=outputfile, options=scenopt)
nco.ncatted(input=outputfile,
output=outputfile,
options='-O -h -a units,scen_new,c,c,"no"')
nco.ncatted(input=outputfile,
output=outputfile,
options='-O -h -a long_name,scen_new,c,c,"scenario"')
# add irr dimension
nirr = 1 + irrflag
irr_range = ','.join([str(i) for i in range(1, nirr + 1)])
irr_lname = ['ir', 'rf'][:1 + irrflag] if irr1st else ['rf', 'ir'][:1 +
irrflag]
irropt = '-O -h -s \'defdim("irr",%d)\' -s "irr[irr]={%s}"' % (nirr,
irr_range)
nco.ncap2(input=outputfile, output=outputfile, options=irropt)
nco.ncatted(input=outputfile,
output=outputfile,
options='-O -h -a units,irr,c,c,"mapping"')
nco.ncatted(input=outputfile,
output=outputfile,
options='-O -h -a long_name,irr,c,c,"%s"' %
','.join(irr_lname))
# refactor variables
for i in range(len(vars)):
var = str(vars[i])
# create new variable
opt = '-O -h -s "\'%s_new\'[lat,scen_new,irr,lon,time]=0.0f"' % var
nco.ncap2(input=outputfile, output=outputfile, options=opt)
# set attributes
opt = '-O -h -a _FillValue,%s_new,c,f,%s' % (var, fill_value)
nco.ncatted(input=outputfile, output=outputfile, options=opt)
if vunits[i]:
opt = '-O -h -a units,%s_new,c,c,"%s"' % (var, str(vunits[i]))
nco.ncatted(input=outputfile, output=outputfile, options=opt)
if vlnames[i]:
opt = '-O -h -a long_name,%s_new,c,c,"%s"' % (var, str(vlnames[i]))
nco.ncatted(input=outputfile, output=outputfile, options=opt)
# set value
opt = '-O -h -s "\'%s_new\'(:,:,:,:,:)=\'%s\'"' % (var, var)
nco.ncap2(input=outputfile, output=outputfile, options=opt)
# remove old variable
opt = '-O -h -x -v %s' % var
nco.ncks(input=outputfile, output=outputfile, options=opt)
# rename new variable
opt = '-O -h -v %s_new,%s' % (var, var)
nco.ncrename(input=outputfile, output=outputfile, options=opt)
# remove old scenario dimension
nco.ncks(input=outputfile, output=outputfile, options='-O -h -x -v scen')
nco.ncrename(input=outputfile,
output=outputfile,
options='-O -h -v scen_new,scen')
# limit spatial extent to sim grid
nco.ncks(input=outputfile,
output=outputfile,
options='-O -h -d lon,%f,%f' % (lon0, lon1))
dirs.append(dir_ini)
dir_hs = 'processed_hillshade'
dirs.append(dir_hs)
for dir_processed in dirs:
if not os.path.isdir(os.path.join(idir, dir_processed)):
os.mkdir(os.path.join(idir, dir_processed))
pvars = ('thk', 'usurf', 'velsurf_mag', 'velbase_mag')
fill_value = -2e9
v_str = ' '.join('='.join([x, str(fill_value) + ';']) for x in pvars)
m_str = 'sftgif=mask*0; where(thk>10) {sftgif=1;}; where(usurf>300) {sftgif=3;};'
ncap2_str = 'where(thk<10) {{ {} }}; {}'.format(v_str, m_str)
exp_files = glob(os.path.join(idir, 'state', '*.nc'))
for exp_file in exp_files:
exp_basename = os.path.split(exp_file)[-1].split('.nc')[0]
exp_nc_wd = os.path.join(idir, dir_nc, exp_basename + '.nc')
exp_gtiff_wd = os.path.join(idir, dir_gtiff, exp_basename + '.tif')
logger.info('masking variables where ice thickness < 10m')
nco.ncap2(input='-s "{}" {}'.format(ncap2_str, exp_file), output=exp_nc_wd, overwrite=True)
opt = [c.Atted(mode="o", att_name="_FillValue", var_name=myvar, value=fill_value) for myvar in pvars]
nco.ncatted(input=exp_nc_wd, options=opt)
logger.info('extracting ice-noice transition')
exp_ini_wd = os.path.join(idir, dir_ini, exp_basename + '.shp')
cmd = ['extract_interface.py', '-m', 'sftgif', '-t', 'ice_noice', '--epsg', '26710', '-o', exp_ini_wd, exp_nc_wd]
sub.call(cmd)
for mvar in pvars:
m_exp_nc_wd = 'NETCDF:{}:{}'.format(exp_nc_wd, mvar)
m_exp_gtiff_wd = os.path.join(idir, dir_gtiff, mvar + '_' + exp_basename + '.tif')
logger.info('Converting variable {} to GTiff and save as {}'.format(mvar, m_exp_gtiff_wd))
gdal.Translate(m_exp_gtiff_wd, m_exp_nc_wd, options=gdal_gtiff_options)
def download(config):
days_to_keep = config['days_to_keep']
# Download new data
if not os.path.exists(config['download_folder']):
os.mkdir(config['download_folder'])
nco = Nco(debug=True)
for name, opt in config['sources'].items():
if name[0] == '#':
print('Skipping ' + name[1::])
continue
print('='*30)
print(name)
print('-'*30)
folder = os.path.join(config['download_folder'], name)
if not os.path.exists(folder):
os.mkdir(folder)
agg_file = os.path.join(folder, name + '_aggregate.nc')
try:
if config['testing'] is True:
opt['opt'].append(opt['subset'])
new_download = False
print(opt['url'])
if opt['url'] == datetime.now().strftime(opt['url']):
###########################
# Download from aggregate
###########################
if 'stride' in opt:
stride = opt['stride']
else:
stride = 1
d = Dataset(opt['url'])
remote_tvar = d.variables['time']
remote_time = num2date(remote_tvar[-2:], remote_tvar.units)
remote_time_step = remote_time[1] - remote_time[0]
local_time_step = remote_time_step*stride
remote_end_time = remote_time[1]
remote_num_times = len(remote_tvar)
if os.path.exists(agg_file):
l = Dataset(agg_file)
local_tvar = l.variables['time']
local_time = num2date(local_tvar[:], local_tvar.units)
if remote_end_time <= local_time[-1] + local_time_step:
print('No new data for %s, last time: %s' % (name, local_time[-1]))
continue
download_from = remote_end_time - timedelta(hours=opt['forecast_hours'])
if download_from > local_time[-1]:
download_from = local_time[-1]
newfile = False
else:
download_from = datetime.now() - timedelta(days=1) # Only 1 day, otherwise too large file
newfile = True
# Find temporal subset to download
last_index = remote_num_times - stride + 1
if stride == 1:
last_index = last_index - 1
first_index = np.int32(last_index - stride*np.ceil((remote_end_time - download_from).total_seconds()/local_time_step.total_seconds()) - 1)
new_times = num2date(remote_tvar[first_index:last_index+1], remote_tvar.units)
if newfile is False:
print('%s ends %s -> downloading %s -> %s' % (name, local_time[-1], new_times[0], new_times[-1]))
else:
print('Downloading new file for %s: %s -> %s' % (name, new_times[0], new_times[-1]))
ts = "-d time,%i,%i" % (first_index, last_index)
if stride > 1:
ts = ts + ',%i' % stride
fname = new_times[0].strftime(name+'_%Y%m%d%H.nc')
fullname = os.path.join(folder, fname)
#if newfile:
# fullname = agg_file
options = opt['opt']
options.append(ts)
options.append("-7") # netCDF4 CLASSIC to avoid file size limitation
options.append("--mk_rec_dmn time") # Make time unlimited for aggregation
nco.ncks(input=opt['url'], output=fullname, options=options)
if os.path.exists(fullname):
new_download = True
d.close()
else:
#################################
# Downloading from single URLS
#################################
fnames, urls = get_names_and_urls(name, opt)
for fname, url in zip(fnames, urls):
fullname = os.path.join(folder, fname)
if os.path.exists(fullname):
print('%s exists, skipping' % fullname)
continue
print('Checking: ' + url)
# Check if URL is available
# Using requests, since .netrc file is needed
resp = requests.get(url + '.das', timeout=3)
if resp.status_code >= 400:
print('Requests: error code %s' % resp.status_code)
continue
# Download from URL
print('Downloading %s from %s...' %
(fullname, url))
try:
nco.ncks(input=url, output=fullname,
options=opt['opt'])
if os.path.exists(fullname):
new_download = True
if name in ['meps', 'aromearctic'] and 'acc' in opt['opt']:
print('Converting accumulated precipitation to hourly')
nco.ncap2(input=fullname, output=fullname,
options=['-O'],
spt='\'precipitation_amount=precipitation_amount_acc(1:$time.size-1,0,:,:)-precipitation_amount_acc(0:$time.size-2,0,:,:)\'')
except Exception as e:
print('DOWNLOAD NOT SUCCESSFUL!')
print(e)
print('STOPPING')
#sys.exit('stop')
##################
# Postprocessing
##################
if new_download is False:
print('No download, hence no postprocessing')
continue
else:
print('Postprocessing downloaded data')
# Remove overlapping times from downloaded files
timestep = opt['timestep']
folder = os.path.join(config['download_folder'], name)
catfolder = os.path.join(folder, 'concatenate')
if not os.path.exists(folder):
os.mkdir(folder)
if not os.path.exists(catfolder):
os.mkdir(catfolder)
print(folder, catfolder)
files = sorted(glob.glob(
os.path.join(folder, name + '_??????????.nc')))
print(files)
if len(files)==0:
continue
times = [datetime.strptime(f[-13:-3], '%Y%m%d%H') for f in files]
print('Making aggregate from: ', name, files)
for i in range(len(times)-1,-1,-1):
print(i, times[i])
filename = files[i]
if times[i] < datetime.now() - timedelta(hours=24*(days_to_keep+1)):
print('Deleting old input file:' + filename)
os.remove(filename)
continue
if i == len(times)-1: # Last file in list
print('Using whole file')
else:
print(times, 'times')
steps = (times[i+1]-times[i]).total_seconds()/(timestep*3600)
print(steps, 'steps')
catfile = os.path.join(catfolder, name +
times[i].strftime('_%Y%m%d%H') +
times[i+1].strftime('_%Y%m%d%H.nc'))
print(catfile, 'catfile')
if not os.path.exists(catfile):
print('Cutting...')
try:
# Cut part of file for later concatenation
nco.ncks(input=filename, output=catfile,
options=['-d time,0,%i' % (steps-1)])
# Delete the contents of original file to save space
os.remove(filename)
# Create empty file, to prevent new download
open(filename, 'a').close()
except:
print('Cutting failed, hence removing input file: ' + filename)
os.remove(filename)
try:
os.remove(catfile)
except:
pass
########################
print('Maintainance')
########################
catfiles = sorted(glob.glob(os.path.join(catfolder, name + '*.nc')))
print(catfiles, 'CATFILES')
for i, f in enumerate(catfiles):
starttime = datetime.strptime(f[len(os.path.join(catfolder+name))+2:-14],
'%Y%m%d%H')
endtime = datetime.strptime(f[-13:-3], '%Y%m%d%H')
if endtime < datetime.now() - timedelta(hours=24*days_to_keep):
print('Deleting old catfile:' + f)
os.remove(f)
previous_endtime = endtime
previous_starttime = starttime
continue
if i>0 and starttime < previous_endtime:
print('Overlapping catfiles, cutting the first: (%s, %s)' %
(catfiles[i-1], catfiles[i]))
steps = (previous_endtime-starttime).total_seconds()/(timestep*3600)
totalsteps = (previous_endtime-previous_starttime).total_seconds()/(timestep*3600)
stepstocut = totalsteps - steps
newname = catfiles[i-1][0:-14] + starttime.strftime('_%Y%m%d%H.nc')
nco.ncks(input=catfiles[i-1], output=newname,
options=['-d time,0,%i' % (stepstocut-1)])
os.remove(catfiles[i-1])
previous_endtime = endtime
previous_starttime = starttime
# Finally, concatenate all files in catfolder + latest-file
print(files, 'FILES')
infiles = sorted(glob.glob(os.path.join(
catfolder, name + '*.nc'))) + [files[-1]]
if len(infiles) > 1:
print('Concatinating: ' + str(infiles))
nco.ncra(output=os.path.join(folder, name + '_aggregate.nc'),
input=infiles,
options=['-Y ncrcat', '-O']) # Since Windows lack link ncrcat -> ncra
# https://sourceforge.net/p/nco/discussion/9830/thread/e8b45a9cdb/
elif len(infiles) == 1:
shutil.copyfile(infiles[0], agg_file)
except Exception as e:
print('Downloading failed:')
print(e)
import traceback
print(traceback.format_exc())
for name, opt in config['sources'].items():
tmpfiles = sorted(glob.glob(os.path.join(config['download_folder'], name, '*.nc*tmp')))
for tmpfile in tmpfiles:
print('Deleting tmp-file: ' + tmpfile)
os.remove(tmpfile)
# Sum up the coverage of aggregates
print()
print('=========================================================')
print('Aggregate files:')
for name, opt in config['sources'].items():
if name[0] != '#':
print(os.path.join(config['download_folder'],
name, name + '_aggregate.nc'))
print('=========================================================')
print('Coverage of aggregate files:')
notify = ''
for name, opt in config['sources'].items():
if name[0] == '#':
continue
timestep = opt['timestep']
folder = os.path.join(config['download_folder'], name)
aggfile = os.path.join(folder, name + '_aggregate.nc')
try:
d = Dataset(aggfile)
except:
print('Missing: ' + aggfile)
continue
times = d.variables['time']
times = num2date(times[:], times.units)
d.close()
minushours = np.round((times[0]-datetime.now()).total_seconds()/3600)
plushours = np.round((times[-1]-datetime.now()).total_seconds()/3600)
expected_steps = (plushours-minushours+1)/timestep
if expected_steps != len(times) and name != 'ecmwf' and False: # NB: temporarily disabled for missing times
warn = 'Expected %d steps, files has %d steps' % (
expected_steps, len(times))
else:
warn = ''
if plushours < 24:
warn += ', short forecast'
else:
warn += ''
if minushours > -24:
warn += ', short history'
else:
warn += ''
summary = ('%12s%11sUTC - %9sUTC (%+d, %+d) %s' %
(name, times[0].strftime('%d %b %H'),
times[-1].strftime('%d %b %H'),
minushours, plushours, warn))
print(summary)
if warn != '':
notify = notify + '\n' + summary
print('=========================================================')
if notify != '' and config['email_notification'] is not None:
try:
import smtplib
FROM = '*****@*****.**'
TO = [config['email_notification']]
message = """From: %s\nTo: %s\nSubject: %s\n\n%s""" % (
FROM, ', '.join(TO),
'Thredds Harvest - short timeseries', notify)
server = smtplib.SMTP('localhost')
server.sendmail(FROM, TO, message)
server.quit()
print('Email sent to ' + TO[0])
print(message)
except Exception as ex:
print('Could not send mail report on missing data')
print(ex)
var_name='global',
value="+init=epsg:3413",
stype='c'),
c.Atted(mode="o",
att_name="units",
var_name='velsurf_mag',
value="m year-1")
]
nco.ncatted(input=ofile_merged_wp, options=opt)
for mvar in ('vx', 'vy', 'ex', 'ey'):
ifile = '.'.join(['_'.join([basename, mvar, version]), 'tif'])
ofile = '.'.join(['_'.join([basename, mvar, version]), 'nc'])
ofile_wp = os.path.join(basedir, ofile)
ifile_wp = os.path.join(basedir, ifile)
gdal.Translate(ofile_wp, ifile_wp, options=gdal_nc_options)
rDict = {'Band1': var_dict[mvar]}
opt = [c.Rename('variable', rDict)]
nco.ncrename(input=ofile_wp, options=opt)
opt = [
c.Atted(mode="o",
att_name="units",
var_name=var_dict[mvar],
value="m year-1")
]
nco.ncatted(input=ofile_wp, options=opt)
nco.ncks(input=ofile_wp, output=ofile_merged_wp, append=True)
ncap2_str = 'velsurf_mag_error=sqrt(uvelsurf_error^2+vvelsurf_error^2); velsurf_normal_error=velsurf_mag_error;'
nco.ncap2(input='-s "{}" {}'.format(ncap2_str, ofile_merged_wp),
output=ofile_merged_wp,
overwrite=True)
logger = logging.getLogger('calculate_topg_delta')
logger.setLevel(logging.DEBUG)
# create file handler which logs even debug messages
fh = logging.handlers.RotatingFileHandler('extract.log')
fh.setLevel(logging.DEBUG)
# create console handler with a higher log level
ch = logging.StreamHandler()
ch.setLevel(logging.INFO)
# create formatter
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(module)s:%(lineno)d - %(message)s')
# add formatter to ch and fh
ch.setFormatter(formatter)
fh.setFormatter(formatter)
# add ch to logger
logger.addHandler(ch)
logger.addHandler(fh)
# set up the option parser
parser = ArgumentParser()
parser.description = "Calculate topg-topg_initial."
parser.add_argument("INFILE", nargs=1)
parser.add_argument("OUTFILE", nargs=1)
options = parser.parse_args()
nco.ncap2(input=options.INFILE, output=options.OUTFILE, options='''-O -s "topg_delta=topg-topg_initial;"''')
exp_file
]
#sub.call(cmd)
logger.info('extracting ice ocean interface')
exp_io_wd = os.path.join(idir, dir_io, exp_basename + '.shp')
cmd = [
'extract_interface.py', '-t', 'ice_ocean', '-o', exp_io_wd, exp_file
]
#sub.call(cmd)
logger.info('masking variables where ice thickness < 10m')
nco.ncks(input=exp_file,
output=exp_nc_wd,
variable=','.join([x for x in pvars]),
overwrite=True)
nco.ncap2(input='-6 -s "{}" {}'.format(ncap2_str, exp_nc_wd),
output=exp_nc_wd,
overwrite=True)
opt = [
c.Atted(mode="o",
att_name="_FillValue",
var_name=myvar,
value=fill_value) for myvar in ppvars
]
nco.ncatted(input=exp_nc_wd, options=opt)
for mvar in pvars:
m_exp_nc_wd = 'NETCDF:{}:{}'.format(exp_nc_wd, mvar)
m_exp_gtiff_wd = os.path.join(idir, dir_gtiff,
mvar + '_' + exp_basename + '.tif')
logger.info('Converting variable {} to GTiff and save as {}'.format(
mvar, m_exp_gtiff_wd))
gdal.Translate(m_exp_gtiff_wd, m_exp_nc_wd, options=gdal_gtiff_options)
