def dispatch_entry(config, entry, dry_run=None, log_name=LOGGER):
"""Dispacthes one entry of distribution list"""
logger = loghelper.get_logger(log_name)
address = expand(entry['mailto'], config)
subject = expand(entry['subject'], config)
body = expand(entry['body'], config)
from_addr = expand(entry['from'], config)
attachments = [expand(a, config) for a in entry['attach']]
logger.debug('dispatch_entry() called')
if type(attachments)==type([]):
a_arg = ' '.join(['-a %s' % a for a in attachments])
else:
a_arg = '-a %s' % attachments
if 'cc' in entry:
cc_arg = '-c %s' % entry['cc']
else:
cc_arg = ''
if 'content_type' in entry:
ct_arg = '-e "my_hdr Content-Type: :%s"' % dist['content_type']
else:
ct_arg = ''
cmd = """EMAIL="%s" mutt %s -s"%s" %s %s -- %s < %s """ %(from_addr, ct_arg, subject, a_arg, cc_arg, address, body)
logger.debug(cmd)
logger.debug(dry_run)
if not dry_run:
subprocess.call(cmd, shell=True)
def dispatch_entry(config, entry, dry_run=None, log_name=LOGGER):
"""Dispacthes one entry of distribution list"""
logger = loghelper.get_logger(log_name)
address = expand(entry['mailto'], config)
subject = expand(entry['subject'], config)
body = expand(entry['body'], config)
from_addr = expand(entry['from'], config)
attachments = [expand(a, config) for a in entry['attach']]
logger.debug('dispatch_entry() called')
if type(attachments)==type([]):
a_arg = ' '.join(['-a %s' % a for a in attachments])
else:
a_arg = '-a %s' % attachments
if 'cc' in entry:
cc_arg = '-c %s' % entry['cc']
else:
cc_arg = ''
if 'content_type' in entry:
ct_arg = '-e "my_hdr Content-Type: :%s"' % dist['content_type']
else:
ct_arg = ''
cmd = """EMAIL="%s" mutt %s -s"%s" %s %s -- %s < %s """ %(from_addr, ct_arg, subject, a_arg, cc_arg, address, body)
logger.debug(cmd)
if not dry_run:
subprocess.call(cmd, shell=True)
def dump(files, entry, scope, log_name=LOGGER):
logger = loghelper.get_logger(log_name)
vars = entry['tseries_vars']
global_atts = entry['global_atts']
var_atts = entry['var_atts']
coord_vars = entry['coord_vars']
format = entry['format'].strip()
#logger.warn("subsetting at read time is not implemented")
# Read all data into memory as pandas Series objects
logger.debug("ncdump called with arguments")
logger.debug("\t files: %s" % str(files))
logger.debug("\t vars: %s" % str(vars))
logger.debug("\t global_atts: %s" % str(global_atts))
logger.debug("\t var_atts: %s" % str(var_atts))
logger.debug("\t coord_vars: %s" % str(coord_vars))
logger.debug("\t log_name: %s" % str(log_name))
#for file in files:
#logger.debug(file)
frame = frame_from_nc(files, vars, global_atts, var_atts, coord_vars,
log_name)
if format not in FORMATS:
logger.error("format %s not understood" % format)
raise UnknownFormat("format not understood")
if format == 'txt':
pass
#write_txt_files(frame, entry['dir'], entry['dimspec'], log_name)
elif format == 'json':
write_json_files(frame, entry['dir'], expand(entry['fname'], scope),
entry['tseries_vars'], entry['dimspec'],
entry['drop'], entry['rename'], entry['float_format'],
log_name)
elif format == 'csv':
write_csv_files(frame,
entry['dir'],
expand(entry['fname'], scope),
entry['tseries_vars'],
entry['dimspec'],
entry['drop'],
values='value',
rows=entry['rows'],
cols=entry['cols'],
sort_by=entry['sort_by'],
rename=entry['rename'],
float_format=entry['float_format'],
na_rep=entry['na_rep'],
log_name=log_name)
elif format == 'aot':
write_aot_files(frame, entry['dir'])
def ncdump(config):
logger = loghelper.get_logger(config['log.name'])
# subset of config to be used for expanding filenames
scope = {'init_time' : config['init_time'],
'grid_id' : config['grid_id']}
for name, entry in config['ncdump'].items():
logger.debug("procesing entry %s " % name)
if config.get('<files>'):
files = config['<files>']
if type(files)!=type([]):
files = [files]
else:
tseries_files = expand(entry['tseries_file'], config)
logger.debug("expanding file list from pattern and init time")
#logger.debug(tseries_files)
files = glob.glob(tseries_files)
logger.debug("found %d files" % len(files))
dump(files, entry, scope, log_name=config['log.name'])
def ncdump(config):
logger = loghelper.get_logger(config['log.name'])
# subset of config to be used for expanding filenames
scope = {'init_time' : config['init_time'],
'grid_id' : config['grid_id']}
for name, entry in config['ncdump'].items():
logger.debug("procesing entry %s " % name)
if config.get('<files>'):
files = config['<files>']
if type(files)!=type([]):
files = [files]
else:
tseries_files = expand(entry['tseries_file'], config)
logger.debug("expanding file list from pattern and init time")
files = glob.glob(tseries_files)
logger.debug("found %d files" % len(files))
dump(files, entry, scope, log_name=config['log.name'])
def dump(files,entry,scope,log_name=LOGGER):
logger = loghelper.get_logger(log_name)
vars = entry['tseries_vars']
global_atts = entry['global_atts']
var_atts = entry['var_atts']
coord_vars = entry['coord_vars']
format = entry['format'].strip()
#logger.warn("subsetting at read time is not implemented")
# Read all data into memory as pandas Series objects
logger.debug("ncdump called with arguments")
logger.debug("\t files: %s" % str(files))
logger.debug("\t vars: %s" % str(vars))
logger.debug("\t global_atts: %s" % str(global_atts))
logger.debug("\t var_atts: %s" % str(var_atts))
logger.debug("\t coord_vars: %s" % str(coord_vars))
logger.debug("\t log_name: %s" % str(log_name))
for file in files:
logger.debug(file)
frame = frame_from_nc([file], vars, global_atts, var_atts, coord_vars,log_name)
if format not in FORMATS:
logger.error("format %s not understood" % format)
raise UnknownFormat("format not understood")
if format=='txt' :
pass
#write_txt_files(frame, entry['dir'], entry['dimspec'], log_name)
elif format=='json':
write_json_files(frame, entry['dir'], expand(entry['fname'], scope), entry['tseries_vars'], entry['dimspec'], entry['drop'], entry['rename'], entry['float_format'], log_name)
elif format=='csv':
write_csv_files(frame, entry['dir'], expand(entry['fname'], scope), entry['tseries_vars'],entry['dimspec'], entry['drop'], values='value', rows=entry['rows'],cols=entry['cols'],sort_by=entry['sort_by'],rename=entry['rename'],float_format=entry['float_format'], na_rep=entry['na_rep'], log_name=log_name)
elif format=='aot':
write_aot_files(frame, entry['dir'])
def power(config):
"""Reads 'time series' from netcdf time series file, and adds power as a variable. """
if __name__ == "__main__":
logger = loghelper.create_logger(config)
else:
logger = loghelper.get_logger(config['log.name'])
# Number of samples to use should be in here
# Whether to normalise power should be in here
pnorm = config['pnorm']
pdist = config['pdist']
sstd = config['sstd']
dstd = config['dstd']
pquants = config['pquants']
quantiles = np.array(pquants)
logger.debug(pnorm)
if pdist:
n=pdist
grid_id = config['grid_id']
init_time = config['init_time']
pcurve_dir = config['pcurve_dir']
ts_dir = config['tseries_dir']
tseries_file = expand(config['tseries_file'], config)
power_file = expand(config['power_file'], config)
logger.info('Estimating power from time series: %s ' % tseries_file)
logger.info('Writing power time series to: %s ' % power_file)
dataset_in = Dataset(tseries_file, 'a')
# Get dimensions
dims = dataset_in.dimensions
ntime = len(dims['time'])
nloc = len(dims['location'])
nheight = len(dims['height'])
loc_str_len = len(dims['loc_str_length'])
# Get coordinate variables
nctime = dataset_in.variables['time']
datetimes = netcdftime.num2date(nctime, nctime.units)
location = [''.join(l.filled(' ')).strip() for l in dataset_in.variables['location']]
height = dataset_in.variables['height']
# Get attributes
metadata = config['metadata']
if power_file == tseries_file:
dataset_out = dataset_in
else:
dataset_out = Dataset(power_file, 'w')
# Get number of quantiles
nq = len(quantiles)
pdata = np.ma.zeros((ntime,nloc,nheight,nq+1), np.float) # mean will be 1st value
use_locs = []
for l,loc in enumerate(location):
pcurve_file = '%s/%s.csv' %(pcurve_dir, loc)
# mask power data if no power curve found for this park
if not os.path.exists(pcurve_file):
#logger.debug("Power curve: %s not found, skipping" % pcurve_file)
pdata[:,l,:,:] = np.ma.masked
continue
logger.info('Predicting power output for %s' % loc )
#
# Open power curve
#
use_locs.append(l)
pcurve = from_file(pcurve_file)
for h in range(nheight):
speed = dataset_in.variables['SPEED'][:,l,h]
direction = dataset_in.variables['DIRECTION'][:,l,h]
#pwr = pcurve.power(speed,direction)
# pdist will create a distribution for each timetep based on sampling
# n times from a normal distribution.
pdist = pcurve.power_dist(speed, direction, sstd=sstd,dstd=dstd,n=n, normalise=pnorm)
pmean = np.mean(pdist, axis=1)
pquants = scipy.stats.mstats.mquantiles(pdist, prob=quantiles/100.0,axis=1, alphap=0.5, betap=0.5)
pdata[:,l,h,0] = pmean
pdata[:,l,h,1:] = pquants[:,:]
logger.info('finished %s' % loc)
use_inds = np.array(use_locs)
logger.debug(use_inds)
logger.debug(pdata.shape)
logger.debug(pdata[:,use_inds,:,:].shape)
if dataset_out != dataset_in:
dataset_out.createDimension('time', None)
dataset_out.createVariable('time', 'float', ('time',))
dataset_out.variables['time'][:] = nctime[:]
dataset_out.variables['time'].units = nctime.units
dataset_out.variables['time'].calendar = nctime.calendar
dataset_out.createDimension('location', len(use_locs))
dataset_out.createDimension('loc_str_length', loc_str_len)
loc_data =np.array([list(l.ljust(loc_str_len, ' ')) for l in location])
dataset_out.createVariable('location', 'c', ('location', 'loc_str_length'))
dataset_out.variables['location'][:] = loc_data[use_inds,:]
dataset_out.createDimension('height', nheight)
dataset_out.createVariable('height', 'i', ('height',))
dataset_out.variables['height'][:] = height[:]
dataset_out.GRID_ID = dataset_in.GRID_ID
dataset_out.DX = dataset_in.DX
dataset_out.DY = dataset_in.DY
try:
dataset_out.variables['height'].units = height.units
except Exception:
logger.warn("height units missing")
pdata = pdata[:, use_inds, :, :]
for key in metadata.keys():
key = key.upper()
logger.debug(key)
dataset_out.setncattr(key,dataset_in.getncattr(key))
pavg = dataset_out.createVariable('POWER','f',('time','location','height'))
pavg.units = 'kW'
pavg.description = 'forecast power output'
pavg[:] = pdata[:,:,:,0]
for q, qval in enumerate(quantiles):
varname = 'POWER.P%02d' % qval
logger.debug("creating variable %s" % varname)
var = dataset_out.createVariable(varname,'f',('time','location','height'))
if pnorm:
var.units = 'ratio'
else:
var.units = 'kW'
var.description = 'forecast power output'
print pdata[:,:,:,q+1]
var[:] = pdata[:,:,:,q+1]
dataset_in.close()
if dataset_out!=dataset_in:
dataset_out.close()
def power(config):
"""Reads 'time series' from netcdf time series file, and adds power as a variable. """
if __name__ == "__main__":
logger = loghelper.create_logger(config)
else:
logger = loghelper.get_logger(config['log.name'])
# Number of samples to use should be in here
# Whether to normalise power should be in here
pnorm = config['pnorm']
pdist = config['pdist']
sstd = config['sstd']
dstd = config['dstd']
pquants = config['pquants']
quantiles = np.array(pquants)
logger.debug(pnorm)
if pdist:
n = pdist
grid_id = config['grid_id']
init_time = config['init_time']
pcurve_dir = config['pcurve_dir']
ts_dir = config['tseries_dir']
tseries_file = expand(config['tseries_file'], config)
power_file = expand(config['power_file'], config)
logger.info('Estimating power from time series: %s ' % tseries_file)
logger.info('Writing power time series to: %s ' % power_file)
dataset_in = Dataset(tseries_file, 'a')
# Get dimensions
dims = dataset_in.dimensions
ntime = len(dims['time'])
nloc = len(dims['location'])
nheight = len(dims['height'])
loc_str_len = len(dims['loc_str_length'])
# Get coordinate variables
nctime = dataset_in.variables['time']
datetimes = netcdftime.num2date(nctime, nctime.units)
location = [
''.join(l.filled(' ')).strip()
for l in dataset_in.variables['location']
]
height = dataset_in.variables['height']
# Get attributes
metadata = config['metadata']
if power_file == tseries_file:
dataset_out = dataset_in
else:
dataset_out = Dataset(power_file, 'w')
# Get number of quantiles
nq = len(quantiles)
pdata = np.ma.zeros((ntime, nloc, nheight, nq + 1),
np.float) # mean will be 1st value
use_locs = []
for l, loc in enumerate(location):
pcurve_file = '%s/%s.csv' % (pcurve_dir, loc)
# mask power data if no power curve found for this park
if not os.path.exists(pcurve_file):
#logger.debug("Power curve: %s not found, skipping" % pcurve_file)
pdata[:, l, :, :] = np.ma.masked
continue
logger.info('Predicting power output for %s' % loc)
#
# Open power curve
#
use_locs.append(l)
pcurve = from_file(pcurve_file)
for h in range(nheight):
speed = dataset_in.variables['SPEED'][:, l, h]
direction = dataset_in.variables['DIRECTION'][:, l, h]
#pwr = pcurve.power(speed,direction)
# pdist will create a distribution for each timetep based on sampling
# n times from a normal distribution.
pdist = pcurve.power_dist(speed,
direction,
sstd=sstd,
dstd=dstd,
n=n,
normalise=pnorm)
pmean = np.mean(pdist, axis=1)
pquants = scipy.stats.mstats.mquantiles(pdist,
prob=quantiles / 100.0,
axis=1,
alphap=0.5,
betap=0.5)
pdata[:, l, h, 0] = pmean
pdata[:, l, h, 1:] = pquants[:, :]
logger.info('finished %s' % loc)
use_inds = np.array(use_locs)
logger.debug(use_inds)
logger.debug(pdata.shape)
logger.debug(pdata[:, use_inds, :, :].shape)
if dataset_out != dataset_in:
dataset_out.createDimension('time', None)
dataset_out.createVariable('time', 'float', ('time', ))
dataset_out.variables['time'][:] = nctime[:]
dataset_out.variables['time'].units = nctime.units
dataset_out.variables['time'].calendar = nctime.calendar
dataset_out.createDimension('location', len(use_locs))
dataset_out.createDimension('loc_str_length', loc_str_len)
loc_data = np.array(
[list(l.ljust(loc_str_len, ' ')) for l in location])
dataset_out.createVariable('location', 'c',
('location', 'loc_str_length'))
dataset_out.variables['location'][:] = loc_data[use_inds, :]
dataset_out.createDimension('height', nheight)
dataset_out.createVariable('height', 'i', ('height', ))
dataset_out.variables['height'][:] = height[:]
dataset_out.GRID_ID = dataset_in.GRID_ID
dataset_out.DX = dataset_in.DX
dataset_out.DY = dataset_in.DY
try:
dataset_out.variables['height'].units = height.units
except Exception:
logger.warn("height units missing")
pdata = pdata[:, use_inds, :, :]
for key in metadata.keys():
key = key.upper()
logger.debug(key)
dataset_out.setncattr(key, dataset_in.getncattr(key))
pavg = dataset_out.createVariable('POWER', 'f',
('time', 'location', 'height'))
pavg.units = 'kW'
pavg.description = 'forecast power output'
pavg[:] = pdata[:, :, :, 0]
for q, qval in enumerate(quantiles):
varname = 'POWER.P%02d' % qval
logger.debug("creating variable %s" % varname)
var = dataset_out.createVariable(varname, 'f',
('time', 'location', 'height'))
if pnorm:
var.units = 'ratio'
else:
var.units = 'kW'
var.description = 'forecast power output'
print pdata[:, :, :, q + 1]
var[:] = pdata[:, :, :, q + 1]
dataset_in.close()
if dataset_out != dataset_in:
dataset_out.close()
