def output_fig(self, fig,
out_dir = None, overwrite=None,
#figure file controls
fmt='svg',
transparent=True,
dpi = 150,):
#======================================================================
# defaults
#======================================================================
if out_dir is None: out_dir = self.out_dir
if overwrite is None: overwrite = self.overwrite
log = self.logger.getChild('output')
#======================================================================
# output
#======================================================================
#file setup
out_fp = os.path.join(out_dir, '%s_smry_plot.%s'%(self.name, fmt))
if os.path.exists(out_fp):
msg = 'passed output file path already esists :\n %s'%out_fp
if overwrite:
log.warning(msg)
else:
raise Error(msg)
#write the file
try:
fig.savefig(out_fp, dpi = dpi, format = fmt, transparent=transparent)
log.info('saved figure to file: %s'%out_fp)
except Exception as e:
raise Error('failed to write figure to file w/ \n %s'%e)
return out_fp
def load_layers(
self, #load data to project (for console runs)
rfp_l,
finv_fp,
providerLib='ogr'):
"""
special input loader for standalone runs
Im assuming for the plugin these layers will be loaded already"""
log = self.logger.getChild('load_layers')
#======================================================================
# load rasters
#======================================================================
raster_d = dict()
for fp in rfp_l:
rlayer = self.load_rlay(fp)
#add it in
basefn = os.path.splitext(os.path.split(fp)[1])[0]
raster_d[basefn] = rlayer
#======================================================================
# load finv vector layer
#======================================================================
fp = finv_fp
assert os.path.exists(fp)
basefn = os.path.splitext(os.path.split(fp)[1])[0]
vlay_raw = QgsVectorLayer(fp, basefn, providerLib)
# checks
if not isinstance(vlay_raw, QgsVectorLayer):
raise IOError
#check if this is valid
if not vlay_raw.isValid():
raise Error(
'loaded vlay \'%s\' is not valid. \n \n did you initilize?' %
vlay_raw.name())
#check if it has geometry
if vlay_raw.wkbType() == 100:
raise Error('loaded vlay has NoGeometry')
self.mstore.addMapLayer(vlay_raw)
vlay = vlay_raw
dp = vlay.dataProvider()
log.info(
'loaded vlay \'%s\' as \'%s\' %s geo with %i feats from file: \n %s'
% (vlay.name(), dp.storageType(), QgsWkbTypes().displayString(
vlay.wkbType()), dp.featureCount(), fp))
#======================================================================
# wrap
#======================================================================
return list(raster_d.values()), vlay
def get_cf_fp(self):
cf_fp = self.lineEdit_cf_fp.text()
if cf_fp is None or cf_fp == '':
raise Error('need to specficy a control file path')
if not os.path.exists(cf_fp):
raise Error('need to specficy a valid control file path')
if not os.path.splitext(cf_fp)[1] == '.txt':
raise Error('unexpected extension on Control File')
return cf_fp
def _set_depth_conv(
self, #get conversion dictionary form hazus labels
raw_l, #example depths ['ft04m', 'ft03m', 'ft02m', 'ft01m', 'ft00', 'ft01', 'ft02', 'ft03'
exposure_units=None, #for changing units
):
log = self.logger.getChild('get_depth_conv')
assert isinstance(raw_l, list)
if exposure_units is None: exposure_units = self.exposure_units
#=======================================================================
# get scale
#=======================================================================
if exposure_units == 'meters':
scale = self.ft_m
elif exposure_units == 'ft':
scale = 1
else:
raise Error('unrecognized exposure_units: %s' % exposure_units)
#=======================================================================
# get conversions
#=======================================================================
d = dict()
for raw in raw_l:
assert raw.startswith('ft'), raw
val = raw.replace('ft', '')
if raw.endswith('m'):
iscale = scale * -1 #invert scale for negatives
val = val[:-1]
else:
iscale = scale
try:
d[raw] = round(float(val) * iscale, self.prec)
except Exception as e:
raise Error('failed to float %s=\'%s\' \n %s' %
(raw, val, e))
#=======================================================================
# wrap
#=======================================================================
assert np.all(np.diff(pd.Series(d)) > 0), 'non-monotonic'
log.info('finished w/ %i' % len(d))
"""
d.keys()
view(pd.Series(d))
"""
self.depthc_d = d
return
def _prep_vlay(self, vlay_raw, keep_fnl, log=None):
if log is None: log = self.logger.getChild('_prep_vlay')
assert vlay_raw.crs() == self.qproj.crs(
), 'crs mismatch: \n %s\n %s' % (vlay_raw.crs(),
self.qproj.crs())
df_raw = vlay_get_fdf(vlay_raw, logger=log,
feedback=self.feedback).drop(['ogc_fid', 'fid'],
axis=1,
errors='ignore')
df_raw['fid'] = df_raw.index
#======================================================================
# drop to keeper fields
#======================================================================
if keep_fnl == 'all':
log.info('keeping all fields')
df1 = df_raw.copy()
elif isinstance(keep_fnl, list):
#check the request
miss_l = set(keep_fnl).difference(df_raw.columns)
if len(miss_l) > 0:
raise Error(
'%i requested keeper fields not in data: \n %s' %
(len(miss_l), miss_l))
#make sure the linker is in there
if not self.cid in keep_fnl:
keep_fnl.append(self.cid)
if not 'fid' in keep_fnl:
keep_fnl.append('fid')
#make the slice
df1 = df_raw.loc[:, keep_fnl]
log.info('dropped to %i columns (from %i)' %
(len(df1.columns), len(df_raw.columns)))
else:
raise Error('unexpected type on keep_fnl')
if not df1[self.cid].is_unique:
raise Error('non-unique vlay keys')
return df1
def load_data(self):
logger = self.logger.getChild('load_data')
self.filepath = self.get_filepath()
d = self.loadr_real(self.filepath, multi = True)
#=======================================================================
# sort and attach
#=======================================================================
for k, v in d.items():
logger.debug('sending \'%s\' for cleaning'%k)
df1 = self.clean_binv_data(v)
#cleaning for wsl
if k in ['dry', 'wet']:
df2 = self.wsl_clean(df1, tag=k)
self.wsl_d[k] = df2
#cleaning for area protection
elif k == 'aprot':
self.aprot_df = df1.astype(np.int)
else:
raise Error('got unexpected tab name \'%s\''%k)
return
def prep_model(
self,
event_slice=False, #allow the expolike data to pass MORE events than required
):
if self.as_inun:
raise Error('risk2 inundation percentage not implemented')
#data loaders
self.set_finv()
self.set_evals()
self.set_dmgs()
if not self.exlikes == '':
self.set_exlikes(event_slice=event_slice)
if self.attriMode:
"""the control file parameter name changes depending on the model"""
self.set_attrimat()
self.promote_attrim()
#activate plot styles
self.logger.debug('finished on Risk2')
return
def _get_val_str(
self, #helper to get value string for writing text on the plot
val_str, #cant be a kwarg.. allowing None
impactFmtFunc=None,
):
"""
generally just returns the val_str
but also provides some special handles
"""
#=======================================================================
# defaults
#=======================================================================
if impactFmtFunc is None: impactFmtFunc = self.impactFmtFunc
if val_str is None:
val_str = self.val_str
#=======================================================================
# special keys
#=======================================================================
if isinstance(val_str, str):
if val_str == '*default':
assert isinstance(self.ead_tot, float)
val_str = 'total annualized impacts = ' + impactFmtFunc(
self.ead_tot)
elif val_str == '*no':
val_str = None
elif val_str.startswith('*'):
raise Error('unrecognized val_str: %s' % val_str)
return val_str
def __init__(self, iface):
"""Constructor.
:param iface: An interface instance that will be passed to this class
which provides the hook by which you can manipulate the QGIS
application at run time.
:type iface: QgsInterface
"""
self.iface = iface
self.dialogs_d = dict()
#=======================================================================
# build dialog children
#=======================================================================
"""todo: only construct these on first pass"""
for attn, DialogClass in self.dialogPars_d.items():
try:
self.dialogs_d[attn] = DialogClass(self.iface, session=self)
except Exception as e:
raise Error('failed to load \'%s\' w/ \n %s' % (attn, e))
# Check if plugin was started the first time in current QGIS session
# Must be set in initGui() to survive plugin reloads
self.first_start = None
self.pars_dir = os.path.join(os.path.dirname(__file__), '_pars')
self.icon_dir = os.path.join(os.path.dirname(__file__), 'icons')
#=======================================================================
# logger
#=======================================================================
self.logger = plugLogger(self, log_nm='CanFlood')
def set_cf_pars(
self, #update the control file w/ the passed parameters
new_pars_d, #new paraemeters
# {section : ({valnm : value } OR string (for notes)})
cf_fp=None,
logger=None,
):
"""
should this be on the MOdel only?
"""
#=======================================================================
# defautls
#=======================================================================
if logger is None: logger = self.logger
log = logger.getChild('set_cf_pars')
#get defaults
if cf_fp is None: cf_fp = self.cf_fp
assert isinstance(
cf_fp, str), '%s got bad cf_fp type: %s' % (self.name, type(cf_fp))
assert os.path.exists(cf_fp), 'bad cf_fp: \n %s' % cf_fp
#initiliae the parser
pars = configparser.ConfigParser(allow_no_value=True)
_ = pars.read(cf_fp) #read it from the new location
#loop and make updates
cnt = 0
for section, val_t in new_pars_d.items():
assert isinstance(
val_t,
tuple), '\"%s\' has bad subtype: %s' % (section, type(val_t))
assert section in pars, 'requested section \'%s\' not in the pars!' % section
for subval in val_t:
#value key pairs
if isinstance(subval, dict):
for valnm, value in subval.items():
assert isinstance(value, str), \
'failed to get a str on %s.%s: \'%s\''%(section, valnm, type(value))
pars.set(section, valnm, value)
cnt += 1
#single values(for notes mostly)
elif isinstance(subval, str):
pars.set(section, subval)
cnt += 1
else:
raise Error('unrecognized value type: %s' % type(subval))
#write the config file
with open(cf_fp, 'w') as configfile:
pars.write(configfile)
log.info('updated control file w/ %i pars at :\n %s' % (cnt, cf_fp))
return
def check_dfeat(self): #standard checking fcommands
if not self.mind == self.model.mind:
raise IOError
#=======================================================================
# check hierarchy
#=======================================================================
if self.parent is None:
raise IOError
if not self.hse_o == self.parent.parent:
raise IOError
#check acode
if 'run' in self.session.state:
hp_oop.check_match(self, self.hse_o, attn='acode')
if not self.acode == self.parent.get_acode():
raise Error('acode mismiatch with parent')
gp = self.parent.parent
if not self.model == gp.model:
raise Error(
'\"%s\' model (%s) \n does not match their grandparents \'%s\' (%s)'
% (self.name, self.model, gp.name, gp.model))
if self.hse_o.geo_dxcol is None:
raise IOError
if not self.parent.__class__.__name__ == 'Dfunc':
raise IOError
#=======================================================================
# check meta_df
#=======================================================================
if not self.dfloc in self.parent.childmeta_df.index.tolist():
raise Error(
'could not find my dfloc \'%s\' in my parents \'%s\' index' %
(self.dfloc, self.parent.name))
if not self.name in self.parent.childmeta_df.loc[:,
'name'].values.tolist(
):
raise IOError
def gsamp(self, #resample results to a grid (from single asset res layer)
avlay, #asset results layer
gvlay=None, #new polygon grid to sample
gid=None,
res_fnl = ['ead'], #list of result fields to downsample
use_raw_fn=True, #whether to convert the summary field names back to raw.
logger=None,
discard_nomatch=False,
**jkwargs #joinbylocationsummary kwargs
):
"""
resample results
"""
#=======================================================================
# defaults
#=======================================================================
if logger is None: logger=self.logger
log=logger.getChild('gsamp')
if gvlay is None: gvlay = self.gvlay
if gid is None: gid=self.gid
log.info('downsampling \'%s\' (%i feats) to \'%s\' (%i)'%(
avlay.name(), avlay.dataProvider().featureCount(), gvlay.name(),
gvlay.dataProvider().featureCount()))
#=======================================================================
# prechecks
#=======================================================================
fn_l = [f.name() for f in gvlay.fields()]
assert gid in fn_l, gid
fn_l = [f.name() for f in avlay.fields()]
s = set(res_fnl).difference(fn_l)
if not len(s)==0:
raise Error('\'%s\' missing requested results fields: %s'%(avlay.name(), s))
#check the gids
gid_d = vlay_get_fdata(gvlay, fieldn=gid, logger=log)
assert pd.Series(gid_d).is_unique, '%s has bad gid=\'%s\''%(gvlay.name(), gid)
#=======================================================================
# calc
#=======================================================================
gvlay1, nfn_l = self.joinbylocationsummary(gvlay, avlay, res_fnl, use_raw_fn=use_raw_fn,
discard_nomatch=discard_nomatch,
**jkwargs)
#=======================================================================
# wrap
#=======================================================================
if not discard_nomatch:
assert gvlay1.dataProvider().featureCount()==gvlay.dataProvider().featureCount()
return gvlay1, nfn_l
def run_vuln(self):
log = self.logger.getChild('run_vuln')
log.debug('start')
self.set_setup() #attach all the commons
self.feedback.setProgress(5)
from misc.dikes.vuln import Dvuln
#=======================================================================
# collect inputs
#=======================================================================
#=======================================================================
# init
#=======================================================================
kwargs = {
attn: getattr(self, attn)
for attn in self.inherit_fieldNames
}
wrkr = Dvuln(**kwargs)
self.feedback.setProgress(10)
#=======================================================================
# load
#=======================================================================
wrkr._setup(dexpo_fp=self.lineEdit_v_dexpo_fp.text(),
dcurves_fp=self.lineEdit_v_dcurves_fp.text())
self.feedback.setProgress(20)
#==========================================================================
# execute
#==========================================================================
pf_df = wrkr.get_failP()
self.feedback.setProgress(60)
#=======================================================================
# length effect
#=======================================================================
#collect method from user
if self.radioButton_v_lfx_none.isChecked():
method = None
elif self.radioButton_v_lfx_urs.isChecked():
method = 'URS2007'
else:
raise Error('unrecognized lfx method')
if not method is None:
wrkr.set_lenfx(method=method) #apply length effects
self.feedback.setProgress(80)
#=======================================================================
# outputs
#=======================================================================
pfail_fp = wrkr.output_vdfs()
self.feedback.setProgress(95)
#=======================================================================
# update gui
#=======================================================================
self.lineEdit_ja_pfail_fp.setText(pfail_fp)
log.info('finished Dike Vuln w/ %s' % str(pf_df.shape))
self.feedback.upd_prog(None)
def get_breach_area(self):
"""
this would be very tricky to automate....
check:
SAGA 'fill sinks' for a simple dtm filler
SAGA 'Lake flood' for filling a dtm up to a specified depth
"""
raise Error('not implemented')
def get_wd(self):
wd = self.lineEdit_wd.text()
if wd is None or wd == '':
raise Error('need to specficy a Working Directory')
if not os.path.exists(wd):
os.makedirs(wd)
self.logger.info('built new working directory at:\n %s' % wd)
return wd
def wsl_clean(self, df_raw, tag='?'):
logger = self.logger.getChild('wsl_clean')
df = df_raw.copy()
#=======================================================================
# drop columns
#=======================================================================
"""not implemented"""
#any flagged columns
boolcol = df.columns.astype(str).str.startswith('~')
if np.any(boolcol):
logger.warning('dropping %i \'%s\' columns with \'~\' flag'%( boolcol.sum(), tag))
df1 = df.loc[:,~boolcol]
else:
df1 = df
#drop any coordinate columns
df1 = df1.drop(labels=['X','x','Y','y'], axis=1, errors='ignore')
#===================================================================
# headers
#===================================================================
#reformat columns
try:
df1.columns = df1.columns.astype(int) #reformat the aeps as ints
except:
raise Error('failed to recast columns as int: \n %s'%(df1.columns.tolist()))
#sort the columns
df2 = df1.reindex(columns = sorted(df1.columns))
#reformat values
df2 = df2.astype('float32')
#=======================================================================
# clean the user provided null
#=======================================================================
if not self.na_value is None:
boolar = df2.values == self.na_value
df2[boolar] = np.nan
logger.warning('for set %i user identified values to null with \'%s\''%
(boolar.sum().sum(), self.na_value))
"""not working for some reason
hp_pd.cleaner_report(df, df2)"""
logger.debug('cleaned to %s'%str(df2.shape))
return df2
def _get_from_cpar(
self, #special parameter extraction recognizing object's t ype
cpars,
sectName,
varName,
logger=None):
"""each parameter should exist on teh class instance.
we use this to set the type"""
if logger is None: logger = self.logger
log = logger.getChild('_get_from_cpar')
#=======================================================================
# get native type on class
#=======================================================================
assert hasattr(
self, varName), '\'%s\' does not exist on %s' % (varName, self)
#get class instance's native type
ntype = type(getattr(self, varName))
#==============================================================
# retrive and typeset (using native t ype)
#==============================================================
assert isinstance(cpars, configparser.ConfigParser)
csect = cpars[sectName]
pval_raw = csect[varName] #raw value (always a string)
#boolean
if ntype == bool:
pval = csect.getboolean(varName)
#no check or type conversion
elif getattr(self, varName) is None:
pval = pval_raw
#other types
else:
try:
pval = ntype(pval_raw)
except Exception as e:
raise Error(
'failed to set %s.%s with input \'%s\' (%s) to %s \n %s' %
(sectName, varName, pval_raw, type(pval_raw), ntype, e))
#=======================================================================
# blank set
#=======================================================================
"""seems like we're setup for ''.... not sure the value in switching everything over
if pval == '':
pval = np.nan"""
log.debug('retrieved \'%s.%s\'=\'%s\' w/ type: \'%s\'' %
(sectName, varName, pval, type(pval)))
return pval
def output_df(
self, #dump some outputs
df,
out_fn,
out_dir=None,
overwrite=None,
write_index=True,
logger=None,
):
#======================================================================
# defaults
#======================================================================
if out_dir is None: out_dir = self.out_dir
if overwrite is None: overwrite = self.overwrite
if logger is None: logger = self.logger
log = logger.getChild('output_df')
#======================================================================
# prechecks
#======================================================================
assert isinstance(
out_dir, str), 'unexpected type on out_dir: %s' % type(out_dir)
assert os.path.exists(
out_dir
), 'requested output directory doesnot exist: \n %s' % out_dir
assert isinstance(df, pd.DataFrame)
assert len(df) > 0, 'no data'
#extension check
if not out_fn.endswith('.csv'):
out_fn = out_fn + '.csv'
#output file path
out_fp = os.path.join(out_dir, out_fn)
#======================================================================
# checeks
#======================================================================
if os.path.exists(out_fp):
log.warning('file exists \n %s' % out_fp)
if not overwrite:
raise Error('file already exists')
#======================================================================
# writ eit
#======================================================================
df.to_csv(out_fp, index=write_index)
log.info('wrote to %s to filezzz: \n %s' % (str(df.shape), out_fp))
self.out_fp = out_fp #set for other methods
return out_fp
def output_fig(
self,
fig,
#file controls
out_dir=None,
overwrite=None,
fname=None, #filename
#figure write controls
fmt='svg',
transparent=True,
dpi=150,
logger=None,
):
#======================================================================
# defaults
#======================================================================
if out_dir is None: out_dir = self.out_dir
if overwrite is None: overwrite = self.overwrite
if logger is None: logger = self.logger
log = logger.getChild('output_fig')
#=======================================================================
# precheck
#=======================================================================
assert isinstance(fig, self.matplotlib.figure.Figure)
log.debug('on %s' % fig)
#======================================================================
# output
#======================================================================
#file setup
if fname is None:
try:
fname = fig._suptitle.get_text()
except:
fname = self.name
out_fp = os.path.join(out_dir, '%s.%s' % (fname, fmt))
if os.path.exists(out_fp): assert overwrite
#write the file
try:
fig.savefig(out_fp, dpi=dpi, format=fmt, transparent=transparent)
log.info('saved figure to file: %s' % out_fp)
except Exception as e:
raise Error('failed to write figure to file w/ \n %s' % e)
return out_fp
def initGui(self): #add UI elements to Qgis
"""
called on Qgis Load?
"""
log = self.logger.getChild('initGui')
#=======================================================================
# configure toolbar
#=======================================================================
"""Create the menu entries and toolbar icons inside the QGIS GUI."""
toolbar = self.iface.addToolBar(self.menu_name) #build a QToolBar
toolbar.setObjectName(self.menu_name)
#=======================================================================
# setup actions
#=======================================================================
self.actions_d = {'toolbar': dict(), 'menu': dict()}
cnt = 0
for attn, wrkr in self.dialogs_d.items():
try:
#build the icon
icon_fp = os.path.join(self.icon_dir, wrkr.icon_fn)
assert os.path.exists(icon_fp), 'bad filepath: %s' % icon_fp
icon = QIcon(icon_fp)
#assemble the action
action = QAction(icon, wrkr.icon_name, self.iface.mainWindow())
action.setObjectName(wrkr.icon_name)
action.setCheckable(False)
action.triggered.connect(wrkr.launch)
#add to the gui
if wrkr.icon_location == 'toolbar':
toolbar.addAction(action)
elif wrkr.icon_location == 'menu':
self.iface.addPluginToMenu(self.menu_name, action)
self.actions_d[wrkr.icon_location][attn] = action
cnt += 1
except Exception as e:
raise Error('failed to build action for \'%s\' w/ \n %s' %
(attn, e))
#wrap
self.toolbar = toolbar
log.debug('attached %i actions' % cnt)
def _agri(
self,
df_raw,
gname,
crve_d,
sourceCn='FunctionSource',
assetCn='Crop',
logger=None,
):
raise Error(
'not really sure how these curves work... skipping for now')
#=======================================================================
# defaults
#=======================================================================
if logger is None: logger = self.logger
log = logger.getChild(gname)
"""
view(df_raw)
"""
#=======================================================================
# meta update
#=======================================================================
crve_d.update({
'desc': crve_d['desc'] + ' for Agriculture',
'scale_var': 'building replacement cost',
'scale_units': 'monetary',
'impact_units': 'pct',
'impact_var': 'loss'
})
srceLib_d = dict()
for srce, gdf in df_raw.groupby(sourceCn):
crve_d['source'] = '%s, table=\'%s\' file=%s' % (srce, gname,
self.source_str)
#log.debug('%s w/\n%s'%(srce, gdf['Occupancy'].value_counts()))
srceLib_d[srce] = dict()
for crop, cdf in gdf.groupby(assetCn):
"""
view(cdf)
"""
print(srce, crop)
return dict()
def write_res(
self,
vlay,
out_dir=None, #directory for puts
names_d=None, #names conversion
rname_l=None,
):
log = self.logger.getChild('run')
#======================================================================
# defaults
#======================================================================
if names_d is None: names_d = self.names_d
if rname_l is None: rname_l = self.rname_l
if out_dir is None: out_dir = self.out_dir
res_name = vlay.name()
#======================================================================
# prechekss
#======================================================================
assert os.path.exists(out_dir), 'bad out_dir'
#======================================================================
# write data----------------
#======================================================================
#extract data
df = vlay_get_fdf(vlay)
#rename
if len(names_d) > 0:
df = df.rename(columns=names_d)
log.info('renaming columns: %s' % names_d)
#check the raster names
miss_l = set(rname_l).difference(df.columns.to_list())
if len(miss_l) > 0:
raise Error(
'failed to map %i raster layer names onto results: \n %s' %
(len(miss_l), miss_l))
out_fp = self.output_df(df,
'%s.csv' % res_name,
out_dir=out_dir,
write_index=False)
self.out_fp = out_fp
return
def treat_wetnull(self): #apply the wetnull_code algorhitim to the dry
#=======================================================================
# defaults
#=======================================================================
logger = self.logger.getChild('treat_wetnull')
dfwet = self.wsl_d['wet']
dfdry = self.wsl_d['dry']
dfwet_raw = dfwet.copy()
dfdry_raw = dfdry.copy()
#=======================================================================
# precheck
#=======================================================================
if self.db_f:
if np.any(pd.isnull(dfdry)):
logger.debug('nulls per column: \n%s'%pd.isnull(dfdry).sum(axis=0))
logger.debug('nulls per row: \n%s'%pd.isnull(dfdry).sum(axis=1))
raise Error('got %i null values for dfdry'%pd.isnull(dfdry).sum().sum())
#=======================================================================
# take _wet
#=======================================================================
if self.wetnull_code == 'take_dry':
#identify location of null values in the dry frame
boolar = pd.isnull(dfwet.values)
#replace all the null values with the value from dfwet
dfwet = dfwet.where(~boolar, other=dfdry)
logger.info('set %i values from the wet flood to the dry flood'%boolar.sum())
else: raise IOError
#=======================================================================
# reset into dict
#=======================================================================
self.wsl_d['wet'] = dfwet
self.wsl_d['dry'] = dfdry
return
def _check_finv(self,
logger=None): #check the finv and some paramter logic
"""
see also Model.check_finv() for data level checks
"""
if logger is None: logger = self.logger
log = logger.getChild('_check_finv')
#=======================================================================
# selection checks
#=======================================================================
assert not self.cid is None, 'must specify a valid cid'
assert isinstance(
self.finv_vlay,
QgsVectorLayer), 'must select a VectorLayer for the finv'
#=======================================================================
# data checks
#=======================================================================
#CRS
assert self.finv_vlay.crs() == self.qproj.crs(
), 'finv CRS (%s) does not match projects (%s)' % (
self.finv_vlay.crs(), self.qproj.crs())
#cid in the fields
fields_d = {f.name(): f for f in self.finv_vlay.fields()}
assert self.cid in fields_d, 'specified cid not found on finv'
#field type
assert 'int' in fields_d[self.cid].typeName().lower(), \
'cid field \'%s\' must be integer type not \'%s\''%(
self.cid, fields_d[self.cid].typeName())
#unique values
cid_ser = hlpr.Q.vlay_get_fdata(self.finv_vlay,
fieldn=self.cid,
fmt='df',
logger=log)
boolidx = cid_ser.duplicated(keep=False)
if boolidx.any():
log.debug('duplicated values \n%s' % cid_ser[boolidx])
raise Error(
'passed finv cid=\'%s\' values contain %i duplicates... see logger'
% (self.cid, boolidx.sum()))
def get_tests(
self, #generate the test suites
wFLow_l,
**kwargs):
log = self.logger.getChild('get_tests')
#=======================================================================
# defaults
#=======================================================================
#===========================================================================
# assemble suite
#===========================================================================
suite = unittest.TestSuite() #start the suite container
for fWrkr in wFLow_l:
#===================================================================
# setup the flow
#===================================================================
"""tests handle flows AFTER they've run
lets us run many tests on a completed flow without having to re-run teh flow each time"""
try:
runr = self._run_wflow(fWrkr, **kwargs)
except Exception as e:
raise Error('failed to execute %s w/ \n %s' %
(fWrkr.__name__, e))
runr.load_pick()
#build a test for each mathing method in the class
for tMethodName in TestLoader().getTestCaseNames(runr.Test):
"""inits TestU
only 1 test method per TeestU for now"""
suite.addTest(runr.Test(tMethodName, runr=runr))
runr.mstore.removeAllMapLayers()
#wrap
log.info(
'constructed test suite from %i flows w/ %i tests in %s\n \n' %
(len(wFlow_l), suite.countTestCases(),
datetime.datetime.now() - start))
return suite
def update_cf(self, #update one parameter control file
new_pars_d, #new paraemeters {section : {valnm : value }}
cf_fp = None):
log = self.logger.getChild('update_cf')
#get defaults
if cf_fp is None: cf_fp = self.cf_fp
assert os.path.exists(cf_fp), 'bad cf_fp: %s'%cf_fp
#initiliae the parser
pars = configparser.ConfigParser(allow_no_value=True)
_ = pars.read(cf_fp) #read it from the new location
#loop and make updates
for section, val_t in new_pars_d.items():
assert isinstance(val_t, tuple), '\"%s\' has bad subtype: %s'%(section, type(val_t))
assert section in pars, 'requested section \'%s\' not in the pars!'%section
for subval in val_t:
#value key pairs
if isinstance(subval, dict):
for valnm, value in subval.items():
pars.set(section, valnm, value)
#single values
elif isinstance(subval, str):
pars.set(section, subval)
else:
raise Error('unrecognized value type: %s'%type(subval))
#write the config file
with open(cf_fp, 'w') as configfile:
pars.write(configfile)
log.info('updated contyrol file w/ %i pars at :\n %s'%(
len(new_pars_d), cf_fp))
return
def check_binv_data(self, df):
#=======================================================================
# defaults
#=======================================================================
logger = self.logger.getChild('check_binv_data')
binv_df = self.parent.kids_d['binv'].childmeta_df
#null check
hp_pd.fancy_null_chk(df, detect='error', dname=self.name, logger=logger)
#length check
if not len(df) == len(binv_df):
raise Error('my data length (%i) does not match the binv length (%i)'%(len(df), len(binv_df)))
#check for index match
if not np.all(df.index == binv_df.index):
raise IOError
def to_curveset(
self,
df_raw,
bsmt_ht=None, #for combination curves,
nrpParkAD=215.0, #default $/m2 for NRP uderground parking
#metatdata default
metac_d={
'desc': 'rfda format converted to CanFlood format',
'location': 'Calgary and Edmonton, AB',
'date': 2014,
},
logger=None,
):
"""
converting rfda style residential + nrp curves into CanFlood
TODO: add for displacement stlye?
"""
#=======================================================================
# defaults
#=======================================================================
if logger is None: logger = self.logger
if bsmt_ht is None: bsmt_ht = self.bsmt_ht
log = logger.getChild('to_curveset')
#=======================================================================
# precheck
#=======================================================================
assert isinstance(df_raw, pd.DataFrame)
assert isinstance(bsmt_ht, float)
log.debug('on %s' % str(df_raw.shape))
#=======================================================================
# update the defaults
#=======================================================================
crve_d = self.crve_d.copy() #start with a copy
for k, v in {
**{
'source':
'CanFlood.%s_%s_%s' % (mod_name, self.tag,
datetime.datetime.today().strftime('%Y%m%d')),
#'bsmt_ht':bsmt_ht,
},
**metac_d
}.items():
crve_d[k] = v
#==============================================================================
# load
#==============================================================================
#drop the counts
df = df_raw.drop(0, axis=0)
#set the index
df = df.set_index(0)
df.index.name = 'cname'
#get the curve name prefix
df['cnp'] = df.index.str.slice(start=0, stop=2)
#set the dcount columns
df = df.rename(columns={1: 'dcount'})
#re-order the columns
boolcol = df.columns.isin(['dcount', 'cnp'])
df = df.loc[:, ~boolcol].join(df.loc[:, boolcol])
#==============================================================================
# convert residential tags
#==============================================================================
#identify the residentials
rboolidx = df.loc[:, 24].isin(['MC', 'MS', 'BC', 'BS'])
#build new index
df['nindex'] = df.loc[rboolidx, 'cnp'] + '_' + df.loc[rboolidx, 24]
df.loc[~rboolidx, 'nindex'] = df[~rboolidx].index
df['oindex'] = df.index
df = df.set_index('nindex', drop=True)
#ctype = df.loc[boolidx,24].to_dict() #get all the types
#==============================================================================
# create individuals--------------------
#==============================================================================
res_d = dict() #container for CanFlood function tabs
dd_set_d = dict() #container for all the depth damages
dd_set_d2 = dict()
boolar = df.columns.isin(['dcount', 'cnp', 'oindex'])
for cname, row in df.iterrows():
#==========================================================================
# set meta info
#==========================================================================
dcurve_d = crve_d.copy()
dcurve_d['tag'] = cname
#==========================================================================
# depth damage info
#==========================================================================
#get just depth damage
dd_ser = row[~boolar].dropna()
#identify depths (evens)
bool_dep = dd_ser.index.values % 2 == 0
#identiy damages
bool_dmg = np.invert(bool_dep)
#bundle depth:damage
dd_d = dict(
zip(dd_ser[bool_dep].tolist(), dd_ser[bool_dmg].tolist()))
#check for validty
if max(dd_d.values()) == 0:
print('%s has no damages! skipping' % cname)
#add it in
res_d[cname] = {**dcurve_d, **dd_d}
dd_set_d[cname] = dd_d #used below B+M
dd_set_d2[cname] = dd_d #used below S+C
print('added %s' % dcurve_d['tag'])
#==============================================================================
# create combined basement+mf----------------
#==============================================================================
#slice to this
boolidx = df.loc[:, 24].isin(['MC', 'MS', 'BC', 'BS'])
assert boolidx.any(
), 'unable to find expected curve type keys in column 24'
df_res = df.loc[boolidx, :].dropna(axis=1, how='all')
df_res = df_res.rename(columns={24: 'ctype'})
cnp_l = df_res.loc[:, 'cnp'].unique().tolist()
#loop and collect
res_bm_C_d = dict() #collect just these
res_bm_S_d = dict() #collect just these
for cnp in cnp_l:
#loop on structural and contents
for ctype in ('S', 'C'):
#get this
boolidx1 = np.logical_and(
df_res['cnp'] == cnp, #this class
df_res['ctype'].str.contains(ctype), #this ctype
)
#check it
if not boolidx1.sum() == 2:
raise IOError('unexpected count')
#======================================================================
# #collect by floor
#======================================================================
fdd_d = dict()
for floor in ('M', 'B'):
boolidx2 = np.logical_and(
boolidx1, df_res['ctype'].str.contains(floor))
if not boolidx2.sum() == 1:
raise IOError('unexpected count')
#get this dict
cname = df_res.index[boolidx2][0]
fdd_d[floor] = dd_set_d.pop(cname)
#======================================================================
# adjust basement
#======================================================================
#add bsmt_ht to all the basement
res_serf = pd.Series(fdd_d['B'])
if bsmt_ht > max(res_serf.index):
raise IOError(
'requested basement height %.2f out of range' %
bsmt_ht)
res_serf.index = res_serf.index - bsmt_ht
res_serf.index = res_serf.index.values.round(2)
#get max value
dmgmax = max(res_serf)
#drop all positives (basement cant have posiitive depths)
res_ser = res_serf[res_serf.index <= 0].sort_index(
ascending=True)
#set highest value to max
res_ser.loc[0] = dmgmax
#======================================================================
# assemble
#======================================================================
mf_ser = pd.Series(fdd_d['M']) + dmgmax
res_ser = res_ser.append(
mf_ser, ignore_index=False).sort_index(ascending=True)
#only take positive values
res_ser = res_ser[res_ser > 0]
#======================================================================
# set meta
#======================================================================
tag = '%s_%s' % (cnp, ctype)
dcurve_d = crve_d.copy()
dcurve_d['tag'] = tag
dcurve_d[
'desc'] = ' %s \nrfda converted and combined w/ bsmt_ht = %.2f, M+B ' % (
dcurve_d['desc'], bsmt_ht)
#add it in
res_d[tag] = {**dcurve_d, **res_ser.to_dict()}
if ctype == 'S':
res_bm_S_d[cnp] = res_ser
elif ctype == 'C':
res_bm_C_d[cnp] = res_ser
else:
raise Error('bad ctype')
print('added %s' % tag)
#======================================================================
# create combined mf+bsmt+S+C----------
#======================================================================
for cnp, Sser in res_bm_S_d.items():
Cser = res_bm_C_d[cnp]
assert np.array_equal(Sser.index, Cser.index), 'index mismatch'
assert not cnp in res_d, 'tag already taken'
#==================================================================
# cpombine
#==================================================================
res_ser = Cser + Sser
dcurve_d = crve_d.copy()
dcurve_d['tag'] = cnp
dcurve_d[
'desc'] = ' %s \nrfda converted and combined w/ bsmt_ht = %.2f, BC+BS+MC+MS' % (
dcurve_d['desc'], bsmt_ht)
res_d[cnp] = {**dcurve_d, **res_ser.to_dict()}
#======================================================================
# combine Contes + Struc
#======================================================================
for cnp in cnp_l:
for floor in ('B', 'M'):
dd_C_ser = dd_set_d2['%s_%sC' % (cnp, floor)]
dd_S_ser = dd_set_d2['%s_%sS' % (cnp, floor)]
res_ser = pd.Series(dd_C_ser) + pd.Series(dd_S_ser)
tag = '%s_%s' % (cnp, floor)
assert not tag in res_d
dcurve_d = crve_d.copy()
dcurve_d['tag'] = tag
dcurve_d[
'desc'] = ' %s \nrfda converted and combined w/ bsmt_ht = %.2f, C+S' % (
dcurve_d['desc'], bsmt_ht)
res_d[tag] = {**dcurve_d, **res_ser.to_dict()}
#=======================================================================
# NRP underground parking------------
#=======================================================================
tag = 'nrpUgPark'
dcurve_d = crve_d.copy()
dcurve_d['tag'] = tag
dcurve_d['desc'] = ' %s \nrfda underground parking fixed %.2f $/m2' % (
dcurve_d['desc'], nrpParkAD)
res_d[tag] = {
**dcurve_d,
**{
0: nrpParkAD,
10: nrpParkAD,
}
}
#==============================================================================
# convert and check
#==============================================================================
df_d = dict()
for cname, d in res_d.items():
self.check_crvd(d)
df_d[cname] = pd.Series(d).to_frame()
#======================================================================
# wrap
#======================================================================
log.info('finished w/ %i' % len(df_d))
return df_d
def output_set(
self, #output a collection of librarries
lib_d,
logger=None,
out_dir=None,
plot=True, #whether to also plot
):
#=======================================================================
# defaults
#=======================================================================
if logger is None: logger = self.logger
log = logger.getChild('output_set')
if out_dir is None: out_dir = self.out_dir
#=======================================================================
# loop and write
#=======================================================================
meta_d = dict()
for srce_raw, l1_d in lib_d.copy().items():
srce = srce_raw.replace('.', '').replace(' ', '')
od1 = os.path.join(out_dir, 'HAZUS_%s' % srce)
if not os.path.exists(od1): os.makedirs(od1)
meta_d[srce] = dict()
for gn1_raw, l2_d in l1_d.items():
gn1 = gn1_raw[2:]
meta_d[srce][gn1] = dict()
#check subtypes
for k, df in l2_d.items():
if not isinstance(df, pd.DataFrame):
raise Error('bad type on %s' % gn1)
#setup the filename
ofn = '%s_%s_%s' % (self.libName, srce, gn1)
ofn = ofn.replace(' ', '')
ofn = ofn.replace('.', '')
#do the write
meta_d[srce][gn1]['xls_fp'] = self.output(l2_d,
ofn=ofn + '.xls',
out_dir=od1)
#===========================================================================
# plots
#===========================================================================
if plot:
fig = self.plotAll(
l2_d,
title='%s_%s_%s' % (self.libName, srce, gn1),
lib_as_df=True,
xlim=(0, 100),
)
meta_d[srce][gn1]['fig_fp'] = self.output_fig(fig,
out_dir=od1,
fname=ofn)
log.info('wrote %i sets' % (len(meta_d)))
return meta_d
