def worker(jobq,pid): while True: args = jobq.get() if args == None: break dname = args[1][0].split('.')[0].split('\\')[-1] smp = pu.smp(args[1][0],load=True) rtype = smp.records.keys()[0] smp.records[dname] = smp.records[smp.records.keys()[0]] smp.records.pop(rtype) for smpname in args[1][1:]: smp2 = pu.smp(smpname,load=True) rtype = smp2.records.keys()[0] smp2.records[dname] = smp2.records[smp2.records.keys()[0]] smp2.records.pop(rtype) smp.merge(smp2,how='left') print '----'+args[0] smp.save(args[0]) jobq.task_done() jobq.task_done() return
import numpy from datetime import datetime import pandas import pestUtil reload(pestUtil) #--load a single site smp = pestUtil.smp('heads_gap.out') #smp = pestUtil.smp('s20_h_dbkey13037.smp',date_fmt='%m/%d/%Y') site_names,offset_idx = smp.get_unique_from_file(smp.site_index,needindices=True) #--load only one site site_0 = smp.load(site=site_names[0]) #--load a single site with pandas smpp = pestUtil.smp('heads_gap.out',pandas=True) #smp = pestUtil.smp('s20_h_dbkey13037.smp',date_fmt='%m/%d/%Y') site_names,offset_idx = smpp.get_unique_from_file(smp.site_index,needindices=True) #--load only one site site_0p = smpp.load(site=site_names[0]) #--load all sites smp2 = pestUtil.smp('heads_gap.out',load=True) #smp2 = pestUtil.smp('s20_h_dbkey13037.smp',date_fmt='%m/%d/%Y',load=True) site_0,dates,vals = smp2.get_site(site_names[0]) #--load all sites with pandas smp2p = pestUtil.smp('heads_gap.out',load=True,pandas=True)
on_date += timedelta(days=1.) plot_dates.append(on_date) plot_dates = np.array(plot_dates) #--initialize figure and figure/plot counters ifigure = 1 iplot = 1 nplots = 6 fig = Make_NewFigure() #--matplotlib date specification years, months = mdates.YearLocator(), mdates.MonthLocator( ) #every year, every month yearsFmt = mdates.DateFormatter('%Y') #--open smp file smp = pestUtil.smp(ObsFile, load=False, date_fmt='%m/%d/%Y') #--process each station in the xml file iwell = 0 pct_lay = np.empty((nlay), np.float) active_gwobs = [] active_gwobs_stats = [] for idx, gwhead in enumerate(root.findall('gwHead')): station = gwhead.attrib['name'] print 'Locating {0} in the model grid'.format(station) coordType = gwhead.find('coordType').text #--determine the location of the well if coordType.lower() == 'model': #--convert to zero based irow = int(gwhead.find('row').text) - 1 icol = int(gwhead.find('column').text) - 1 elif coordType.lower() == 'site':
import numpy as np import pylab import pandas import pestUtil as pu from bro_pred import flow smp = pu.smp('noaa_sp.smp',load=True,pandas=True) df = smp.records df = df.resample('1M',how='mean') #--use the last value in the dataframe as the base for the projections start_value = df['noaa'][-1] print start_value low_rate = 0.015 #9 inches in 50 years med_rate = 0.0275 #16.5 inches in 50 years high_rate = 0.05 #24 inches in 50 years df_slr = pandas.DataFrame({'no_rise':start_value},index=flow.sp_end) nyears = df_slr.index.year - df_slr.index[0].year df_slr['low_rise'] = start_value + low_rate * nyears df_slr['high_rise'] = start_value + high_rate * nyears df_slr['med_rise'] = start_value + med_rate * nyears df_slr.to_csv('noaa_slr.csv',index_label='datetime') df_slr.plot() pylab.show()
sim_mult = 1.0 c = swflow.find('simmult') if c != None: sim_mult = float( c.text ) print 'Processing flow data for...{0} - reach {1}->{2}'.format( station, usreach, dsreach ) obs.fill( no_data ) processObs = False nobs = 0 child = swflow.find('ObsItems') if child != None: processObs = True for childitem in child.findall('ObsFile'): print 'processing observation data...{0}'.format(childitem.text) #--get the smp file data smpstation = os.path.basename(childitem.text).replace('.smp','') smp = pestUtil.smp(childitem.text,load=True,date_fmt='%m/%d/%Y') nobs, obs = dataAdd(unit_conv,sim_dates,obs,smp.records[smpstation][:,0],smp.records[smpstation][:,1]) # obs *= unit_conv #--read the simulated data if dsreach > 0: #SWRObj = mfb.SWR_Record(-2,SWRFlowFile) #SWRObj.rewind_file() #ce1 = SWRObj.get_gage(rec_num=usreach,iconn=dsreach) ce1 = SWRObj.get_time_gage(rec_num=usreach,iconn=dsreach) nt = np.shape(ce1)[0] sim = np.zeros( (nt,2), np.float ) for jdx in xrange(0,nt): sim[jdx,0] = ce1[jdx,itime] sim[jdx,1] = ce1[jdx,iflow] * unit_conv * -1. * sim_mult #--plot the data ax = fig.add_subplot(nplots,1,iplot)
import numpy as np import pylab import pestUtil as pu #--observation smps obs_files = ['UMD.01\\obsref\\head\heads.smp'] mod_files= ['UMD.01\\modref\\head\mheads.smp'] plt_dir = 'png\\' for obs_file,mod_file in zip(obs_files,mod_files): obs_smp = pu.smp(obs_file,load=True,date_fmt='%m/%d/%Y') mod_smp = pu.smp(mod_file,load=True,date_fmt='%m/%d/%Y') sites = obs_smp.records.keys() for site in sites: obs = obs_smp.records[site] mod = mod_smp.records[site] fig = pylab.figure(figsize=(5,5)) ax = pylab.subplot(111) ax.plot(obs[:,0],obs[:,1],'b-',label='obs') ax.plot(mod[:,0],mod[:,1],'g-',label='mod') ax.grid() ax.legend() fname = plt_dir+site+'.png' pylab.savefig(fname,dpi=300,format='png',bbox_inches='tight') pylab.close('all') #break
import os import numpy as np #from dateutil.relativedelta import relativedelta from datetime import timedelta import pandas from shapely.geometry import Point import shapefile import pestUtil as pu from bro import seawat '''spatailly declusters relconc data and thin out site that are not in the active domain ''' #--load the relconc smp for spatial declustering df = pu.smp('relconc.smp', load=True, pandas=True).records #--get the nwis site no and locations nwis_shapename = '..\\_gis\\scratch\\broward_nwis_gw_conc_depth' shapes, records = shapefile.load_as_dict(nwis_shapename) nwis_sitenos = records['site_no'] nwis_icbnd = records['icbnd'] #nwis_names = [] #for n in records['station_nm']: # n = n.replace(' ','') # n = n.replace(',','') # nwis_names.append(n) nwis_pts = [] for shape in shapes: pt = Point(shape.points[0]) nwis_pts.append(pt)
from datetime import datetime import pandas import shapefile import pestUtil as pu from bro import flow #--get the dbhydro sampled monthly series db_dir = '..\\_dbhydro\\stressperiod_stage_smp_navd\\' db_files = os.listdir(db_dir) #db_names = [] db_dict = {} for dfile in db_files: dname = dfile.split('.')[0].upper().strip() #db_names.append(dname) smp = pu.smp(db_dir+dfile,load=True,pandas=True) db_dict[dname] = copy.deepcopy(smp.records) #--get the coastal stage record - sampled to stress periods noaa_file = '..\\_noaa\\noaa_sp.smp' noaa_smp = pu.smp(noaa_file,load=True,pandas=True) noaa_df = noaa_smp.records #--load the reach shapefile from swrpre swr_shapename = '..\\_gis\\scratch\\sw_reaches_conn_SWRpolylines_2' shp = shapefile.Reader(swr_shapename) fnames = shapefile.get_fieldnames(swr_shapename) ibnd_idx,stg_idx,reach_idx = 16,17,22 stg_dict = {} m_range = flow.sp_end
# e = t - d # elevs.append(e) # except: # elevs.append(None) d = None if hd != '': d = float(hd) elif wd != '': d = float(wd) if d != None: e = t - d elevs.append(e) else: elevs.append(None) #--load the decluster dataframe smp = pu.smp('..\\..\\_nwis\\navd_declustered.smp', load=True, pandas=True) df = smp.records f = open('..\\..\\_nwis\\nwis_navd_bore_coords.dat', 'w', 0) f_miss = open('..\\..\\_nwis\\navd.missing', 'w', 0) wr = shapefile.writer_like(shapename) wr.field('fixed_name', fieldType='C', size=50) for siteno in df.keys(): if siteno in sitenos: idx = sitenos.index(siteno) name = fixed[idx] r = rows[idx] c = cols[idx] g = geom[:, int(float(r)) - 1, int(float(c)) - 1] x = xs[idx] y = ys[idx]
import pylab import pestUtil as pu smp_obs = pu.smp(fname='UMD.01\\obsref\\head\\heads.smp',load=True,date_fmt='%m/%d/%Y') #smp_mod = pu.smp(fname='UMD.01\\modref\\head\\mheads.smp',load=True,date_fmt='%m/%d/%Y') smp_filt = pu.load_smp_from_tsproc('processed_mod_vs_obs_biweek_filtered.dat') #--get a list of 'observation' series names osites = [] for site in smp_filt.records.keys(): if site.endswith('_o') and site not in osites: osites.append(site) plt_dir = 'png\\filt' for site in osites: print site filt_obs = smp_filt.records[site] raw_obs = smp_obs.records[site[:-2].upper()] filt_mod = smp_filt.records[site[:-2]] #raw_mod = smp_mod.records[site[:-2].upper()] #fig = pylab.figure(figsize=(5,5)) fig = pylab.figure() ax = pylab.subplot(111) #ax.plot(filt_obs[:,0],filt_obs[:,1],'b.',color='b') ax.plot(filt_obs[:,0],filt_obs[:,1],'b-',label='processed',lw=0.5,color='b') #ax.plot(filt_mod[:,0],filt_mod[:,1],'g.',color='g') #ax.plot(filt_mod[:,0],filt_mod[:,1],'g-',label='modfilt',lw=0.5,color='g') ax.plot(raw_obs[:,0],raw_obs[:,1],'g-',label='native',lw=0.5,color='g')
import os import pestUtil smp_dir = 'raw_smp\\' smp_files = os.listdir(smp_dir) for smp_file in smp_files: print smp_file smp = pestUtil.smp(smp_dir+smp_file,load=True) smp.make_unique() smp.save('test\\'+smp_file)
f = open(stg_reach_file, 'r') reach_dict = {} header = f.readline() for line in f: raw = line.strip().split(',') name = raw[0].upper().replace(' ', '_').replace('-', '') if name.endswith('W'): name = name[:-1] reach_dict[name] = int(raw[1]) f.close() #parser = lambda x: datetime.strptime(x,tc.DATE_FMT+' %H:%M:%S') #stage_df = pandas.read_table(stg_obs_file,header=None,parse_dates=[[1,2]],date_parser=parser,sep='\s*') #stage_df.columns = ['datetime','site','value'] stage_smp = pu.smp(stg_obs_file, date_fmt=tc.DATE_FMT, pandas=True, load=True) stage_sites = stage_smp.records.keys() for site in stage_sites: if site not in reach_dict.keys(): print 'site not found in reach dict', site obs_names = [] mod_names = [] reach_numbers = [] smp_site_names = [] for i, site in enumerate(reach_dict.keys()): if site not in stage_sites: print 'site not found in smp file', site reach_dict.pop(site) else: obs_names.append('ost_{0:03.0f}or'.format(i + 1))
raise Exception('Duplicate names: '+n) fixed.append(n) #--build sample elevation list elevs = [] for r,c,hd,wd in zip(rows,cols,hdepths,wdepths): t = geom[0,int(float(r))-1,int(float(c))-1] try: d = float(hd) except: d = float(wd) e = t - d elevs.append(e) #--load the thinned dataframe smp = pu.smp('..\\..\\_nwis\\relconc_thinned.smp',load=True,pandas=True) df = smp.records f = open('..\\..\\_nwis\\nwis_bore_coords.dat','w',0) wr = shapefile.writer_like(shapename) wr.field('fixed_name',fieldType='C',size=50) for siteno in df.keys(): if siteno in sitenos: idx = sitenos.index(siteno) name = fixed[idx] r = rows[idx] c = cols[idx] g = geom[:,int(float(r))-1,int(float(c))-1] x = xs[idx] y = ys[idx]
raise Exception('Duplicate names: ' + n) fixed.append(n) #--build sample elevation list elevs = [] for r, c, hd, wd in zip(rows, cols, hdepths, wdepths): t = geom[0, int(float(r)) - 1, int(float(c)) - 1] try: d = float(hd) except: d = float(wd) e = t - d elevs.append(e) #--load the decluster dataframe smp = pu.smp('..\\..\\_nwis\\relconc_declustered.smp', load=True, pandas=True) df = smp.records f = open('..\\..\\_nwis\\nwis_conc_bore_coords.dat', 'w', 0) wr = shapefile.writer_like(shapename) wr.field('fixed_name', fieldType='C', size=50) for siteno in df.keys(): if siteno in sitenos: idx = sitenos.index(siteno) name = fixed[idx] r = rows[idx] c = cols[idx] g = geom[:, int(float(r)) - 1, int(float(c)) - 1] x = xs[idx] y = ys[idx] e = elevs[idx]
''' K = 200.0 L = 500.0 A = 500.0 * 100.0 cond = K * A / L print 'loading north,south model-aligned dataframe and interpolation factors' df_ns_data = pandas.read_csv( '..\\..\\_nwis\\dataframes\\ghb_NS_stages_model.csv', index_col=0, parse_dates=True) df_ns_fac = pandas.read_csv('ghb_NS_factors.dat', index_col=[0, 1]) #--process stage record for coastal GHBs - presampled noaa_smp = pu.smp('..\\..\\_noaa\\noaa_sp.smp', load=True) coastal_stages = noaa_smp.records['noaa'][:, 1] #--process stage records for WCA ghbs - these records should have been presampled to stress period dimensions print 'processing EDEN records' smp_dir = '..\\..\\_eden\\stage_smp_full\\' smp_files = os.listdir(smp_dir) eden_ids = [] eden_sp_vals = {} for sfile in smp_files: print 'loading smp file', sfile, '\r', eden_id = int(sfile.split('.')[0]) eden_ids.append(eden_id) smp = pu.smp(smp_dir + sfile, load=True) rec = smp.records[str(eden_id)] #--some defense
import pandas import pestUtil as pu import bro #--model stress period date range m_range = bro.sp_end #--merged daily smp files smp_dir = 'daily_stage_smp_navd\\' smp_files = os.listdir(smp_dir) out_dir = 'stressperiod_stage_smp_navd\\' dfs = [] for i,sfile in enumerate(smp_files): print 'processing ',sfile,i,' of ',len(smp_files),'\r', depname = sfile.split('.')[0] smp = pu.smp(smp_dir+sfile,load=True,pandas=True) df = smp.records rname = df.keys()[0] df[depname] = df[rname] df.pop(rname) df = df.astype(float) df = df.resample(bro.pandas_freq,how=np.mean) #--create dataframe that is aligned with model stress periods #--merge in the record and fill with 0.0 df_mn = pandas.DataFrame({depname:np.NaN},index=m_range) df_mn = df_mn.combine_first(df) df_mn = df_mn[bro.start:bro.end] df_mn = df_mn.dropna() dfs.append(df_mn) smp.records = df_mn
import numpy as np import pylab import pestUtil as pu #--observation smps obs_files = ['UMD.01\\obsref\\head\heads.smp'] mod_files = ['UMD.01\\modref\\head\mheads.smp'] plt_dir = 'png\\' for obs_file, mod_file in zip(obs_files, mod_files): obs_smp = pu.smp(obs_file, load=True, date_fmt='%m/%d/%Y') mod_smp = pu.smp(mod_file, load=True, date_fmt='%m/%d/%Y') sites = obs_smp.records.keys() for site in sites: obs = obs_smp.records[site] mod = mod_smp.records[site] fig = pylab.figure(figsize=(5, 5)) ax = pylab.subplot(111) ax.plot(obs[:, 0], obs[:, 1], 'b-', label='obs') ax.plot(mod[:, 0], mod[:, 1], 'g-', label='mod') ax.grid() ax.legend() fname = plt_dir + site + '.png' pylab.savefig(fname, dpi=300, format='png', bbox_inches='tight') pylab.close('all') #break
def main(): #--instance tsproc_infile = 'tsproc_setup.dat' tsp = tc.tsproc(tsproc_infile,out_file='processed.dat',out_fmt='long',) if os.path.exists('pest.pst'): os.remove('pest.pst') pest_mblocks,pest_oblocks = [],[] #--------------------------- #--heads #-------------------------- print 'procesing heads' ohead_name = 'smp\\obs\\hds.smp' mhead_name = 'smp\\mod\\hds.smp' #shutil.copy2(mhead_name,ohead_name) ohead = pu.smp(ohead_name,load=False,site_index=True) mhead = pu.smp(mhead_name,load=False,site_index=True) #--some model sites are not included - thin them out keep_mlist = [] names = [] mlist = list(mhead.site_list) olist = list(ohead.site_list) for site in mlist: if site in olist: keep_mlist.append(site) else: print 'missing mhead site',site #missing.append(site) oblocks = tsp.get_mul_series_ssf(keep_mlist,ohead_name,suffix='oh',\ role="final",context=tc.PEST_CONTEXT) mblocks = tsp.get_mul_series_ssf(keep_mlist,mhead_name,suffix='sh') renamed = tsp.copy_2_series(mblocks,keep_mlist,role='final',wght=1.0) pest_oblocks.extend(oblocks) pest_mblocks.extend(renamed) print 'build a list of template and model-equivalent files' tpl_dir = 'tpl\\' modin_dir = os.path.join("model","ref") tpl_files,modin_files = [],[] files = os.listdir(tpl_dir) for f in files: modin_file = f.replace(".tpl",'') modin_files.append(os.path.join(modin_dir,modin_file)) tpl_file = os.path.join(tpl_dir,f) tpl_files.append(tpl_file) par_file = os.path.join("misc","par.info") grp_file = os.path.join("misc","grp.info") #--write the model run tspoc file print 'writing model run tsproc file' tsp.set_context('model_run') tsp.tsproc_file = 'misc\\tsproc_model_run.dat' tsp.write_tsproc() #--write the setup tsproc file print 'writing setup tsproc file' tsp.write_pest(tpl_files,modin_files,pest_oblocks,pest_mblocks,\ svd=True,parms=par_file,parm_grp=grp_file,\ model_cmd="model.bat") tsp.set_context(tc.PEST_CONTEXT) tsp.tsproc_file = 'misc\\tsproc_setup.dat' tsp.write_tsproc() f = open(os.path.join('misc','tsproc_setup.in'),'w') f.write(os.path.join('misc','tsproc_setup.dat')+'\n'+\ os.path.join('misc','tsproc_setup.out')+\ '\ny\ny\n') f.close() f = open(os.path.join('misc','tsproc_model_run.in'),'w') f.write(os.path.join('misc','tsproc_model_run.dat')+'\n'+\ os.path.join('misc','tsproc_model_run.out')+'\ny\ny\n') f.close() print 'running tsproc' os.system(os.path.join("exe",'tsproc.exe')+' <'+os.path.join("misc","tsproc_setup.in")+\ ' >'+os.path.join("misc","tsproc_screen.out"))
raise Exception('Duplicate names: '+n) fixed.append(n) #--build sample elevation list elevs = [] for r,c,hd,wd in zip(rows,cols,hdepths,wdepths): t = geom[0,int(float(r))-1,int(float(c))-1] try: d = float(hd) except: d = float(wd) e = t - d elevs.append(e) #--load the decluster dataframe smp = pu.smp('..\\..\\_nwis\\relconc_declustered.smp',load=True,pandas=True) df = smp.records f = open('..\\..\\_nwis\\nwis_conc_bore_coords.dat','w',0) wr = shapefile.writer_like(shapename) wr.field('fixed_name',fieldType='C',size=50) for siteno in df.keys(): if siteno in sitenos: idx = sitenos.index(siteno) name = fixed[idx] r = rows[idx] c = cols[idx] g = geom[:,int(float(r))-1,int(float(c))-1] x = xs[idx] y = ys[idx] e = elevs[idx]
raise Exception('Duplicate names: ' + n) fixed.append(n) #--build sample elevation list elevs = [] for r, c, hd, wd in zip(rows, cols, hdepths, wdepths): t = geom[0, int(float(r)) - 1, int(float(c)) - 1] try: d = float(hd) except: d = float(wd) e = t - d elevs.append(e) #--load the thinned dataframe smp = pu.smp('..\\..\\_nwis\\relconc_thinned.smp', load=True, pandas=True) df = smp.records f = open('..\\..\\_nwis\\nwis_bore_coords.dat', 'w', 0) wr = shapefile.writer_like(shapename) wr.field('fixed_name', fieldType='C', size=50) for siteno in df.keys(): if siteno in sitenos: idx = sitenos.index(siteno) name = fixed[idx] r = rows[idx] c = cols[idx] g = geom[:, int(float(r)) - 1, int(float(c)) - 1] x = xs[idx] y = ys[idx] e = elevs[idx]
import os import numpy as np import pylab import shapefile import pestUtil as pu #smp_dir = 'pws_smp\\' #smp_files = os.listdir(smp_dir) #smp_files = os.listdir(smp_dir)#convert = 7.481/24.0/60.0 #to gpm smp_dir ='.\\' smp_files = ['sum.smp'] convert = 7.481/1.0e6/30 #to mgd - approx 30 days in a month plt_dir = 'png\\' f_out = open('missing.dat','w') for smp_file in smp_files: print smp_file smp = pu.smp(smp_dir+smp_file,load=True) for site,record in smp.records.iteritems(): smp.records[site][:,1] *= convert ax = smp.plot(None) pylab.show() pass
date_fmt = '%m/%d/%Y' smp_dir = 'UMD.01\\obsref\\head\\' smp_files = os.listdir(smp_dir) start = datetime(1996, 1, 1, 12) end = datetime(2004, 12, 31, 12) site_names = [] f = open('misc\\bore_coords.dat', 'r') for line in f: site_names.append(line.split()[0].strip().upper()) f.close() obs_smp = pu.smp('UMD.01\\obsref\\head\ALL_NWIS_GW.smp', date_fmt=date_fmt, load=True) filt_smp = pu.smp('UMD.01\\obsref\\head\heads.smp', date_fmt=date_fmt) f = open('misc\\missing_sites.dat', 'w') for site in obs_smp.records.keys(): #drange = obs_smp.get_daterange(site_name=site) if site.replace('-', '').strip().upper() in site_names: filt_smp.records[site.replace('-', '')] = obs_smp.records[site] else: print 'missing site', site f.write('missing site ' + site + '\n') f.close() filt_smp.set_daterange(start, end)
import os import pandas import shapefile import pestUtil as pu #--get a list of site that are within the broward domain shape_name = '..\\_gis\\scratch\\broward_nwis_gw_conc_depth' records = shapefile.load_as_dict(shape_name,loadShapes=False) sitenos = records['site_no'] dir_dict = {'chl':'smp_rel_conc_chl\\','cond':'smp_rel_conc_regressed\\','tds':'smp_rel_conc_tds\\'} df_dict = {} for dtype,smp_dir in dir_dict.iteritems(): files = os.listdir(smp_dir) dfs = [] for f in files: raw = f.split('.') siteno,sitename = raw[1],raw[2] if siteno in sitenos: print dtype,f smp = pu.smp(smp_dir+f,load=True,pandas=True) df = smp.records df[siteno] = df['site'] df.pop('site') dfs.append(df) df = pandas.concat(dfs,axis=1) df.to_csv('dataframes\\'+dtype+'.csv',index_label='datetime')
import numpy as np import pylab import pandas import pestUtil as pu from bro_pred import flow smp = pu.smp('noaa_sp.smp', load=True, pandas=True) df = smp.records df = df.resample('1M', how='mean') #--use the last value in the dataframe as the base for the projections start_value = df['noaa'][-1] print start_value low_rate = 0.015 #9 inches in 50 years med_rate = 0.0275 #16.5 inches in 50 years high_rate = 0.05 #24 inches in 50 years df_slr = pandas.DataFrame({'no_rise': start_value}, index=flow.sp_end) nyears = df_slr.index.year - df_slr.index[0].year df_slr['low_rise'] = start_value + low_rate * nyears df_slr['high_rise'] = start_value + high_rate * nyears df_slr['med_rise'] = start_value + med_rate * nyears df_slr.to_csv('noaa_slr.csv', index_label='datetime') df_slr.plot() pylab.show()
import os import numpy as np import pylab import pandas import pestUtil as pu obs_dir = "smp\\obs\\" mod_dir = "smp\\mod\\" obs_files, mod_files = os.listdir(obs_dir), os.listdir(mod_dir) obs_color, mod_color = "b", "g" plt_dir = "png\\obs_vs_sim\\" for ofile in obs_files: assert ofile in mod_files, ofile + " not found in model smp files" osmp = pu.smp(obs_dir + ofile, load=True, pandas=True) msmp = pu.smp(mod_dir + ofile, load=True, pandas=True) odf = osmp.records mdf = msmp.records # --plotting if "NAVD" in ofile.upper(): ylabel = "water level $ft NAVD$" ylim = [-10.0, 10.0] plt_prefix = plt_dir + "navd_" else: ylabel = "relative concetration" ylim = [-0.1, 1.0] plt_prefix = plt_dir + "conc_" for site in odf.keys(): print site
date_fmt = '%m/%d/%Y' smp_dir = 'UMD.01\\obsref\\head\\' smp_files = os.listdir(smp_dir) start = datetime(1996,1,1,12) end = datetime(2004,12,31,12) site_names = [] f = open('misc\\bore_coords.dat','r') for line in f: site_names.append(line.split()[0].strip().upper()) f.close() obs_smp = pu.smp('UMD.01\\obsref\\head\ALL_NWIS_GW.smp',date_fmt=date_fmt,load=True) filt_smp = pu.smp('UMD.01\\obsref\\head\heads.smp',date_fmt=date_fmt) f = open('misc\\missing_sites.dat','w') for site in obs_smp.records.keys(): #drange = obs_smp.get_daterange(site_name=site) if site.replace('-','').strip().upper() in site_names: filt_smp.records[site.replace('-','')] = obs_smp.records[site] else: print 'missing site',site f.write('missing site '+site+'\n') f.close() filt_smp.set_daterange(start,end) filt_smp.save('UMD.01\\obsref\\head\heads.smp')
import pylab import pestUtil as pu smp_obs = pu.smp(fname='UMD.01\\obsref\\head\\heads.smp', load=True, date_fmt='%m/%d/%Y') #smp_mod = pu.smp(fname='UMD.01\\modref\\head\\mheads.smp',load=True,date_fmt='%m/%d/%Y') smp_filt = pu.load_smp_from_tsproc('processed_mod_vs_obs_biweek_filtered.dat') #--get a list of 'observation' series names osites = [] for site in smp_filt.records.keys(): if site.endswith('_o') and site not in osites: osites.append(site) plt_dir = 'png\\filt' for site in osites: print site filt_obs = smp_filt.records[site] raw_obs = smp_obs.records[site[:-2].upper()] filt_mod = smp_filt.records[site[:-2]] #raw_mod = smp_mod.records[site[:-2].upper()] #fig = pylab.figure(figsize=(5,5)) fig = pylab.figure() ax = pylab.subplot(111) #ax.plot(filt_obs[:,0],filt_obs[:,1],'b.',color='b') ax.plot(filt_obs[:, 0], filt_obs[:, 1], 'b-', label='processed',
import numpy from datetime import datetime import pandas import pestUtil reload(pestUtil) #--load a single site smp = pestUtil.smp('heads_gap.out') #smp = pestUtil.smp('s20_h_dbkey13037.smp',date_fmt='%m/%d/%Y') site_names, offset_idx = smp.get_unique_from_file(smp.site_index, needindices=True) #--load only one site site_0 = smp.load(site=site_names[0]) #--load a single site with pandas smpp = pestUtil.smp('heads_gap.out', pandas=True) #smp = pestUtil.smp('s20_h_dbkey13037.smp',date_fmt='%m/%d/%Y') site_names, offset_idx = smpp.get_unique_from_file(smp.site_index, needindices=True) #--load only one site site_0p = smpp.load(site=site_names[0]) #--load all sites smp2 = pestUtil.smp('heads_gap.out', load=True) #smp2 = pestUtil.smp('s20_h_dbkey13037.smp',date_fmt='%m/%d/%Y',load=True) site_0, dates, vals = smp2.get_site(site_names[0]) #--load all sites with pandas smp2p = pestUtil.smp('heads_gap.out', load=True, pandas=True) #smp2 = pestUtil.smp('s20_h_dbkey13037.smp',date_fmt='%m/%d/%Y',load=True) site_0, datesp, valsp = smp2p.get_site(site_names[0])
from datetime import datetime import pandas import shapefile import pestUtil as pu from bro import flow #--get the dbhydro sampled monthly series db_dir = '..\\_dbhydro\\stressperiod_stage_smp_navd\\' db_files = os.listdir(db_dir) #db_names = [] db_dict = {} for dfile in db_files: dname = dfile.split('.')[0].upper().strip() #db_names.append(dname) smp = pu.smp(db_dir + dfile, load=True, pandas=True) db_dict[dname] = copy.deepcopy(smp.records) #--get the coastal stage record - sampled to stress periods noaa_file = '..\\_noaa\\noaa_sp.smp' noaa_smp = pu.smp(noaa_file, load=True, pandas=True) noaa_df = noaa_smp.records #--load the reach shapefile from swrpre swr_shapename = '..\\_gis\\scratch\\sw_reaches_conn_SWRpolylines_2' shp = shapefile.Reader(swr_shapename) fnames = shapefile.get_fieldnames(swr_shapename) ibnd_idx, stg_idx, reach_idx = 16, 17, 22 stg_dict = {} m_range = flow.sp_end
# e = t - d # elevs.append(e) # except: # elevs.append(None) d = None if hd != '': d = float(hd) elif wd != '': d = float(wd) if d != None: e = t - d elevs.append(e) else: elevs.append(None) #--load the decluster dataframe smp = pu.smp('..\\..\\_nwis\\navd_declustered.smp',load=True,pandas=True) df = smp.records f = open('..\\..\\_nwis\\nwis_navd_bore_coords.dat','w',0) f_miss = open('..\\..\\_nwis\\navd.missing','w',0) wr = shapefile.writer_like(shapename) wr.field('fixed_name',fieldType='C',size=50) for siteno in df.keys(): if siteno in sitenos: idx = sitenos.index(siteno) name = fixed[idx] r = rows[idx] c = cols[idx] g = geom[:,int(float(r))-1,int(float(c))-1] x = xs[idx] y = ys[idx]
import os import numpy as np import pylab import shapefile import pestUtil as pu #smp_dir = 'pws_smp\\' #smp_files = os.listdir(smp_dir) #smp_files = os.listdir(smp_dir)#convert = 7.481/24.0/60.0 #to gpm smp_dir = '.\\' smp_files = ['sum.smp'] convert = 7.481 / 1.0e6 / 30 #to mgd - approx 30 days in a month plt_dir = 'png\\' f_out = open('missing.dat', 'w') for smp_file in smp_files: print smp_file smp = pu.smp(smp_dir + smp_file, load=True) for site, record in smp.records.iteritems(): smp.records[site][:, 1] *= convert ax = smp.plot(None) pylab.show() pass
import os import numpy as np #from dateutil.relativedelta import relativedelta from datetime import timedelta import pandas from shapely.geometry import Point import shapefile import pestUtil as pu from bro import seawat '''spatailly declusters relconc data and thin out site that are not in the active domain sites that are missing depth info should not be in the shapefile ''' #--load the relconc smp for spatial declustering df = pu.smp('relconc.smp',load=True,pandas=True).records #--get the nwis site no and locations nwis_shapename = '..\\_gis\\scratch\\broward_nwis_gw_conc_depth' shapes,records = shapefile.load_as_dict(nwis_shapename) nwis_sitenos = records['site_no'] nwis_ibnd = records['icbnd'] #nwis_names = [] #for n in records['station_nm']: # n = n.replace(' ','') # n = n.replace(',','') # nwis_names.append(n) nwis_pts = [] for shape in shapes: pt = Point(shape.points[0])
site_salt_idxs[site_salt_params.index('chloride')] ] if 'dissolved' in site_salt_params and 'chloride' in site_salt_params: tds_chl_idxs[site_no] = [ site_salt_idxs[site_salt_params.index('dissolved')], site_salt_idxs[site_salt_params.index('chloride')] ] #--load the paired salt files and look for data on the same day spec_data, chl_data = [], [] dfs = [] site_nos = [] for site_no, idxs in spec_chl_idxs.iteritems(): print 'loading smp files:', smp_files[idxs[0]], smp_files[idxs[1]] spec_smp = pestUtil.smp(smp_in_dir + smp_files[idxs[0]], load=True, pandas=True) chl_smp = pestUtil.smp(smp_in_dir + smp_files[idxs[1]], load=True, pandas=True) #--filter obvious outliers for dt, val in spec_smp.records['site'].iteritems(): if val > 50000: spec_smp.records['site'][dt] = np.NaN df = pandas.DataFrame({ 'spec': spec_smp.records['site'], 'chl': chl_smp.records['site'] }) df = df.dropna() if not df.empty: dfs.append(df)
#--plot relative concentration records - combined plt of chl, spec, and tds for file in unique_files: files = [] types = [] reclens = [] if file in chl_files: files.append(chl_dir+file) types.append('chl') if file in spec_files: files.append(spec_dir+file) types.append('spec') if file in tds_files: files.append(tds_dir+file) types.append('tds') smp = pestUtil.smp(files[0],load=True,pandas=True) smp.records[types[0]] = smp.records['site'] smp.records.pop('site') reclens.append(types[0]+' '+str(len(smp.records[types[0]]))) #--if more than one file was found if len(files) > 1: for file,dtype in zip(files[1:],types[1:]): other = pestUtil.smp(file,load=True,pandas=True) other.records[dtype] = other.records['site'] other.records.pop('site') reclens.append(dtype+' '+str(len(other.records[dtype]))) smp.records = pandas.merge(smp.records,other.records,left_index=True,right_index=True) print if not smp.records.empty: try:
'''for now we assume that flow and seawat have the same row col ''' K = 200.0 L = 500.0 A = 500.0 * 100.0 cond = K * A / L print 'loading north,south model-aligned dataframe and interpolation factors' df_ns_data = pandas.read_csv('..\\..\\_nwis\\dataframes\\ghb_NS_stages_model.csv',index_col=0,parse_dates=True) df_ns_fac = pandas.read_csv('ghb_NS_factors.dat',index_col=[0,1]) #--process stage record for coastal GHBs - presampled noaa_smp = pu.smp('..\\..\\_noaa\\noaa_sp.smp',load=True) coastal_stages = noaa_smp.records['noaa'][:,1] #--process stage records for WCA ghbs - these records should have been presampled to stress period dimensions print 'processing EDEN records' smp_dir = '..\\..\\_eden\\stage_smp_full\\' smp_files = os.listdir(smp_dir) eden_ids = [] eden_sp_vals = {} for sfile in smp_files: print 'loading smp file',sfile,'\r', eden_id = int(sfile.split('.')[0]) eden_ids.append(eden_id) smp = pu.smp(smp_dir+sfile,load=True) rec = smp.records[str(eden_id)] #--some defense
import pandas import shapefile import pestUtil as pu #--get a list of site that are within the broward domain shape_name = '..\\_gis\\scratch\\broward_nwis_gw_conc_depth' records = shapefile.load_as_dict(shape_name, loadShapes=False) sitenos = records['site_no'] dir_dict = { 'chl': 'smp_rel_conc_chl\\', 'cond': 'smp_rel_conc_regressed\\', 'tds': 'smp_rel_conc_tds\\' } df_dict = {} for dtype, smp_dir in dir_dict.iteritems(): files = os.listdir(smp_dir) dfs = [] for f in files: raw = f.split('.') siteno, sitename = raw[1], raw[2] if siteno in sitenos: print dtype, f smp = pu.smp(smp_dir + f, load=True, pandas=True) df = smp.records df[siteno] = df['site'] df.pop('site') dfs.append(df) df = pandas.concat(dfs, axis=1) df.to_csv('dataframes\\' + dtype + '.csv', index_label='datetime')
import numpy as np import pandas import shapefile import pestUtil as pu from bro_pred import flow #--get the dbhydro sampled monthly series and calc average monthly values db_dir = '..\\_dbhydro\\stressperiod_stage_smp_navd\\' db_files = os.listdir(db_dir) #db_names = [] db_dict = {} for dfile in db_files: dname = dfile.split('.')[0].upper().strip() #db_names.append(dname) smp = pu.smp(db_dir + dfile, load=True, pandas=True) df = smp.records site_name = df.keys()[0] df_monthly = df.groupby(lambda x: x.month).mean() df['monthly'] = np.NaN for dt in df.index: df['monthly'][dt] = df_monthly[site_name][dt.month] df[site_name] = df['monthly'] df.pop('monthly') db_dict[dname] = copy.deepcopy(df) #--get the coastal stage record - sampled to stress periods #noaa_file = '..\\_noaa\\noaa_slr.smp' #noaa_smp = pu.smp(noaa_file,load=True,pandas=True) #noaa_df = noaa_smp.records noaa_df = pandas.read_csv('..\\_noaa\\noaa_slr.csv',
tpl_files.append('tpl\\SWR_Dataset11.tpl') #--start and end start = datetime(1997,1,1,hour=12) end = datetime(2010,12,31,hour=12) start_str = start.strftime(tc.DATE_FMT) end_str = end.strftime(tc.DATE_FMT) date_dir = 'date_files\\' #--instance tsproc_infile = 'tsproc_setup.dat' tsp = tc.tsproc(tsproc_infile,out_file='processed.dat',out_fmt='long') hobs_file = 'UMD.01\\obsref\\head\\heads.smp' hobs_smp = pu.smp(hobs_file,date_fmt = tc.DATE_FMT,load=True) hobs_start,hobs_end = hobs_smp.get_daterange(site_name='all',startmin=start,endmax=end) mobs_file = 'UMD.01\\modref\\head\\mheads.smp' mobs_smp = pu.smp(mobs_file,date_fmt = tc.DATE_FMT,load=True) site_names = hobs_smp.records.keys() #--generate base names for processing obs_names = [] mod_names = [] for i,s in enumerate(site_names): obs_names.append('ogw_{0:03.0f}or'.format(i+1)) mod_names.append('mgw_{0:03.0f}or'.format(i+1)) #--write the load series block
import copy import os import pestUtil as pu date_fmt = '%m/%d/%Y' smp_dir = 'UMD.01\\obsref\\head\\' smp_files = os.listdir(smp_dir) obs_smp = pu.smp('UMD.01\\obsref\\head\heads.smp',date_fmt=date_fmt) mod_smp = pu.smp('UMD.01\\modref\\head\heads.smp',date_fmt=date_fmt) fcount = 1 for smp_file in smp_files: smp = pu.smp(smp_dir+smp_file,date_fmt=date_fmt,load=True) for site,record in smp.records.iteritems(): obs_name = site mod_name = copy.deepcopy(site) if len(obs_name) > 7: print 'truncate',obs_name obs_name = obs_name[:7] if obs_name in obs_smp.records.keys(): fc_str = str(fcount) obs_name = fc_str + obs_name[:-len(fc_str)] fcount += 1 print 'duplicate',obs_name obs_smp.records[obs_name] = record #mod_smp.records[obs_name] = record obs_smp.save('UMD.01\\obsref\\head\heads.smp') #mod_smp.save('UMD.01\\modref\\head\heads.smp')
png_idx = 15 smp_dir = 'smp_waterlevel_navd\\' for line in f: raw = line.strip().split(',') png_name = raw[png_idx] raw = png_name.split('\\') site_no = raw[-1].split('.')[1] smp_name = smp_dir + raw[-1].replace('png', 'smp').replace('"', '') assert os.path.exists(smp_name) smp_files[site_no] = (smp_name) #--load each smp file and concat into a single df dfs = [] null_dict = {} for site_no, smp in smp_files.iteritems(): df = pu.smp(smp, load=True, pandas=True).records df[site_no] = df['site'] df.pop('site') dfs.append(df.dropna()) null_dict[site_no] = np.NaN df_south = pandas.concat(dfs, axis=1) #--calc julian day means jd_means = df_south.groupby(lambda x: x.timetuple()[7]).mean() #--create a null daily df over the model time span dr = pandas.date_range(bro.start, bro.end, freq='1D') df_daily = pandas.DataFrame(null_dict, index=dr) #--fill in where we have data
parsed_smp_names = [] for file in smp_files: parsed = parse_smp_filename(file) parsed_smp_names.append(parsed) #--get a list of navd files navd_files = [] for file,attributes in zip(smp_files,parsed_smp_names): if attributes['param'] == 'navd': navd_files.append(file) #--write new smp files NGVD to NAVD ngvd2navd = -1.5 for file,attributes in zip(smp_files,parsed_smp_names): if attributes['param'] == 'ngvd': smp = pestUtil.smp(smp_in_dir+file,load=True) smp.records['site'][:,1] += ngvd2navd attributes['param'] = 'navd' new_name = build_smp_filename(attributes) if new_name in navd_files: other_smp = pestUtil.smp(smp_in_dir+new_name,load=True) smp.merge(other_smp) #--remove other smp from the navd list navd_files.pop(navd_files.index(new_name)) smp.save(smp_out_dir+new_name) #print for file in navd_files: shutil.copy(smp_in_dir+file,smp_out_dir+file)
sitenos = {} drange = pandas.date_range(seawat.start,seawat.end,freq='1D') for df in dfs: for siteno in df.keys(): if siteno not in sitenos.keys(): sitenos[siteno] = np.NaN df_mod = pandas.DataFrame(sitenos,index=drange) #--fill the null df by merging the data, starting with most accurate for df in dfs: df_mod = df_mod.combine_first(df) non_nans = df_mod.count() print non_nans.sum() df_mod.to_csv(df_dir+'relconc_merged_daily.csv',index_label='datetime') smp = pu.smp(None,load=False,pandas=True) smp.records = df_mod smp.save('relconc.smp',dropna=True) #--not using yet...going to try to use mod2obs to interpolate...we'll see... #--thin the data if more than one value in a month #groups = df_mod.groupby([lambda x:x.year,lambda x:x.month]).count() #print groups #for tup,count_rec in groups.iteritems(): # if count_rec.count() > 1: # print tup # print count_rec #df_mod_resamp = df_mod.resample(seawat.pandas_freq,how='mean') #non_nans = df_mod_resamp.count()
import pandas import pestUtil as pu import bro #--model stress period date range m_range = bro.sp_end #--merged daily smp files smp_dir = 'daily_stage_smp_navd\\' smp_files = os.listdir(smp_dir) out_dir = 'stressperiod_stage_smp_navd\\' dfs = [] for i, sfile in enumerate(smp_files): print 'processing ', sfile, i, ' of ', len(smp_files), '\r', depname = sfile.split('.')[0] smp = pu.smp(smp_dir + sfile, load=True, pandas=True) df = smp.records rname = df.keys()[0] df[depname] = df[rname] df.pop(rname) df = df.astype(float) df = df.resample(bro.pandas_freq, how=np.mean) #--create dataframe that is aligned with model stress periods #--merge in the record and fill with 0.0 df_mn = pandas.DataFrame({depname: np.NaN}, index=m_range) df_mn = df_mn.combine_first(df) df_mn = df_mn[bro.start:bro.end] df_mn = df_mn.dropna() dfs.append(df_mn) smp.records = df_mn
dry_month_end = 6 #--dimensions nlay, nrow, ncol = 3, 189, 101 start_date = dt.datetime(year=1996, month=1, day=1, hour=12) end_date = dt.datetime(year=2010, month=12, day=31, hour=12) #end_date = dt.datetime(year=1998,month=1,day=31,hour=12) num_days = end_date - start_date extract_well = ['SC', 'SW', 'ORR'] #extract_well = ['SC'] #--read pumpage data from smp files smp_ref = os.path.join('D:/', 'Data', 'Users', 'jdhughes', 'Projects', '2080DBF00', 'UMD', 'Data', 'WEL', 'pumpwell2_extendedto2010.smp') smp = pu.smp(smp_ref, load=True, date_fmt='%m/%d/%Y') site_names = smp.get_unique_from_file(smp.site_index) WellQ = 0.0 WellQDry = 0.0 IsDry = True DryDays = 0 for iday in xrange(0, num_days.days + 1): ondate = start_date + dt.timedelta(days=iday) if IsDry == True: if ondate.month == dry_month_end: IsDry = False elif IsDry == False: if ondate.month == dry_month_start: IsDry = True if IsDry == True:
#--initialize data inflow.fill(Missing) outflow.fill(Missing) sim.fill(Missing) pools = [] pools_dict = {} #--get smp inflow files from xml file processInflow = False child = swbudget.find('inflow_items') if child != None: processInflow = True for childitem in child.findall('smpitem'): print 'processing inflow...{0}'.format(childitem.text) #--get the smp file data smpstation = os.path.basename(childitem.text).replace('.smp', '') smp = pestUtil.smp(childitem.text, load=True, date_fmt='%m/%d/%Y') inflow = dataAdd(sim_dates, inflow, smp.records[smpstation][:, 0], smp.records[smpstation][:, 1]) #--get smp outflow files from xml file child = swbudget.find('outflow_items') if child != None: for childitem in child.findall('smpitem'): print 'processing outflow...{0}'.format(childitem.text) #--get the smp file data smpstation = os.path.basename(childitem.text).replace('.smp', '') smp = pestUtil.smp(childitem.text, load=True, date_fmt='%m/%d/%Y') outflow = dataAdd(sim_dates, outflow, smp.records[smpstation][:, 0], smp.records[smpstation][:, 1]) #--get station data from xml file station = swbudget.attrib['name']
while on_date < end_date: on_date += timedelta(days=1.) plot_dates.append( on_date ) plot_dates = np.array( plot_dates ) #--initialize figure and figure/plot counters ifigure = 1 iplot = 1 nplots = 6 fig = Make_NewFigure() #--matplotlib date specification years, months = mdates.YearLocator(), mdates.MonthLocator() #every year, every month yearsFmt = mdates.DateFormatter('%Y') #--open smp file smp = pestUtil.smp(ObsFile,load=False,date_fmt='%m/%d/%Y') #--process each station in the xml file iwell = 0 pct_lay = np.empty( (nlay), np.float ) active_gwobs = [] active_gwobs_stats = [] for idx,gwhead in enumerate( root.findall('gwHead') ): station = gwhead.attrib['name'] print 'Locating {0} in the model grid'.format( station ) coordType = gwhead.find('coordType').text #--determine the location of the well if coordType.lower() == 'model': #--convert to zero based irow = int( gwhead.find('row').text ) - 1 icol = int( gwhead.find('column').text ) - 1 elif coordType.lower() == 'site':
#--for each shape, look for matching records for record,shape in zip(records,shapes): #site_no = records['site_no'][i] site_no = record[site_no_idx] site_type = record[site_type_idx] #--look for a matching navd smp file record.append(-1) record.append('') for filename,attrib,pltname in zip(navd_files,navd_attribs,navd_pltnames): if attrib['site_no'] == site_no: smp = pestUtil.smp(navd_dir+filename,load=True,pandas=True) record[-2] = smp.records.shape[0] record[-1] = pltname break #--look for a matching conc smp file record.append(-1) record.append('') for filename,attrib,pltname in zip(conc_files,conc_attribs,conc_pltnames): if attrib['site_no'] == site_no: smp = pestUtil.smp(conc_dir+filename,load=True,pandas=True) record[-2] = smp.records.shape[0] record[-1] = pltname break if site_type.upper().startswith('GW'): wr_gw.poly([shape.points],shapeType=shape.shapeType)
f.close() model_start = grid.start obs_start = grid.start + timedelta(days=14) obs_end = grid.sp_end[-2] pred_start = grid.sp_end[-2] pred_end = grid.end date_dir = 'date_files\\' tsproc_infile = 'tsproc_setup.dat' tsp = tc.tsproc(tsproc_infile,out_file='processed.dat',out_fmt='long') pest_oblocks,pest_mblocks = [],[] hobs_file = '_misc\\heads.smp' hobs_smp = pu.smp(hobs_file,date_fmt=tc.DATE_FMT,load=True) mobs_file = 'mheads.smp' mobs_smp = pu.smp(mobs_file,date_fmt=tc.DATE_FMT,load=True) full_file = date_dir+'full_range.dat' tc.write_date_file(full_file,obs_start,obs_end,None) pred_file = date_dir+'pred_range.dat' tc.write_date_file(pred_file,pred_start,pred_end,None) #--swr - for prediction ost_names = ['obf_1'] mst_names = ['mbf_1'] rgp_nums = [1]
#--main script dry_month_start = 11 dry_month_end = 6 #--dimensions nlay,nrow,ncol = 3,189,101 start_date = dt.datetime(year=1996,month=1,day=1,hour=12) end_date = dt.datetime(year=2010,month=12,day=31,hour=12) #end_date = dt.datetime(year=1998,month=1,day=31,hour=12) num_days = end_date - start_date extract_well = ['SC', 'SW', 'ORR'] #extract_well = ['SC'] #--read pumpage data from smp files smp_ref = os.path.join( 'D:/','Data','Users','jdhughes','Projects','2080DBF00','UMD','Data','WEL', 'pumpwell2_extendedto2010.smp') smp = pu.smp(smp_ref,load=True,date_fmt='%m/%d/%Y') site_names = smp.get_unique_from_file(smp.site_index) WellQ = 0.0 WellQDry = 0.0 IsDry = True DryDays = 0 for iday in xrange(0,num_days.days+1): ondate = start_date + dt.timedelta(days=iday) if IsDry == True: if ondate.month == dry_month_end: IsDry = False elif IsDry == False: if ondate.month == dry_month_start: IsDry = True if IsDry == True:
df = dfs[wnum] gm = geom[:, int(float(row)) - 1, int(float(col)) - 1] midpts, groups = group_intervals(df.keys(), gm) mod_dicts = {} for ilay, group in groups.iteritems(): sname = 'ftl' + str(wnum) + '_' + str(ilay + 1) + 'L' print sname line = sname + ' {0:20.8G} {1:20.8G} {2:d}\n'.format(x, y, ilay + 1) bore_coords_lines.append(line) #print seawat.layer_botm_names[ilay] mod_dicts[sname] = df[group].mean(axis=1).dropna() df = pandas.DataFrame(mod_dicts) df.to_csv(df_dir + str(wnum) + '_mod.csv', index_label='datetime') mod_dfs.append(df) df = pandas.concat(mod_dfs, axis=1) smp = pu.smp(None, load=False, pandas=True) smp.records = df smp_dir = '..\\..\\_ftl_salt\\' smp.save(smp_dir + 'ftl_cali.smp', dropna=True) f = open(smp_dir + 'ftl_borecoords.dat', 'w', 0) for line in bore_coords_lines: f.write(line) for wnum, df in zip(wellnums, mod_dfs): fig = pylab.figure() ax = pylab.subplot(111) for site, record in df.iteritems(): print site ax.plot(record.index, record.values, '.', label=site) ax.grid()
f = open(stg_reach_file,'r') reach_dict = {} header = f.readline() for line in f: raw = line.strip().split(',') name = raw[0].upper().replace(' ','_').replace('-','') if name.endswith('W'): name = name[:-1] reach_dict[name] = int(raw[1]) f.close() #parser = lambda x: datetime.strptime(x,tc.DATE_FMT+' %H:%M:%S') #stage_df = pandas.read_table(stg_obs_file,header=None,parse_dates=[[1,2]],date_parser=parser,sep='\s*') #stage_df.columns = ['datetime','site','value'] stage_smp = pu.smp(stg_obs_file,date_fmt=tc.DATE_FMT,pandas=True,load=True) stage_sites = stage_smp.records.keys() for site in stage_sites: if site not in reach_dict.keys(): print 'site not found in reach dict',site obs_names = [] mod_names = [] reach_numbers = [] smp_site_names = [] for i,site in enumerate(reach_dict.keys()): if site not in stage_sites: print 'site not found in smp file',site reach_dict.pop(site) else: obs_names.append('ost_{0:03.0f}or'.format(i+1))
#--check for different types of salt data if len(site_salt_params) > 1: #--for spec and chlorida if 'spec' in site_salt_params and 'chloride' in site_salt_params: spec_chl_idxs[site_no] = [site_salt_idxs[site_salt_params.index('spec')],site_salt_idxs[site_salt_params.index('chloride')]] if 'dissolved' in site_salt_params and 'chloride' in site_salt_params: tds_chl_idxs[site_no] = [site_salt_idxs[site_salt_params.index('dissolved')],site_salt_idxs[site_salt_params.index('chloride')]] #--load the paired salt files and look for data on the same day spec_data,chl_data = [],[] dfs = [] site_nos = [] for site_no,idxs in spec_chl_idxs.iteritems(): print 'loading smp files:',smp_files[idxs[0]],smp_files[idxs[1]] spec_smp = pestUtil.smp(smp_in_dir+smp_files[idxs[0]],load=True,pandas=True) chl_smp = pestUtil.smp(smp_in_dir+smp_files[idxs[1]],load=True,pandas=True) #--filter obvious outliers for dt,val in spec_smp.records['site'].iteritems(): if val > 50000: spec_smp.records['site'][dt] = np.NaN df = pandas.DataFrame({'spec':spec_smp.records['site'],'chl':chl_smp.records['site']}) df = df.dropna() if not df.empty: dfs.append(df) site_nos.append(site_no) df = pandas.concat(dfs,keys=site_nos) #--calc a filtered global regression equation x,y = df['spec'].values.astype(np.float64),df['chl'].values.astype(np.float64)
import os import shutil import pandas import pylab import pestUtil as pu mod2obs_ins = ['mod2obs.in','mod2obs_l.in','mod2obs_rc.in'] names = ['base','layer','rowcol'] colors = ['k','b','g'] dfs = [] exe = 'mod2obs.exe' for m2o_in,name in zip(mod2obs_ins,names): cmd_line = exe + ' < ' + m2o_in print cmd_line os.system(cmd_line) shutil.copy('mheads.smp',name+'.smp') smp = pu.smp('mheads.smp',load=True,pandas=True) dfs.append(smp.records) for site in dfs[0].columns: fig = pylab.figure(figsize=(6,6)) ax = pylab.subplot(111) for df,name,color in zip(dfs,names,colors): ax.plot(df[site].index,df[site].values,color=color) ax.set_title(site) ax.legend(names) pylab.savefig('png\\'+site+'.png',fmt='png',dpi=300,bbox_inches='tight')