def create_nei_runlist(tdirpath,
hdirpath,
sourcedf,
sdate,
edate,
timechunks,
units="ppb",
tcm=False):
from os import walk
# dstart = sdate
# dend = edate
hysplitdir = os.path.join(hdirpath, 'exec')
runlist = []
dtree = dirtree(tdirpath, sdate, edate, chkdir=False, dhour=timechunks)
for dirpath in dtree:
for src in nei_source_generator(sourcedf):
suffix = "EIS" + src["EIS_ID"].strip()
sdate = dir2date(tdirpath, dirpath)
pdate = sdate - datetime.timedelta(hours=timechunks)
pdir = date2dir(tdirpath, pdate, dhour=timechunks)
parinitA = "PARDUMP." + suffix
parinitB = "PARINIT." + suffix
parinit = (pdir, parinitA, parinitB)
run = RunDescriptor(dirpath, suffix, hysplitdir, "hycs_std",
parinit)
runlist.append(run)
return runlist
def obs2datem(self, edate, ochunks=(1000, 1000), tdir="./"):
"""
##https://aqsdr1.epa.gov/aqsweb/aqstmp/airdata/FileFormats.html
##Time GMT is time of dat that sampling began.
edate: datetime object
ochunks: tuple (integer, integer)
Each represents hours
tdir: string
top level directory for output.
"""
logger.info("WRITING Datem FILE " + edate.strftime("%Y-%m-%d"))
#print(self.obs["units"].unique())
d1 = edate
done = False
iii = 0
maxiii = 1000
oe = ochunks[1]
oc = ochunks[0]
while not done:
d2 = d1 + datetime.timedelta(hours=oc - 1)
d3 = d1 + datetime.timedelta(hours=oe - 1)
odir = date2dir(tdir, d1, dhour=oc, chkdir=True)
dname = odir + "datem.txt"
datem.write_datem(self.obs,
sitename="siteid",
drange=[d1, d3],
dname=dname,
fillhours=1,
verbose=False)
d1 = d2 + datetime.timedelta(hours=1)
iii += 1
if d1 > self.d2:
done = True
if iii > maxiii:
done = True
logger.warning("WARNING: obs2datem, loop exceeded maxiii")
def create_controls(
tdirpath,
hdirpath,
sdate,
edate,
timechunks,
metfmt,
units="ppb",
tcm=False,
orislist=None,
moffset=0,
overwritelanduse=True,
):
"""
read the base control file in tdirpath CONTROL.0
read the base SETUP.0 file in tdirpath
walk through all subdirectories in tdirpath.
For each file with EMIT as part of the filename
1. read the file and get number of records in each cycle.
2. take the suffix from the file
3. Print out a CONTROL file with same suffix and
1. duration set by timechunks input
2. number of starting locations matching the EMITIMES file
3. start date matching EMITIMES file.
4. meteorological files matching the dates
4. Print out a SETUP file with same suffix and
1. initialized with parinit file from previous time period.
2. output pardump file with same suffix at end of run.
#3. set ninit to 1.
4. set delt=5 (TO DO- why?)
5. write out landuse file
6. write script in tdirpath for running HYSPLIT.
tdirpath: str
top level directory for output.
tdirpath: str
directory for hysplit executable
sdate : datetime object
edate : datetime object
timechunks : integer
run duration.
RETURNS:
runlist : list of runDescriptor objects
# TODO - add ability to base numpar on amount of emissions.
# will need to read EMITTIME file and find max emission for that time
# period. Then use
"""
from os import walk
from utilhysplit import emitimes
# from arlhysplit.runh import getmetfiles
dstart = sdate
dend = edate
##determines meteorological files to use.
met_files = MetFiles(metfmt)
# due to missing files MetFiles does not always automatically calculte
# correct time spacing of files for the SREF.
if "sref" in metfmt:
met_files.set_mdt(24)
elif "href" in metfmt:
met_files.set_mdt(24)
runlist = [] # list of RunDescriptor Objects
if hdirpath[-1] != "/":
hdirpath += "/"
hysplitdir = hdirpath + "exec/"
landusedir = hdirpath + "bdyfiles/"
dtree = dirtree(tdirpath, sdate, edate, chkdir=False, dhour=timechunks)
iii = 0
# for (dirpath, dirnames, filenames) in walk(tdirpath):
for dirpath in dtree:
for (d1, dirnames, filenames) in walk(dirpath):
for fl in filenames:
if iii == 0:
firstdirpath = dirpath
make = False
if "EMIT" in fl[0:4]:
make = True
if "WARNING" in fl:
make = False
if "MESSAGE" in fl:
make = False
if "VMSDIST" in fl:
make = False
# oris = fl.replace('EMIT','')
# if orislist not None:
# make = True
if make:
suffix = fl[4:8]
temp = fl.split(".")
if temp[1] != "txt":
suffix += "." + temp[1]
else:
suffix = temp[0].replace("EMIT", "")
wdir = dirpath
# read emitfile and modify number of locations
et = emitimes.EmiTimes(filename=dirpath + "/" + fl)
# print(dirpath, fl)
et_not_empty = et.read_file()
# if the emittimes file not empty then
if et_not_empty:
# number of locations is number of records
# in the emitimes file divided by number of speciess.
nrecs = et.cycle_list[0].nrecs / len(et.splist)
ht = et.cycle_list[0].recordra[0].height
sdate = et.cycle_list[0].sdate
##if sdate not within range given,
##then skip the rest of the loop.
if sdate < dstart or sdate > dend:
continue
lat = et.cycle_list[0].recordra[0].lat
lon = et.cycle_list[0].recordra[0].lon
##Write a setup file for this emitimes file
setupfile = NameList("SETUP.0", working_directory=tdirpath)
setupfile.read()
setupfile.add("NDUMP", str(timechunks))
setupfile.add("NCYCL", str(timechunks))
#setupfile.add("POUTF", '"PARDUMP.' + suffix + '"')
setupfile.add("POUTF", 'PARDUMP.' + suffix)
setupfile.add("PINPF", '"PARINIT.' + suffix + '"')
if not tcm:
setupfile.add("NINIT", "1")
setupfile.add("EFILE", '"' + fl + '"')
else:
setupfile.add("NINIT", "0")
# setupfile.add('DELT', '5')
setupfile.rename("SETUP." + suffix,
working_directory=wdir + "/")
setupfile.write(verbose=False)
#print("TDIRPATH", tdirpath)
##Write a control file for this emitimes file
control = HycsControl(fname="CONTROL.0",
working_directory=tdirpath)
control.read()
# make sure that control start is always start of time
# period.
controldate = dir2date(tdirpath, dirpath)
control.date = controldate
# remove species and add new with same
# attributes but different names
# even if EMITTIMES file empty there is still
# a header with species information.
if et.splist.size > 0:
sp = control.species[0]
control.remove_species()
for spec in et.splist:
spnew = sp.copy()
# print('Adding species', spec)
spnew.name = et.sphash[spec]
# print(spnew.strpollutant())
control.add_species(spnew)
# if emit-times file not empty
# remove all the locations first and then add
# locations that correspond to emittimes file.
# if it is empty then still create CONTROL file in case
# there is data from PARDUMP file to run.
# keep location from default CONTROL.0 file.
if et_not_empty:
control.remove_locations()
nlocs = control.nlocs
while nlocs != nrecs:
if nlocs < nrecs:
control.add_location(latlon=(lat, lon), alt=ht)
if nlocs > nrecs:
control.remove_locations(num=0)
nlocs = control.nlocs
control.rename("CONTROL." + suffix, working_directory=wdir)
control.remove_metfile(rall=True)
###Add the met files.
# mfiles = getmetfiles(
# control.date, timechunks, met_type=met_type, mdir=mdir
# )
if tcm:
timechunks = int(control.run_duration)
# ability to have met files cover longer time period.
mdate = control.date - datetime.timedelta(hours=moffset)
metchunks = timechunks + moffset
# mfiles = met_files.get_files(control.date, timechunks)
mfiles = met_files.get_files(mdate, metchunks)
#print("MFILES", mfiles)
#print(mdate, metchunks)
#print(control.date, timechunks)
for mf in mfiles:
if os.path.isfile(mf[0] + mf[1]):
control.add_metfile(mf[0], mf[1])
for cg in control.concgrids:
cg.outfile += "." + suffix
if control.num_met > 12:
metgrid = True
else:
metgrid = False
control.write(metgrid=metgrid, overwrite=True, query=False)
writelanduse(
landusedir=landusedir,
working_directory=wdir + "/",
overwrite=overwritelanduse,
verbose=False,
)
with open(wdir + "/rundatem.sh", "w") as fid:
fid.write("MDL=" + hysplitdir + "/\n")
fid.write(unit_mult(units=units))
fid.write(statmainstr())
if dirpath == firstdirpath:
parinit = (None, None, None)
elif not tcm:
sdate = dir2date(tdirpath, dirpath)
pdate = sdate - datetime.timedelta(hours=timechunks)
pdir = date2dir(tdirpath, pdate, dhour=timechunks)
parinitA = "PARDUMP." + suffix
parinitB = "PARINIT." + suffix
parinit = (pdir, parinitA, parinitB)
run = RunDescriptor(wdir, suffix, hysplitdir, "hycs_std",
parinit)
wr = "a"
if iii == 0:
wr = "w"
# with open(tdirpath + '/' + scr, wr) as fid:
# fid.write(run.script())
# fid.write('\n\n')
runlist.append(run)
iii += 1
return runlist
def nei_controls(
tdirpath,
hdirpath,
sourcedf,
sdate,
edate,
timechunks,
metfmt,
units="ppb",
tcm=False,
overwritelanduse=True,
):
# sourcedf contains info on
# stackheight
# emission rate
# heat input
from os import walk
control = HycsControl(fname="CONTROL.0", working_directory=tdirpath)
control.read()
dstart = sdate
dend = edate
##determines meteorological files to use.
met_files = MetFiles(metfmt)
runlist = [] # list of RunDescriptor Objects
if hdirpath[-1] != "/":
hdirpath += "/"
hysplitdir = os.path.join(hdirpath, "exec")
landusedir = os.path.join(hdirpath, "bdyfiles")
#print("LANDUSEDIR", landusedir, hdirpath)
dtree = dirtree(tdirpath, sdate, edate, chkdir=False, dhour=timechunks)
iii = 0
for dirpath in dtree:
for src in nei_source_generator(sourcedf):
suffix = "EIS" + src["EIS_ID"].strip()
setupfile = NameList("SETUP.0", working_directory=tdirpath)
setupfile.read()
setupfile.rename("SETUP." + suffix,
working_directory=dirpath + "/")
setupfile.add("NDUMP", str(timechunks))
setupfile.add("NCYCL", str(timechunks))
setupfile.add("POUTF", 'PARDUMP.' + suffix)
setupfile.add("PINPF", '"PARINIT.' + suffix + '"')
setupfile.write(verbose=False)
release_rate = src["SO2_kgph"]
##Write a control file
control = HycsControl(fname="CONTROL.0",
working_directory=tdirpath)
control.read()
# make sure that control start is always start of time
# period.
controldate = dir2date(tdirpath, dirpath)
control.date = controldate
control.rename("CONTROL." + suffix, working_directory=dirpath)
# edit locations
control.remove_locations()
control.add_location(
line=False,
latlon=(src["latitude"], src["longitude"]),
alt=src["height"],
rate=0,
)
# set emission rate and duration in the pollutant definition.
for sp in control.species:
sp.rate = release_rate
sp.duration = int(control.run_duration)
###Add the met files.
control.remove_metfile(rall=True)
if tcm:
timechunks = int(control.run_duration)
mfiles = met_files.get_files(control.date, timechunks)
for mf in mfiles:
if os.path.isfile(mf[0] + mf[1]):
control.add_metfile(mf[0], mf[1])
for cg in control.concgrids:
cg.outfile += "." + suffix
if control.num_met > 12:
metgrid = True
else:
metgrid = False
control.write(metgrid=metgrid, overwrite=True, query=False)
writelanduse(
landusedir=landusedir,
working_directory=dirpath + "/",
overwrite=overwritelanduse,
verbose=False,
)
sdate = dir2date(tdirpath, dirpath)
pdate = sdate - datetime.timedelta(hours=timechunks)
pdir = date2dir(tdirpath, pdate, dhour=timechunks)
parinitA = "PARDUMP." + suffix
parinitB = "PARINIT." + suffix
parinit = (pdir, parinitA, parinitB)
run = RunDescriptor(dirpath, suffix, hysplitdir, "hycs_std",
parinit)
runlist.append(run)
return runlist
def create_runlist(tdirpath, hdirpath, sdate, edate, timechunks):
"""
read the base control file in tdirpath CONTROL.0
read the base SETUP.0 file in tdirpath
walk through all subdirectories in tdirpath.
For each file with EMIT as part of the filename
1. read the file and get number of records in each cycle.
2. take the suffix from the file
return list of RunDescriptor objects
tdirpath: str
top level directory for output.
tdirpath: str
directory for hysplit executable
sdate : datetime object
edate : datetime object
timechunks : integer
run duration.
"""
from os import walk
from utilhysplit import emitimes
# from arlhysplit.runh import getmetfiles
dstart = sdate
dend = edate
##determines meteorological files to use.
runlist = []
#if hdirpath[-1] != "/":
# hdirpath += "/"
hysplitdir = os.path.join(hdirpath + "exec")
iii = 0
for (dirpath, dirnames, filenames) in walk(tdirpath):
for fl in filenames:
if iii == 0:
firstdirpath = dirpath
if "EMIT" in fl[0:4]:
et = emitimes.EmiTimes(filename=dirpath + "/" + fl)
# if not et.read_file(): continue
try:
sdate = dir2date(tdirpath, dirpath)
except:
continue
if sdate < dstart or sdate > dend:
continue
suffix = fl[4:8]
temp = fl.split(".")
if temp[1] != "txt":
suffix += "." + temp[1]
temp = fl.replace("EMIT", "")
suffix = temp.replace(".txt", "")
wdir = dirpath
if dirpath == firstdirpath:
parinit = (None, None, None)
else:
pdate = sdate - datetime.timedelta(hours=timechunks)
pdir = date2dir(tdirpath, pdate, dhour=timechunks)
parinitA = "PARDUMP." + suffix
parinitB = "PARINIT." + suffix
parinit = (pdir, parinitA, parinitB)
run = RunDescriptor(wdir, suffix, hysplitdir, "hycs_std",
parinit)
wr = "a"
if iii == 0:
wr = "w"
# with open(tdirpath + '/' + scr, wr) as fid:
# fid.write(run.script())
# fid.write('\n\n')
runlist.append(run)
iii += 1
return runlist
def emit_subroutine(
self,
df,
dfheat,
edate,
schunks,
tdir="./",
unit=True,
bname="EMIT",
emit_area=0,
):
"""
create emitimes file for CEMS emissions.
edate is the date to start the file on.
Currently, 24 hour cycles are hard-wired.
"""
# df = self.get_so2()
# dfheat = self.get_heat()
locs = df.columns.values
prev_oris = "none"
ehash = {}
# get list of oris numbers
orislist = []
unithash = {}
for hdr in locs:
oris = hdr[0]
orislist.append(oris)
unithash[oris] = []
for hdr in locs:
oris = hdr[0]
# print(hdr)
if unit:
mid = hdr[5]
else:
mid = "None"
unithash[oris].append(mid)
orislist = list(set(orislist))
sphash = {1: "MEAS", 2: "EST1", 3: "EST2"}
# create a dictionary with key oris number and value and EmiTimes
# object.
for oris in orislist:
for mid in unithash[oris]:
# output directory is determined by tdir and starting date.
# chkdir=True means date2dir will create the directory if
# it does not exist already.
ename = bname + str(oris)
if unit:
ename = ename + "_" + str(mid)
odir = date2dir(tdir, edate, dhour=schunks, chkdir=True)
ename = odir + ename + ".txt"
if unit:
key = str(oris) + str(mid)
else:
key = oris
ehash[key] = emitimes.EmiTimes(filename=ename)
ehash[key].set_species(sphash)
# now this loop fills the EmitTimes objects
for hdr in locs:
oris = hdr[0]
d1 = edate # date to start emitimes file.
dftemp = df[hdr]
dfh = dfheat[hdr]
dftemp.fillna(0, inplace=True)
dftemp = dftemp[dftemp != 0]
# ename = bname + str(oris)
# if unit:
# sid = hdr[4]
# ename += "." + str(sid)
height = hdr[1]
lat = hdr[2]
lon = hdr[3]
spnum = hdr[4]
key = oris
if unit:
mid = hdr[5]
key += str(mid)
# hardwire 1 hr duraton of emissions.
record_duration = "0100"
# pick which EmitTimes object we are working with.
efile = ehash[key]
# output directory is determined by tdir and starting date.
# chkdir=True means date2dir will create the directory if
# it does not exist already.
# odir = date2dir(tdir, edate, dhour=schunks, chkdir=True)
# ename = odir + ename + ".txt"
# efile = emitimes.EmiTimes(filename=ename)
# this was creating a special file for a pre-processing program
# that would take diameter, temp and velocity to compute plume rise.
if "STACK" in bname:
hstring = efile.header.replace(
"HEAT(w)", "DIAMETER(m) TEMP(K) VELOCITY(m/s)")
efile.modify_header(hstring)
# hardwire 24 hour cycle length
dt = datetime.timedelta(hours=24)
#efile.add_cycle(d1, "0024")
efile.add_cycle(d1, 24)
for date, rate in dftemp.iteritems():
# if spnum!=1: print(date, rate, spnum)
if date >= edate:
heat = dfh[date]
check = efile.add_record(
date,
record_duration,
lat,
lon,
height,
rate,
emit_area,
heat,
spnum,
)
nnn = 0
while not check:
d1 = d1 + dt
efile.add_cycle(d1, 24)
check = efile.add_record(
date,
record_duration,
lat,
lon,
height,
rate,
emit_area,
heat,
spnum,
)
nnn += 1
if nnn > 20:
break
# if not check2:
# print("sverify WARNING: record not added to EmiTimes")
# print(date.strftime("%Y %m %d %H:%M"))
# print(str(lat), str(lon), str(rate), str(heat))
# break
# here we write the EmitTimes files
for ef in ehash.values():
ef.write_new(ef.filename)