def write_daily_prof(df, temp, run_group, state):
'''
Write the daily profile based hourly profiles as a normalized list by year/month/day/hour
Daily factors should be based on sum of hourly haps over sum of annual haps
'''
cols = ['region_cd','scc','run_group','facid','tract','polynumber','src_id','ann_value']
hierarchy = [['region_cd','scc'],['region_cd',]]
df = df[cols].copy().sort('ann_value').drop_duplicates(['run_group','region_cd','facid'],
take_last=True)
temp.aermod.fillna(0, inplace=True)
value_cols = ['month','day','hour','factor']
df = df[['run_group','region_cd','scc',
'ann_value']].copy().drop_duplicates(['run_group','region_cd','scc'])
df = match_temporal(df, temp.aermod, value_cols, hierarchy, temp.use_daily)
df.drop('scc', axis=1, inplace=True)
df['year'] = os.environ['BASE_YEAR'][2:4]
df.drop_duplicates(['region_cd','run_group','year','month','day','hour'], inplace=True)
keys = ['region_cd','run_group','year']
mean_factor = df[['region_cd','factor']].copy().groupby('region_cd', as_index=False).mean()
df = pd.merge(df, mean_factor, on='region_cd', how='left', suffixes=['','_mean'])
df['factor'] = (df['factor']/df['factor_mean'])
df[['month','day','hour']] = df[['month','day','hour']].astype('i')
fname = os.path.join(os.environ['WORK_PATH'],'temporal','tract_%s_%s_hourly.csv' %(state,
run_group))
df.to_csv(fname, columns=['run_group','region_cd','year','month','day','hour','factor'],
index=False)
def write_no_daily_prof(run_df, temp, run_group, state):
'''
Write the standard non-hourly non-daily profile based temporal profiles
'''
cols = ['region_cd','scc','run_group','facid','tract','polynumber','src_id','ann_value']
run_df = run_df[cols].copy().drop_duplicates()
# Use main contributing region_cd and SCC for source temporalization
run_df.sort('ann_value', inplace=True)
run_df.drop_duplicates('facid', keep='last', inplace=True)
scalar_cols = [s_col for s_col in temp.aermod.columns if s_col.startswith('Scalar')]
value_cols = ['qflag',] + scalar_cols
# Only match by fips/scc or fips; SCC only gets default
hierarchy = [['region_cd','scc'],['region_cd',]]
run_df = match_temporal(run_df, temp.aermod, value_cols, hierarchy)
run_df.drop(['scc','ann_value'], axis=1, inplace=True)
run_df.drop_duplicates(inplace=True)
if not run_df[run_df[def_prof.index[0]].isnull()].empty:
if run_group == 'OILGAS4':
run_df = fill_default(run_df, def_oil_prof)
else:
run_df = fill_default(run_df, def_prof)
qflag_list = list(run_df['qflag'].drop_duplicates())
if 'MHRDOW7' in qflag_list:
scalar_cols = ['Scalar%s' %x for x in xrange(1,2017)]
elif 'MHRDOW' in qflag_list:
scalar_cols = ['Scalar%s' %x for x in xrange(1,865)]
elif 'HROFDY' in qflag_list:
scalar_cols = ['Scalar%s' %x for x in xrange(1,25)]
else:
scalar_cols = ['Scalar%s' %x for x in xrange(1,13)]
cols = ['run_group','state','region_cd','tract','polynumber','facid','src_id','qflag']
run_df['state'] = state_dict[state]
fname = os.path.join(os.environ['WORK_PATH'],'temporal','tract_%s_%s_temporal.csv' %(state,
run_group))
run_df.to_csv(fname, index=False, columns=cols+scalar_cols)
def get_no_daily_prof(run_df, temp):
'''
Write the standard non-hourly non-daily profile based temporal profiles
'''
scalar_cols = [
s_col for s_col in temp.aermod.columns if s_col.startswith('Scalar')
]
value_cols = [
'qflag',
] + scalar_cols
# Only match by fips/scc or fips; SCC only gets default
hierarchy = [['region_cd', 'scc'], [
'region_cd',
]]
run_df = match_temporal(run_df, temp.aermod, value_cols, hierarchy)
run_df.drop(['region_cd', 'scc'], axis=1, inplace=True)
run_df.drop_duplicates(inplace=True)
if not run_df[run_df[def_prof.index[0]].isnull()].empty:
run_df = fill_default(run_df, def_prof)
qflag_list = list(run_df['qflag'].drop_duplicates())
if 'MHRDOW7' in qflag_list:
scalar_cols = ['Scalar%s' % x for x in xrange(1, 2017)]
elif 'MHRDOW' in qflag_list:
scalar_cols = ['Scalar%s' % x for x in xrange(1, 865)]
elif 'HROFDY' in qflag_list:
scalar_cols = ['Scalar%s' % x for x in xrange(1, 25)]
else:
scalar_cols = ['Scalar%s' % x for x in xrange(1, 13)]
return run_df[['run_group', 'met_cell', 'src_id', 'qflag'] +
scalar_cols].copy()
def write_temp_factors(df, temp, work_path):
'''
Write the temporal factors by source ID for each facility
Adjust the output columns based on the max number of scalars used within the facility
'''
df = df[[
'region_cd', 'scc', 'facility_id', 'facility_name', 'src_id', 'state'
]].copy()
scalar_cols = [col for col in temp.columns if col.startswith('Scalar')]
hierarchy = [['region_cd', 'scc', 'facility_id'], ['scc', 'facility_id'],
['region_cd', 'facility_id'], [
'facility_id',
], ['region_cd', 'scc'], [
'scc',
], [
'region_cd',
]]
df = match_temporal(df, temp, [
'qflag',
] + scalar_cols, hierarchy)
cols = ['facility_id', 'facility_name', 'src_id', 'qflag']
df = df[cols + scalar_cols].copy().drop_duplicates(
['facility_id', 'src_id'])
qflag_list = list(df['qflag'].drop_duplicates())
if 'MHRDOW7' in qflag_list:
col_names = cols + ['Scalar%s' % x for x in range(1, 2017)]
elif 'MHRDOW' in qflag_list:
col_names = cols + ['Scalar%s' % x for x in range(1, 865)]
elif 'HROFDAY' in qflag_list:
col_names = cols + ['Scalar%s' % x for x in range(1, 25)]
else:
col_names = cols + ['Scalar%s' % x for x in range(1, 13)]
fname = os.path.join(work_path, 'temporal', 'point_temporal.csv')
df.to_csv(fname, columns=col_names, index=False, quotechar=' ')
def get_temp_codes(df, xref):
'''
Get the temporal profile codes for each row. This is used for checking unique temporal profile.
* Assume that only diurnal profiles only vary by weekday/weekend or
are constant across all days of the week.
This assumption is good for all US point sources in 2014v2.
'''
if len(xref[xref['ALLDAY'].isnull()]) > 0:
if 'TUESDAY' in list(xref.columns):
try:
xref.ix[xref['ALLDAY'].isnull(),
'ALLDAY'] = xref.ix[xref['ALLDAY'].isnull(), 'TUESDAY']
except KeyError:
xref.ix[xref['ALLDAY'].isnull(),
'ALLDAY'] = xref.ix[xref['ALLDAY'].isnull(), 'WEEKDAY']
type_cols = ['ALLDAY', 'MONTHLY', 'WEEKLY']
xref.drop_duplicates(['facility_id', 'region_cd', 'scc'], inplace=True)
xref['region_cd'] = xref['region_cd'].apply(fix_fips)
xref['scc'] = xref['scc'].apply(lambda x: str(x).strip())
xref['facility_id'] = xref['facility_id'].apply(lambda x: str(x).strip())
hierarchy = [['region_cd', 'scc', 'facility_id'], ['scc', 'facility_id'],
['region_cd', 'facility_id'], [
'facility_id',
], ['region_cd', 'scc'], [
'scc',
], [
'region_cd',
]]
return match_temporal(df, xref, type_cols, hierarchy)
def write_daily_prof(df, temp):
'''
Write the daily profile based hourly profiles as a normalized list by year/month/day/hour
Daily factors should be based on sum of hourly haps over sum of annual haps
'''
cols = ['region_cd','scc','run_group','met_cell','ann_value','src_id']
hierarchy = [['region_cd','scc'],['region_cd',]]
df = df[cols].copy().sort_values('ann_value').drop_duplicates(['run_group','region_cd','met_cell'],
take_last=True)
temp.profs.fillna(0, inplace=True)
value_cols = ['month','day','hour','factor']
run_group = df['run_group'].values[0]
fips_map = get_fips_map(df[['run_group','region_cd','met_cell','src_id',
'ann_value']].copy())
write_county_xwalk(fips_map, run_group)
df = df[['run_group','region_cd','scc',
'ann_value']].copy().drop_duplicates(['run_group','region_cd','scc'])
for state in list(df['region_cd'].str[:2].drop_duplicates()):
st_df = df.ix[df['region_cd'].str[:2] == state].copy()
st_df = match_temporal(st_df, temp.profs, value_cols, hierarchy, temp.use_daily)
st_df.drop('scc', axis=1, inplace=True)
st_df['year'] = os.environ['BASE_YEAR'][2:4]
st_df.drop_duplicates(['region_cd','run_group','year','month','day','hour'],
inplace=True)
keys = ['region_cd','run_group','year']
mean_factor = st_df[['region_cd','factor']].copy().groupby('region_cd', as_index=False).mean()
st_df = pd.merge(st_df, mean_factor, on='region_cd', how='left', suffixes=['','_mean'])
st_df['factor'] = (st_df['factor']/st_df['factor_mean'])
st_df[['month','day','hour']] = st_df[['month','day','hour']].astype('i')
fname = os.path.join(os.environ['WORK_PATH'],'temporal','%s_%s_hourly.csv' %(run_group, state))
st_df.to_csv(fname, columns=['run_group','region_cd','year','month','day','hour','factor'],
index=False, float_format='%.12g')
def _write_temp_prof(self, df, temp, airport_type):
'''
Write the temporal factors by source ID for each facility
Adjust the output columns based on the max number of scalars used within the facility
'''
cols = ['facility_id','facility_name','src_id']
df = df[cols].copy()
inv_df = self.inv.ix[self.inv['facility_id'].isin(list(df['facility_id'].drop_duplicates())),
['region_cd','scc','facility_id','ann_value']].copy()
inv_df.sort('ann_value', ascending=False, inplace=True)
inv_df.drop_duplicates('facility_id', inplace=True)
scalar_cols = [col for col in temp.columns if col.startswith('Scalar')]
hierarchy = [['region_cd','scc','facility_id'],['scc','facility_id'],['region_cd','facility_id'],
['facility_id',], ['region_cd','scc'], ['scc',], ['region_cd',]]
temp = match_temporal(inv_df, temp, ['qflag',]+scalar_cols, hierarchy)
qflag_list = list(temp['qflag'].drop_duplicates())
if 'MHRDOW7' in qflag_list:
scalar_cols = ['Scalar%s' %x for x in range(1,2017)]
elif 'MHRDOW' in qflag_list:
scalar_cols = ['Scalar%s' %x for x in range(1,865)]
elif 'HROFDY' in qflag_list:
scalar_cols = ['Scalar%s' %x for x in range(1,25)]
else:
scalar_cols = ['Scalar%s' %x for x in range(1,13)]
df = pd.merge(df, temp[['facility_id','qflag']+scalar_cols], on='facility_id', how='left')
df.drop_duplicates(['facility_id','src_id'], inplace=True)
fname = os.path.join(os.environ['WORK_PATH'],'temporal','airport_%s_temporal.csv' %airport_type)
df.to_csv(fname, index=False, quotechar=' ')
def calc_monthly_temp(df, temp):
'''
Get monthly only temporlization. The CMV run group only varies by month.
'''
scalar_cols = [
s_col for s_col in temp.profs.columns if s_col.startswith('Scalar')
]
value_cols = [
'qflag',
] + scalar_cols
# Only match by fips/scc or fips; SCC only gets default
hierarchy = [['region_cd', 'scc'], [
'region_cd',
], [
'scc',
]]
df = match_temporal(df, temp.profs, value_cols, hierarchy)
df['state'] = df['region_cd'].str[:2]
df.drop(['region_cd', 'scc'], axis=1, inplace=True)
df.drop_duplicates(inplace=True)
if len(df[df['qflag'] != 'MONTH']) > 0:
print('WARNING: Non-MONTH profiles found. This should not be for CMV.')
scalar_cols = ['Scalar%s' % x for x in range(1, 13)]
return df[['state', 'facid', 'src_id', 'qflag'] + scalar_cols]