def log():
"""
Provide a basic data structure for a log document.
Returns:
dict: The data structure for a log.
"""
return {
"_id": "l-" + utils.new_uuid(),
"action": "",
"comment": "",
"data_type": "",
"data": "",
"timestamp": utils.make_timestamp(),
"user": "",
}
def main():
train = pd.read_pickle(TRAIN_PATH)
X_test = pd.read_pickle(TEST_PATH)
X_train = train.drop('isFraud', axis=1)
y_train = train['isFraud']
proba, experiment_id, run_uuid = train_and_predict(X_train, y_train,
X_test, PARAMS,
EXPRIMENT_PATH)
timestamp = make_timestamp()
make_submission(
os.path.join(SUBMISSION_DIR, f'{timestamp}_{run_uuid}.csv'), proba)
print_divider('MLflow UI')
print('Run URL: http://127.0.0.1:5000/#/experiments/{0}/runs/{1}\n'.format(
experiment_id, run_uuid))
mlflow_ui()
def add_biobank_form():
if flask.request.method == 'GET':
args = dict(flask.request.args)
elif flask.request.method == 'POST':
args = dict(flask.request.form)
token = args.get('token')
if not token or token not in flask.current_app.config.get('tokens'):
if 'originUrl' in args:
page = FAIL_PAGE.replace('PLACEHOLDER', args['originUrl'])
else:
page = FAIL_PAGE.replace(
'<a href="PLACEHOLDER">Back to the form.</a>', '')
return flask.Response(page, status=401)
args['timestamp'] = utils.make_timestamp()
flask.g.db['responsesAddBiobank'].insert_one(args)
if 'originUrl' in args:
page = SUCCESS_PAGE.replace('PLACEHOLDER', args['originUrl'])
else:
page = SUCCESS_PAGE.replace(
'<a href="PLACEHOLDER">Back to the form.</a>', '')
return flask.Response(page, status=200)
def update_run_model(varlist,
it,
m=m,
homogenous=2,
runyn=True,
plotyn=False,
silent=True,
start_basecase=True,
f_basecase=None,
pooling=True,
output=None,
results=[]):
# Make timestamp
ts = utils.make_timestamp()
print('Running it {} at time {}'.format(it, ts))
if start_basecase:
strt, sconc = get_base_hds_conc(basecase_ws)
if pooling:
model_ws_orig = Path(m.model_ws).as_posix() + ''
tmp = Path(model_ws).joinpath('tmp{}'.format(it))
if not tmp.exists():
tmp.mkdir()
m.model_ws = tmp.as_posix()
print('temp ws', m.model_ws)
# unpack values from varlist
vka = varlist['vka'][it]
al = varlist['al'][it]
dmcoef = varlist['dmcoef'][it]
ss = varlist['ss'][it]
sy = varlist['sy'][it]
riv_stg = varlist['riv_stg'][it]
riv_cond = varlist['riv_cond'][it]
head_inland_sum = varlist['head_inland_sum'][it]
head_inland_wint = varlist['head_inland_wint'][it]
wel = varlist['wel'][:, it]
rech_farm_pct = varlist['rech_farm'][0]
farm_size = (200, 200)
rech_farm = [rech_farm_pct * flx / np.prod(farm_size) for flx in wel]
rech_precip = varlist['rech'][it]
CF_glob = varlist['CF_glob'][it]
CF_var = varlist['CF_var'][it]
seed = varlist['seed'][it]
hk_mean = varlist['hk_mean'][it]
hk_var = varlist['hk_var'][it]
por_mean = varlist['por_mean'][it]
por_var = varlist['por_var'][it]
corr_len = varlist['corr_len'][it]
corr_len_yx = varlist['corr_len_yx'][it]
corr_len_zx = varlist['corr_len_zx'][it]
clay_lyr_yn = varlist['clay_lyr_yn'][it]
vario_type = varlist['vario_type'][it]
#set ghb data and create dicts
chd_data, ssm_data_base, ghb_data, wel_data_base = make_bc_dicts(
(head_inland_sum, head_inland_wint))
utils.save_obj(m.MC_file.parent, wel_data_base, 'wel_data_base')
utils.save_obj(m.MC_file.parent, ssm_data_base, 'ssm_data_base')
ssm_data = {}
# write recharge data
rech_farm_mat = np.zeros((nrow, ncol), dtype=np.float32)
for i in range(len(rech_farm)):
rech_farm_mat[farm_loc_list[i]] = rech_farm[i]
rech_data = {}
for i in range(len(perlen)):
if i in kper_even:
rech_data[i] = rech_precip
elif i in kper_odd:
rech_data[i] = rech_farm_mat
# write wel data
# ssm_data_base = load_obj(m.MC_file.parent, 'ssm_data_base')
# wel_data_base = load_obj(m.MC_file.parent, 'wel_data_base')
wel_data, ssm_data, wel_cells = add_pumping_wells(wel_data_base,
ssm_data_base,
n_wells,
flx=wel,
rowcol=farm_orig,
kper=kper_odd)
if homogenous == 1:
CF_grid = 1
hk_grid = 10**hk_mean
por_grid = .4
elif homogenous == 2:
#Create Gaussian Simulation
lcol = int(corr_len / delr)
llay = int(corr_len * corr_len_zx / np.mean(delv))
lrow = int(corr_len * corr_len_yx / delc)
# fft_grid = np.exp(simulationFFT.simulFFT(nrow, nlay, ncol, mu, sill, vario_type, lrow , llay, lcol))
CF_grid = simulationFFT.simulFFT(nrow,
nlay,
ncol,
CF_glob,
CF_var,
vario_type,
lrow,
llay,
lcol,
seed=seed)
hk_grid = 10**normal_transform(CF_grid, CF_glob, hk_mean,
np.sqrt(CF_var), np.sqrt(hk_var))
por_grid = normal_transform(CF_grid, CF_glob, por_mean,
np.sqrt(CF_var), np.sqrt(por_var))
CF_grid[CF_grid > 1.] = 1.
CF_grid[CF_grid < 0.] = 0.
por_grid[por_grid > 1.] = .99
por_grid[por_grid < 0.] = 0.01
hk_grid[wel_cells] = np.max((hk_grid.max(), 200))
# np.save(m.MC_file.parent.joinpath('{}_hk.npy'.format(ts)),hk_grid)
else:
pass
# # Write river data--take SSM data from WEL!!
# riv_grad = .0005
# riv_data, ssm_data = write_river_data(
# riv_loc, riv_stg, riv_cond, riv_grad, kper_even, ssm_data)
ipakcb = 53
icbund = np.ones((nlay, nrow, ncol), dtype=np.int)
icbund[np.where(m.bas6.ibound.array == -1)] = -1
timprs = np.round(np.linspace(1, np.sum(perlen), 20), decimals=0)
oc_data = {}
for kper in range(nper):
if kper % 5 == 0:
oc_data[(kper, 0)] = ['save head', 'save budget']
flopy.modflow.ModflowBas(m, m.bas6.ibound.array, strt=strt)
flopy.modflow.ModflowDis(m,
nlay,
nrow,
ncol,
nper=nper,
delr=delr,
delc=delc,
laycbd=0,
top=henry_top,
botm=henry_botm,
perlen=perlen,
nstp=nstp,
steady=steady,
itmuni=itmuni,
lenuni=lenuni,
tsmult=tsmult)
flopy.modflow.ModflowGhb(m, stress_period_data=ghb_data)
flopy.modflow.ModflowWel(m, stress_period_data=wel_data, ipakcb=ipakcb)
## REMOVED FOR CONFINED SIMULATION ##
# flopy.modflow.ModflowRch(m, rech=rech_data)
# flopy.modflow.ModflowRiv(m, stress_period_data=riv_data)
# Add LPF package to the MODFLOW model
flopy.modflow.ModflowLpf(m,
hk=hk_grid,
vka=vka,
ipakcb=ipakcb,
laytyp=0,
laywet=1,
ss=ss,
sy=sy)
# Add PCG Package to the MODFLOW model
flopy.modflow.ModflowPcg(m, hclose=1.e-8)
# Add OC package to the MODFLOW model
flopy.modflow.ModflowOc(m, stress_period_data=oc_data, compact=True)
# Create the basic MT3DMS model structure
flopy.mt3d.Mt3dBtn(m,
laycon=m.lpf.laytyp,
htop=henry_top,
dz=m.dis.thickness.get_value(),
prsity=por_grid,
icbund=icbund,
sconc=sconc,
nprs=1,
timprs=timprs)
flopy.mt3d.Mt3dAdv(m, mixelm=-1)
flopy.mt3d.Mt3dDsp(m, al=al, dmcoef=dmcoef)
flopy.mt3d.Mt3dGcg(m, iter1=50, mxiter=1, isolve=1, cclose=1e-5)
flopy.mt3d.Mt3dSsm(m, stress_period_data=ssm_data)
#vdf = flopy.seawat.SeawatVdf(m, iwtable=0, densemin=0, densemax=0,denseref=1000., denseslp=0.7143, firstdt=1e-3)
flopy.seawat.SeawatVdf(m,
mtdnconc=1,
mfnadvfd=1,
nswtcpl=0,
iwtable=1,
densemin=0.,
densemax=0.,
denseslp=denseslp,
denseref=densefresh)
# Write input
m.write_input()
# Try to delete the output files, to prevent accidental use of older files
flist = [
os.path.join(model_ws, 'MT3D.CNF'),
os.path.join(model_ws, 'MT3D001.MAS'),
os.path.join(model_ws, modelname + '.hds'),
os.path.join(model_ws, 'MT3D001.UCN'),
os.path.join(model_ws, 'MT3D001.UCN'),
os.path.join(model_ws, modelname + '.cbc')
]
for f in flist:
try:
os.remove(f)
except:
pass
# Plot model?
if plotyn:
m.plot_hk_ibound(rowslice=farm_orig[0][0], gridon=True)
# Run model
if runyn:
v = m.run_model(silent=silent, report=True)
for idx in range(-3, 0): # Report
print(v[1][idx])
# Record success/failure and store data
varlist['success'][it] = v[0]
if v[0] is False:
pass
else:
# Record final salinity as .npy, also move full CBC and UCN files
# to expt folder
fname = os.path.join(m.model_ws, 'MT3D001.UCN')
totim = flopy.utils.binaryfile.UcnFile(fname).get_times()[-1]
conc_fname = 'conc{}_{}_totim{}.UCN'.format(
it, ts, str(int(totim)))
_ = record_salinity(
m,
ts_hms=ts,
fname_write=m.MC_file.parent.joinpath(conc_fname))
utils.copy_rename(os.path.join(m.model_ws, 'MT3D001.UCN'),
m.MC_file.parent.joinpath(conc_fname).as_posix())
if pooling:
try:
# [print(p) for p in tmp.iterdir() if (p.suffix is not '.UCN')]
[
p.unlink() for p in tmp.iterdir()
if (p.suffix not in ('.UCN', '.list'))
]
# shutil.rmtree(tmp.as_posix())
# tmp.rmdir()
except:
print('didnt work!')
pass
m.model_ws = model_ws_orig
print('resetting ws:', m.model_ws)
if output is None:
return (it, varlist['success'][it])
else:
output.put((it, varlist['success'][it]))
# results.append((it,varlist['success'][it]))
return
else:
utils.save_obj(m.MC_file.parent, (it, varlist['success'][it]),
'success{}'.format(it))
[
p.unlink() for p in model_ws.iterdir()
if (p.suffix not in ('.UCN', '.list'))
]
return m, varlist
def suggest_form():
if flask.request.method == 'GET':
args = dict(flask.request.args)
elif flask.request.method == 'POST':
args = dict(flask.request.form)
if 'g-recaptcha-response' in args:
rec_check = requests.post(
'https://www.google.com/recaptcha/api/siteverify', {
'secret': app.config.get('suggestions')['recaptcha_secret'],
'response': args['g-recaptcha-response']
})
if not rec_check.json()['success']:
return flask.Response(status=400)
else:
return flask.Response(status=400)
args['timestamp'] = utils.make_timestamp()
args['types'] = ', '.join(topic for topic in SUGGESTION_TYPES
if (topic in args and args[topic] == 'on'))
flask.g.db['suggestions'].insert_one(args)
message = flask_mail.Message(
'New suggestion for the COVID-19 Data Portal',
sender=flask.current_app.config.get('mail')['email'],
recipients=app.config.get('suggestions')['email_receivers'])
if args['origin'] == 'general_contact': # some other way to check which form it is
mail_body = GENERAL_SUGGESTION_MAIL_BODY[:] # potentially use a different template for different forms
mail_body = mail_body.replace('PLACEHOLDER_TIMESTAMP',
str(args['timestamp']))
mail_body = mail_body.replace('PLACEHOLDER_NAME', args['Name'])
mail_body = mail_body.replace('PLACEHOLDER_EMAIL', args['Email'])
mail_body = mail_body.replace('PLACEHOLDER_DESCRIPTION',
args['Description'])
mail_body = mail_body.replace('PLACEHOLDER_TYPE', args['types'])
elif args['origin'] == 'omicron_voc_announcement':
mail_body = OMICRON_VOC_ANNOUNCEMENT_MAIL_BODY[:]
mail_body = mail_body.replace('PLACEHOLDER_TIMESTAMP',
str(args['timestamp']))
mail_body = mail_body.replace('PLACEHOLDER_OMICRON_ANN_NAME',
args['omicron_ann_name'])
mail_body = mail_body.replace('PLACEHOLDER_OMICRON_ANN_CONTACT',
args['omicron_ann_contact'])
mail_body = mail_body.replace('PLACEHOLDER_OMICRON_ANN_TITLE',
args['omicron_ann_title'])
mail_body = mail_body.replace('PLACEHOLDER_OMICRON_ANN_TEXT',
args['omicron_ann_text'])
elif args['origin'] == 'omicron_voc_av_data':
mail_body = OMICRON_VOC_AV_DATA_MAIL_BODY[:]
mail_body = mail_body.replace('PLACEHOLDER_TIMESTAMP',
str(args['timestamp']))
mail_body = mail_body.replace('PLACEHOLDER_OMICRON_AV_DATA_NAME',
args['omicron_av_data_name'])
mail_body = mail_body.replace('PLACEHOLDER_OMICRON_AV_DATA_EMAIL',
args['omicron_av_data_email'])
mail_body = mail_body.replace('PLACEHOLDER_OMICRON_AV_DATA_LINK',
args['omicron_av_data_link'])
mail_body = mail_body.replace(
'PLACEHOLDER_OMICRON_AV_DATA_DESCRIPTION',
args['omicron_av_data_description'])
message.body = mail_body
mail.send(message)
if 'originUrl' in args:
page = SUCCESS_PAGE.replace('PLACEHOLDER', args['originUrl'])
else:
page = SUCCESS_PAGE.replace(
'<a href="PLACEHOLDER">Back to the form.</a>', '')
return flask.Response(page, status=200)