def log(): """ Provide a basic data structure for a log document. Returns: dict: The data structure for a log. """ return { "_id": "l-" + utils.new_uuid(), "action": "", "comment": "", "data_type": "", "data": "", "timestamp": utils.make_timestamp(), "user": "", }
def main(): train = pd.read_pickle(TRAIN_PATH) X_test = pd.read_pickle(TEST_PATH) X_train = train.drop('isFraud', axis=1) y_train = train['isFraud'] proba, experiment_id, run_uuid = train_and_predict(X_train, y_train, X_test, PARAMS, EXPRIMENT_PATH) timestamp = make_timestamp() make_submission( os.path.join(SUBMISSION_DIR, f'{timestamp}_{run_uuid}.csv'), proba) print_divider('MLflow UI') print('Run URL: http://127.0.0.1:5000/#/experiments/{0}/runs/{1}\n'.format( experiment_id, run_uuid)) mlflow_ui()
def add_biobank_form(): if flask.request.method == 'GET': args = dict(flask.request.args) elif flask.request.method == 'POST': args = dict(flask.request.form) token = args.get('token') if not token or token not in flask.current_app.config.get('tokens'): if 'originUrl' in args: page = FAIL_PAGE.replace('PLACEHOLDER', args['originUrl']) else: page = FAIL_PAGE.replace( '<a href="PLACEHOLDER">Back to the form.</a>', '') return flask.Response(page, status=401) args['timestamp'] = utils.make_timestamp() flask.g.db['responsesAddBiobank'].insert_one(args) if 'originUrl' in args: page = SUCCESS_PAGE.replace('PLACEHOLDER', args['originUrl']) else: page = SUCCESS_PAGE.replace( '<a href="PLACEHOLDER">Back to the form.</a>', '') return flask.Response(page, status=200)
def update_run_model(varlist, it, m=m, homogenous=2, runyn=True, plotyn=False, silent=True, start_basecase=True, f_basecase=None, pooling=True, output=None, results=[]): # Make timestamp ts = utils.make_timestamp() print('Running it {} at time {}'.format(it, ts)) if start_basecase: strt, sconc = get_base_hds_conc(basecase_ws) if pooling: model_ws_orig = Path(m.model_ws).as_posix() + '' tmp = Path(model_ws).joinpath('tmp{}'.format(it)) if not tmp.exists(): tmp.mkdir() m.model_ws = tmp.as_posix() print('temp ws', m.model_ws) # unpack values from varlist vka = varlist['vka'][it] al = varlist['al'][it] dmcoef = varlist['dmcoef'][it] ss = varlist['ss'][it] sy = varlist['sy'][it] riv_stg = varlist['riv_stg'][it] riv_cond = varlist['riv_cond'][it] head_inland_sum = varlist['head_inland_sum'][it] head_inland_wint = varlist['head_inland_wint'][it] wel = varlist['wel'][:, it] rech_farm_pct = varlist['rech_farm'][0] farm_size = (200, 200) rech_farm = [rech_farm_pct * flx / np.prod(farm_size) for flx in wel] rech_precip = varlist['rech'][it] CF_glob = varlist['CF_glob'][it] CF_var = varlist['CF_var'][it] seed = varlist['seed'][it] hk_mean = varlist['hk_mean'][it] hk_var = varlist['hk_var'][it] por_mean = varlist['por_mean'][it] por_var = varlist['por_var'][it] corr_len = varlist['corr_len'][it] corr_len_yx = varlist['corr_len_yx'][it] corr_len_zx = varlist['corr_len_zx'][it] clay_lyr_yn = varlist['clay_lyr_yn'][it] vario_type = varlist['vario_type'][it] #set ghb data and create dicts chd_data, ssm_data_base, ghb_data, wel_data_base = make_bc_dicts( (head_inland_sum, head_inland_wint)) utils.save_obj(m.MC_file.parent, wel_data_base, 'wel_data_base') utils.save_obj(m.MC_file.parent, ssm_data_base, 'ssm_data_base') ssm_data = {} # write recharge data rech_farm_mat = np.zeros((nrow, ncol), dtype=np.float32) for i in range(len(rech_farm)): rech_farm_mat[farm_loc_list[i]] = rech_farm[i] rech_data = {} for i in range(len(perlen)): if i in kper_even: rech_data[i] = rech_precip elif i in kper_odd: rech_data[i] = rech_farm_mat # write wel data # ssm_data_base = load_obj(m.MC_file.parent, 'ssm_data_base') # wel_data_base = load_obj(m.MC_file.parent, 'wel_data_base') wel_data, ssm_data, wel_cells = add_pumping_wells(wel_data_base, ssm_data_base, n_wells, flx=wel, rowcol=farm_orig, kper=kper_odd) if homogenous == 1: CF_grid = 1 hk_grid = 10**hk_mean por_grid = .4 elif homogenous == 2: #Create Gaussian Simulation lcol = int(corr_len / delr) llay = int(corr_len * corr_len_zx / np.mean(delv)) lrow = int(corr_len * corr_len_yx / delc) # fft_grid = np.exp(simulationFFT.simulFFT(nrow, nlay, ncol, mu, sill, vario_type, lrow , llay, lcol)) CF_grid = simulationFFT.simulFFT(nrow, nlay, ncol, CF_glob, CF_var, vario_type, lrow, llay, lcol, seed=seed) hk_grid = 10**normal_transform(CF_grid, CF_glob, hk_mean, np.sqrt(CF_var), np.sqrt(hk_var)) por_grid = normal_transform(CF_grid, CF_glob, por_mean, np.sqrt(CF_var), np.sqrt(por_var)) CF_grid[CF_grid > 1.] = 1. CF_grid[CF_grid < 0.] = 0. por_grid[por_grid > 1.] = .99 por_grid[por_grid < 0.] = 0.01 hk_grid[wel_cells] = np.max((hk_grid.max(), 200)) # np.save(m.MC_file.parent.joinpath('{}_hk.npy'.format(ts)),hk_grid) else: pass # # Write river data--take SSM data from WEL!! # riv_grad = .0005 # riv_data, ssm_data = write_river_data( # riv_loc, riv_stg, riv_cond, riv_grad, kper_even, ssm_data) ipakcb = 53 icbund = np.ones((nlay, nrow, ncol), dtype=np.int) icbund[np.where(m.bas6.ibound.array == -1)] = -1 timprs = np.round(np.linspace(1, np.sum(perlen), 20), decimals=0) oc_data = {} for kper in range(nper): if kper % 5 == 0: oc_data[(kper, 0)] = ['save head', 'save budget'] flopy.modflow.ModflowBas(m, m.bas6.ibound.array, strt=strt) flopy.modflow.ModflowDis(m, nlay, nrow, ncol, nper=nper, delr=delr, delc=delc, laycbd=0, top=henry_top, botm=henry_botm, perlen=perlen, nstp=nstp, steady=steady, itmuni=itmuni, lenuni=lenuni, tsmult=tsmult) flopy.modflow.ModflowGhb(m, stress_period_data=ghb_data) flopy.modflow.ModflowWel(m, stress_period_data=wel_data, ipakcb=ipakcb) ## REMOVED FOR CONFINED SIMULATION ## # flopy.modflow.ModflowRch(m, rech=rech_data) # flopy.modflow.ModflowRiv(m, stress_period_data=riv_data) # Add LPF package to the MODFLOW model flopy.modflow.ModflowLpf(m, hk=hk_grid, vka=vka, ipakcb=ipakcb, laytyp=0, laywet=1, ss=ss, sy=sy) # Add PCG Package to the MODFLOW model flopy.modflow.ModflowPcg(m, hclose=1.e-8) # Add OC package to the MODFLOW model flopy.modflow.ModflowOc(m, stress_period_data=oc_data, compact=True) # Create the basic MT3DMS model structure flopy.mt3d.Mt3dBtn(m, laycon=m.lpf.laytyp, htop=henry_top, dz=m.dis.thickness.get_value(), prsity=por_grid, icbund=icbund, sconc=sconc, nprs=1, timprs=timprs) flopy.mt3d.Mt3dAdv(m, mixelm=-1) flopy.mt3d.Mt3dDsp(m, al=al, dmcoef=dmcoef) flopy.mt3d.Mt3dGcg(m, iter1=50, mxiter=1, isolve=1, cclose=1e-5) flopy.mt3d.Mt3dSsm(m, stress_period_data=ssm_data) #vdf = flopy.seawat.SeawatVdf(m, iwtable=0, densemin=0, densemax=0,denseref=1000., denseslp=0.7143, firstdt=1e-3) flopy.seawat.SeawatVdf(m, mtdnconc=1, mfnadvfd=1, nswtcpl=0, iwtable=1, densemin=0., densemax=0., denseslp=denseslp, denseref=densefresh) # Write input m.write_input() # Try to delete the output files, to prevent accidental use of older files flist = [ os.path.join(model_ws, 'MT3D.CNF'), os.path.join(model_ws, 'MT3D001.MAS'), os.path.join(model_ws, modelname + '.hds'), os.path.join(model_ws, 'MT3D001.UCN'), os.path.join(model_ws, 'MT3D001.UCN'), os.path.join(model_ws, modelname + '.cbc') ] for f in flist: try: os.remove(f) except: pass # Plot model? if plotyn: m.plot_hk_ibound(rowslice=farm_orig[0][0], gridon=True) # Run model if runyn: v = m.run_model(silent=silent, report=True) for idx in range(-3, 0): # Report print(v[1][idx]) # Record success/failure and store data varlist['success'][it] = v[0] if v[0] is False: pass else: # Record final salinity as .npy, also move full CBC and UCN files # to expt folder fname = os.path.join(m.model_ws, 'MT3D001.UCN') totim = flopy.utils.binaryfile.UcnFile(fname).get_times()[-1] conc_fname = 'conc{}_{}_totim{}.UCN'.format( it, ts, str(int(totim))) _ = record_salinity( m, ts_hms=ts, fname_write=m.MC_file.parent.joinpath(conc_fname)) utils.copy_rename(os.path.join(m.model_ws, 'MT3D001.UCN'), m.MC_file.parent.joinpath(conc_fname).as_posix()) if pooling: try: # [print(p) for p in tmp.iterdir() if (p.suffix is not '.UCN')] [ p.unlink() for p in tmp.iterdir() if (p.suffix not in ('.UCN', '.list')) ] # shutil.rmtree(tmp.as_posix()) # tmp.rmdir() except: print('didnt work!') pass m.model_ws = model_ws_orig print('resetting ws:', m.model_ws) if output is None: return (it, varlist['success'][it]) else: output.put((it, varlist['success'][it])) # results.append((it,varlist['success'][it])) return else: utils.save_obj(m.MC_file.parent, (it, varlist['success'][it]), 'success{}'.format(it)) [ p.unlink() for p in model_ws.iterdir() if (p.suffix not in ('.UCN', '.list')) ] return m, varlist
def suggest_form(): if flask.request.method == 'GET': args = dict(flask.request.args) elif flask.request.method == 'POST': args = dict(flask.request.form) if 'g-recaptcha-response' in args: rec_check = requests.post( 'https://www.google.com/recaptcha/api/siteverify', { 'secret': app.config.get('suggestions')['recaptcha_secret'], 'response': args['g-recaptcha-response'] }) if not rec_check.json()['success']: return flask.Response(status=400) else: return flask.Response(status=400) args['timestamp'] = utils.make_timestamp() args['types'] = ', '.join(topic for topic in SUGGESTION_TYPES if (topic in args and args[topic] == 'on')) flask.g.db['suggestions'].insert_one(args) message = flask_mail.Message( 'New suggestion for the COVID-19 Data Portal', sender=flask.current_app.config.get('mail')['email'], recipients=app.config.get('suggestions')['email_receivers']) if args['origin'] == 'general_contact': # some other way to check which form it is mail_body = GENERAL_SUGGESTION_MAIL_BODY[:] # potentially use a different template for different forms mail_body = mail_body.replace('PLACEHOLDER_TIMESTAMP', str(args['timestamp'])) mail_body = mail_body.replace('PLACEHOLDER_NAME', args['Name']) mail_body = mail_body.replace('PLACEHOLDER_EMAIL', args['Email']) mail_body = mail_body.replace('PLACEHOLDER_DESCRIPTION', args['Description']) mail_body = mail_body.replace('PLACEHOLDER_TYPE', args['types']) elif args['origin'] == 'omicron_voc_announcement': mail_body = OMICRON_VOC_ANNOUNCEMENT_MAIL_BODY[:] mail_body = mail_body.replace('PLACEHOLDER_TIMESTAMP', str(args['timestamp'])) mail_body = mail_body.replace('PLACEHOLDER_OMICRON_ANN_NAME', args['omicron_ann_name']) mail_body = mail_body.replace('PLACEHOLDER_OMICRON_ANN_CONTACT', args['omicron_ann_contact']) mail_body = mail_body.replace('PLACEHOLDER_OMICRON_ANN_TITLE', args['omicron_ann_title']) mail_body = mail_body.replace('PLACEHOLDER_OMICRON_ANN_TEXT', args['omicron_ann_text']) elif args['origin'] == 'omicron_voc_av_data': mail_body = OMICRON_VOC_AV_DATA_MAIL_BODY[:] mail_body = mail_body.replace('PLACEHOLDER_TIMESTAMP', str(args['timestamp'])) mail_body = mail_body.replace('PLACEHOLDER_OMICRON_AV_DATA_NAME', args['omicron_av_data_name']) mail_body = mail_body.replace('PLACEHOLDER_OMICRON_AV_DATA_EMAIL', args['omicron_av_data_email']) mail_body = mail_body.replace('PLACEHOLDER_OMICRON_AV_DATA_LINK', args['omicron_av_data_link']) mail_body = mail_body.replace( 'PLACEHOLDER_OMICRON_AV_DATA_DESCRIPTION', args['omicron_av_data_description']) message.body = mail_body mail.send(message) if 'originUrl' in args: page = SUCCESS_PAGE.replace('PLACEHOLDER', args['originUrl']) else: page = SUCCESS_PAGE.replace( '<a href="PLACEHOLDER">Back to the form.</a>', '') return flask.Response(page, status=200)