def run_me(location_dir, max_t, max_d): data_file = os.path.join(IN_DIR, location_dir, 'simulation.pickle') out_dir = os.path.join(OUT_DIR, location_dir) log_file = os.path.join(out_dir, 'simulation_%.2f-%.2f.log' % (max_t, max_d)) if not os.path.isdir(out_dir): os.makedirs(out_dir) # set loggers logger = logging.getLogger('simulation_study') logger.setLevel(logging.DEBUG) fh = logging.FileHandler(log_file) formatter = logging.Formatter( '%(asctime)s - %(name)s - %(levelname)s - %(message)s') fh.setFormatter(formatter) logger.addHandler(fh) for t in ('point_process.models', 'kde.models'): this_logger = logging.getLogger(t) this_logger.setLevel(logging.DEBUG) this_logger.handlers = [] # clear existing handlers this_logger.addHandler(fh) # replace with the same file output logger.info("Logger set. Script started.") logger.info("Simulation study.") # load data with open(data_file, 'r') as f: data = pickle.load(f) logger.info("Loaded data.") logger.info("Instantiating SEPP object") r = pp_models.SeppStochasticNn(data=data, max_delta_d=max_d, max_delta_t=max_t, bg_kde_kwargs=bg_kde_kwargs, trigger_kde_kwargs=trigger_kde_kwargs) p = estimation.estimator_bowers(data, r.linkage) r.p = p r.set_seed(42) try: logger.info("Starting training run.") r.train(niter=niter) except Exception as exc: logger.error(repr(exc)) res = None finally: file_stem = os.path.join(out_dir, 'simulation_%.2f-%.2f' % (max_t, max_d)) logger.info("Saving results (or None).") with open(file_stem + '-sepp_obj.pickle', 'w') as f: pickle.dump(r, f)
def different_time_kernel(): mu = 1. st = 1. t = np.linspace(-3, 5, 500) k1 = k_models.kernels.MultivariateNormal([mu], [st]) k2 = k_models.kernels.SpaceTimeNormalReflective([mu], [st]) z1 = k1.pdf(t) z2 = k2.pdf(t) t_refl = t[t<0][::-1] z_refl = k1.pdf(t_refl) import seaborn as sns sns.set_context("paper", font_scale=2.0) plt.figure(figsize=(8, 6)) plt.plot(t, z1, 'k') plt.plot(t, z2, 'r-') plt.plot(-t_refl, z_refl, 'r--') plt.legend(('Normal', 'Reflective')) plt.xlabel('Time (days)') plt.ylabel('Density') c = simulate.MohlerSimulation() c.run() data = c.data max_delta_t = 100 max_delta_d = 1. init_est_params = { 'ct': 10, 'cd': .05, 'frac_bg': 0.5, } ra = pp_models.SeppStochasticNn(data=data, max_delta_d=max_delta_d, max_delta_t=max_delta_t) rb = pp_models.SeppStochasticNnReflected(data=data, max_delta_d=max_delta_d, max_delta_t=max_delta_t) ra.train(niter=20) rb.train(niter=20) t = np.linspace(-10, 60, 500) za = ra.trigger_kde.marginal_pdf(t, dim=0, normed=False) / float(ra.ndata) zb = rb.trigger_kde.marginal_pdf(t, dim=0, normed=False) / float(rb.ndata) w = c.trigger_params['time_decay'] th = c.trigger_params['intensity'] ztrue = th * w * np.exp(-w * t) ztrue[t<0] = 0 plt.figure(figsize=(8, 6)) plt.plot(t, za, 'k-') plt.plot(t, zb, 'r-') plt.plot(t, ztrue, 'k--') plt.legend(('Inferred, normal', 'Inferred, reflective', 'True')) plt.xlabel('Time (days)') plt.ylabel('Density')
def simulation_mohler(): c = simulate.MohlerSimulation() c.run() data = c.data max_delta_t = 100. max_delta_d = 1. initial_est = lambda x, y: estimation.estimator_exp_gaussian(x, y, ct=0.1, cd=0.1) r = pp_models.SeppStochasticNn(data=data, max_delta_d=max_delta_d, max_delta_t=max_delta_t, estimation_function=initial_est) r.train(niter=50) pp_plotting.multiplots(r, c)
def apply_sepp_stochastic_nn(data, data_index, domain, grid_squares=None, max_t=90, max_d=500, num_nn=(100, 15), niter_training=50, num_sample_points=10, seed=43): est_fun = lambda x, y: estimation.estimator_bowers_fixed_proportion_bg( x, y, ct=1, cd=10, frac_bg=0.5) trigger_kde_kwargs = {'strict': False, 'number_nn': num_nn[-1]} bg_kde_kwargs = {'strict': True, 'number_nn': num_nn} sepp = pp_models.SeppStochasticNn(data=data, max_delta_t=max_t, max_delta_d=max_d, seed=seed, estimation_function=est_fun, trigger_kde_kwargs=trigger_kde_kwargs, bg_kde_kwargs=bg_kde_kwargs) vb = validate.SeppValidationFixedModelIntegration(data=data, model=sepp, data_index=data_index, spatial_domain=domain, cutoff_t=INITIAL_CUTOFF + T0) if grid_squares: vb.roc.set_sample_units_predefined(grid_squares, num_sample_points) else: vb.set_sample_units(250, num_sample_points) res = vb.run(time_step=1, n_iter=100, verbose=True, train_kwargs={'niter': niter_training}) return res
training_xy = training[:, 1:] - training[:, 1:].mean(axis=0) training_xy = np.dot(rot_mat(np.pi / 4.), training_xy.transpose()).transpose() training[:, 1:] = training_xy # sepp_isotropic = pp_models.SeppStochasticNnIsotropicTrigger(data=training, **model_kwargs) # ps_isotropic = sepp_isotropic.train(niter=niter) # sepp_local = pp_models.LocalSeppDeterministicNn(data=training, **model_kwargs) # ps_local = sepp_local.train(niter=niter) # sepp_det = pp_models.SeppDeterministicNn(data=training, **model_kwargs) # ps_det = sepp_det.train(niter=niter) sepp_xy = pp_models.SeppStochasticNn(data=training, **model_kwargs) sepp_xy.trigger_kde_class = k_models.VariableBandwidthNnTimeGteZeroKde ps_xy = sepp_xy.train(niter=niter) # sepp_fixed = pp_models.SeppStochasticPluginBandwidth(data=training, **model_kwargs) # model_kwargs['bg_kde_kwargs']['bandwidths'] = [1., 10., 10.] # model_kwargs['trigger_kde_kwargs']['bandwidths'] = [1., 10., 10.] # sepp_fixed = pp_models.SeppStochastic(data=training, **model_kwargs) # ps_fixed = sepp_fixed.train(niter=niter) from scipy.stats import wilcoxon ## standardise the data # s = np.std(training, axis=0, ddof=1) ## to preserve scaling, use same std on both X and Y
for i in range(s0.size): tt = sorted([s0.flat[i], s1.flat[i]]) if tuple(tt) in sepp_obj: continue c = simulate.MohlerSimulation() c.t_total = t_total c.bg_params[0]['sigma'] = [1., 1.] c.bg_params[0]['intensity'] = 5 c.trigger_sigma = list(tt) init_est = lambda d, t: estimation.estimator_exp_gaussian( d, t, ct=0.1, cd=np.mean(tt)) c.run() data = c.data r = models.SeppStochasticNn(data=data, max_delta_d=max_delta_d, max_delta_t=max_delta_t, estimation_function=init_est, seed=42, bg_kde_kwargs=bg_kde_kwargs, trigger_kde_kwargs=trigger_kde_kwargs) try: _ = r.train(niter=num_iter) except Exception: continue sepp_obj[tuple(tt)] = copy.deepcopy(r) fullfile = os.path.join(OUT_DIR, '%.2f_%.2f.pickle' % (tt[0], tt[1])) r.pickle(fullfile) finally: os.remove(LOCK_FILE)
x, y, ct=1, cd=10, frac_bg=0.5) # trigger_kde_kwargs = {'strict': False} trigger_kde_kwargs = {'bandwidths': [10, 40, 40]} bg_kde_kwargs = {'strict': False} # sepp = pp_models.SeppStochasticNnBgFixedTrigger(data=data, # max_delta_t=max_t, # max_delta_d=max_d, # seed=seed, # estimation_function=est_fun, # trigger_kde_kwargs=trigger_kde_kwargs, # bg_kde_kwargs=bg_kde_kwargs) sepp = pp_models.SeppStochasticNn(data=data, max_delta_t=max_t, max_delta_d=max_d, seed=seed, estimation_function=est_fun, trigger_kde_kwargs=trigger_kde_kwargs, bg_kde_kwargs=bg_kde_kwargs) # sepp = pp_models.SeppStochasticNnIsotropicReflectedTrigger(data=data, # max_delta_t=max_t, # max_delta_d=max_d, # seed=seed, # estimation_function=est_fun, # trigger_kde_kwargs=trigger_kde_kwargs, # bg_kde_kwargs=bg_kde_kwargs) ps = sepp.train(niter=50)
# data[:, 1:] /= scaling scaling = 1. # est_fun = lambda x, y: estimation.estimator_exp_gaussian(x, y, ct=0.1, cd=50, frac_bg=0.8) est_fun = lambda x, y: estimation.estimator_exp_gaussian( x, y, ct=0.1, cd=50 / scaling, frac_bg=None) for num_nn in nns: trigger_kde_kwargs = {'strict': False, 'number_nn': num_nn[-1]} bg_kde_kwargs = {'strict': False, 'number_nn': list(num_nn)} sepp = pp_models.SeppStochasticNn( data=data, max_delta_t=max_t, max_delta_d=max_d / scaling, seed=42, estimation_function=est_fun, trigger_kde_kwargs=trigger_kde_kwargs, bg_kde_kwargs=bg_kde_kwargs, remove_coincident_pairs=True) sepp.train(niter=niter) filename = 'nn_%d_%d.pickle' % num_nn sepp.pickle(os.path.join(outdir, filename)) res_chic_n[num_nn] = copy.deepcopy(sepp) trigger_kde_kwargs = { 'strict': False, } bg_kde_kwargs = { 'strict': True, }
pre_start_date = start_date - datetime.timedelta(days=training_size) ndays = training_size + num_validation # end date is the last date retrieved from the database of crimes # have to cast this to a date since the addition operation automatically produces a datetime end_date = start_date + datetime.timedelta(days=num_validation - 1) res, t0, cid = chicago.get_crimes_by_type( crime_type='burglary', start_date=pre_start_date, end_date=end_date, ) # train a model training_data = res[res[:, 0] < training_size] r = models.SeppStochasticNn(data=training_data, **model_kwargs) r.train(niter=niter) # centroid method vb_centroid = {} res_centroid = {} vb_centroid_500 = validate.SeppValidationPredefinedModel( data=res, model=r, spatial_domain=south, cutoff_t=training_size) vb_centroid_500.set_sample_units(500) res_centroid[500] = vb_centroid_500.run(time_step=1, n_iter=num_validation, verbose=True) vb_centroid[500] = vb_centroid_500