def one_instance(**kwargs): if kwargs: c_l = kwargs['c_l'] g_r = kwargs['g_r'] o_s = kwargs['o_s'] delta_t, sim_duration, dspt_times, \ stop_locs, demand_rates, board_rates, stop_num, demand_start_times, \ link_mean_speeds, link_cv_speeds, link_lengths, link_start_locs, \ cycle_lengths, green_ratios, signal_offsets, signal_locs, examined_signal \ = get_parameters(cycle_length=c_l, green_ratio=g_r, off_set=o_s) else: delta_t, sim_duration, dspt_times, \ stop_locs, demand_rates, board_rates, stop_num, demand_start_times, \ link_mean_speeds, link_cv_speeds, link_lengths, link_start_locs, \ cycle_lengths, green_ratios, signal_offsets, signal_locs, examined_signal \ = get_parameters() for stop in range(stop_num): total_hdws_dict[stop] = [] processes = [] for i in range(process_num): process = Process(target=get_result, args=(batch_no, delta_t, sim_duration, dspt_times, \ stop_locs, demand_rates, board_rates, stop_num, demand_start_times, \ link_mean_speeds, link_cv_speeds, link_lengths, link_start_locs, \ cycle_lengths, green_ratios, signal_offsets, signal_locs, )) process.start() processes.append(process) for process in processes: process.join() arr_means = [] arr_stds = [] for stop in range(stop_num): arr_mean, arr_std = get_mean_std(total_hdws_dict[stop]) arr_stds.append(arr_std) arr_means.append(arr_mean) return arr_means, arr_stds
def kde_startup(): try: project_path, sessions_path = path.project_path(), path.sessions_path() sessions_dir = os.path.join(project_path + sessions_path) id = [] with open(sessions_dir, 'rt') as sessions_r: active = csv.reader(sessions_r, delimiter=',') for _id in active: if session['username'] in _id[0]: id.append(session['username']) if session['username'] in id: params = par_fn.get_parameters() years, months, codes, kernels, bandwidths, metrics = params[0],\ params[1],\ params[2],\ params[3],\ params[4],\ params[5] months, codes, metrics = [month for month in months.keys()],\ [code for code in codes.keys()],\ [metric for metric in metrics.keys()] kde = fn.set_kde_startup() kde_script, kde_div = components(kde) session_status = 'App ready.' return render_template('mkecs-kde.html', kde_script=kde_script, kde_div=kde_div, months=months, years=years, codes=codes, kernels=kernels, bandwidths=bandwidths, metrics=metrics, session_status=session_status, session_id=session['username']) except: return redirect(url_for('mkecs_kde_session'))
def get_confusion_matrix_and_accuracy(self, fold_num): """Método que calcula a matriz de confusao e a acuracia por classe""" letters = u.get_classes_letters_list(self.part_2) # obtenção das imagens do diretório de testes do dataset # função teste chamada para cada imagem # para adicionar os acertos na matriz de confusao posteriormente parameters_test = p.get_parameters(self.descriptor, self.part_2, self.l1_neurons) dataset_validation = u.get_dataset_list( u.get_classes_list(parameters_test['testpath']), parameters_test['testpath']) for dataset in dataset_validation: self.num_tests_images_per_letter = len(dataset) for image_i, image in enumerate(dataset): self.testing(image, image_i, dataset_tests=True) # matriz de confusão obtained = pd.Series(self.test_results, name=' Obtido') predicted = pd.Series(self.test_predicted, name='Esperado') confusion_matrix = pd.crosstab(obtained, predicted) print(confusion_matrix) # plotar matriz de confusão com o uso das bibliotecas seaborn e pyplot df_cm = pd.DataFrame(confusion_matrix, index=[i for i in letters], columns=[i for i in letters]) if self.part_2: plt.figure(figsize=(17, 10)) else: plt.figure(figsize=(10, 7)) sn.heatmap(df_cm, annot=True, cmap='PuBu', fmt='g') plt.ylabel('Obtido') plt.xlabel('Esperado') file_command = '{output}confusion_matrix-{fold}.jpg'.format(fold=fold_num, output=self.output_directory) plt.savefig(file_command) plt.close() # acuracia accuracy = accuracy_score(obtained, predicted) print("\nAcurácia média: {} \n".format(accuracy)) # acuracia por letra calculada dividindo acertos (diagonal da matriz de confusao) # pelo total de imagens que foram testadas daquela letra for letter_i, letter in enumerate(letters): positive = confusion_matrix[letter][letter] total = self.num_tests_images_per_letter print("Acurácia da classe {} :".format(letter), positive / total)
def main(): # Preparation config = get_parameters() transformer = get_time_frequency_transform(config) data_loader_train, data_loader_valid = get_data(config) audio, target, fname = data_loader_train.dataset[0] batch_audio, batch_target, batch_fname = next( iter(data_loader_train)) # Get single batch spec = transformer(audio) batch_specs = transformer(batch_audio) # Sanity check for shapes assert spec.shape[0] == 1, 'Wrong size for spec' assert spec.shape[1] == config.n_fft // 2 + 1, 'Wrong size for spec' assert spec.shape[2] == config.max_length_frames, 'Wrong size for spec' assert batch_specs.shape[ 0] == config.batch_size, 'Wrong size for batch_spec' assert batch_specs.shape[1] == 1, 'Wrong size for batch_spec' assert batch_specs.shape[ 2] == config.n_fft // 2 + 1, 'Wrong size for batch_spec' assert batch_specs.shape[ 3] == config.max_length_frames, 'Wrong size for batch_spec' # Manual plotting import matplotlib.pyplot as plt fig = plt.figure() im = plt.imshow(spec[0].numpy()[::-1, :], cmap=plt.get_cmap('magma')) fig.colorbar(im) plt.show() # Using librosa utils.show_spectrogram(spec, config) print(spec) # Librosa, by batch utils.show_spectrogram_batch(batch_specs, batch_fname, config, gridSize=4)
edge_file = sys.argv[2] poly_file = sys.argv[3] # Parameters lx = 9 * (2 / (3 * (3**0.5)))**0.5 ly = 4 * (2 / (3**0.5))**0.5 ka = 1. A0 = 1. # current preferred area for polygon gamma = 0.04 * ka * A0 # hexagonal network # gamma = 0.1 * ka * A0 # soft network Lambda = 0.12 * ka * (A0**(3 / 2)) # hexagonal network # Lambda = -0.85 * ka * A0**(3/2) # soft network lmin = 0.2 delta_t = 0.05 eta = 1. # get parameter dictionary parameters = get_parameters(lx, ly, ka, gamma, Lambda, eta, lmin, delta_t) # get vertices vertices = read_vertices(vertex_file) # get edges edges = read_edges(edge_file) # get polygons poly_indices = read_poly_indices(poly_file) polys = build_polygons(poly_indices, A0) steepest_descent(vertices, edges, polys, parameters)
def main(): # Reproducibility np.random.seed(12345) torch.manual_seed(12345) # Preparation config = get_parameters() # Logging configuration writer = None if config.tensorboard: path_tensorboard = f'{config.logging_dir}/{config.experiment_description}' if config.debug_mode: # Clear tensorboard when debugging if os.path.exists(path_tensorboard): shutil.rmtree(path_tensorboard) writer = SummaryWriter(path_tensorboard) data_loader_train, data_loader_valid, data_loader_test = get_data(config) if config.use_time_freq: transforms = get_time_frequency_transform(config) else: transforms = None # ===================================================================== # Visualize some data tmp_audio = None tmp_spec = None tmp_data, targets, _ = data_loader_train.dataset[ 0] # audio is [channels, timesteps] # Is the data audio or image? if len(tmp_data.shape) == 2: tmp_audio = tmp_data else: tmp_spec = tmp_data if config.use_time_freq: tmp_spec = transforms( tmp_audio) # spec is [channels, freq_bins, frames] if tmp_spec is not None: utils.show_spectrogram(tmp_spec, config) if writer is not None: if tmp_audio is not None: # Store 5 secs of audio ind = tmp_audio.shape[-1] if tmp_audio.shape[ -1] <= 5 * config.original_fs else 5 * config.original_fs writer.add_audio('input_audio', tmp_audio[:, 0:ind], None, config.original_fs) tmp_audios = [] fnames = [] for i in range(4): aud, _, fn = data_loader_train.dataset.dataset[i] fnames.append(fn) tmp_audios.append(aud) writer.add_figure( 'input_waveform', utils.show_waveforms_batch(tmp_audios, fnames, config), None) # Analyze some spectrograms if tmp_spec is not None: img_tform = tforms_vision.Compose([ tforms_vision.ToPILImage(), tforms_vision.ToTensor(), ]) writer.add_image('input_spec', img_tform(tmp_spec), None) # Raw tensor writer.add_figure('input_spec_single', utils.show_spectrogram(tmp_spec, config), None) # Librosa if config.use_time_freq: tmp_specs = [] fnames = [] for i in range(4): aud, _, fn = data_loader_train.dataset.dataset[i] tmp_specs.append(transforms(aud)) fnames.append(fn) writer.add_figure( 'input_spec_batch', utils.show_spectrogram_batch(tmp_specs, fnames, config), None) writer.add_figure('input_spec_histogram', utils.get_spectrogram_histogram(tmp_specs), None) del tmp_specs, fnames, aud, fn, i # Class Histograms if not config.dataset_skip_class_hist: fig_classes = utils.get_class_histograms( data_loader_train.dataset, data_loader_valid.dataset, data_loader_test.dataset, one_hot_encoder=utils.OneHot if config.dataset == 'MNIST' else None, data_limit=200 if config.debug_mode else None) if writer is not None: writer.add_figure('class_histogram', fig_classes, None) # ===================================================================== # Train and Test solver = Solver(data_loader_train, data_loader_valid, data_loader_test, config, writer, transforms) solver.train() scores, true_class, pred_scores = solver.test() # ===================================================================== # Save results np.save(open(os.path.join(config.result_dir, 'true_class.npy'), 'wb'), true_class) np.save(open(os.path.join(config.result_dir, 'pred_scores.npy'), 'wb'), pred_scores) utils.compare_predictions(true_class, pred_scores, config.result_dir) if writer is not None: writer.close()
SS_FIG_DIR = "../OUTPUT_BASELINE/sigma3.0" COMPARISON_DIR = "../OUTPUT_BASELINE/sigma3.0" ss_init = os.path.join(SS_FIG_DIR, "SS/SS_vars.pkl") variables = pickle.load(open(ss_init, "rb")) for key in variables: globals()[key] = variables[key] # params_given = os.path.join(SS_FIG_DIR, "Saved_moments/params_given.pkl") # variables = pickle.load(open(params_given, "rb")) # for key in variables: # globals()[key] = variables[key] #globals().update(ogusa.parameters.get_parameters_from_file()) globals().update(parameters.get_parameters(True, {}, {}, {})) param_names = ['S', 'J', 'T', 'BW', 'lambdas', 'starting_age', 'ending_age', 'beta', 'sigma', 'alpha', 'nu', 'Z', 'delta', 'E', 'ltilde', 'g_y', 'maxiter', 'mindist_SS', 'mindist_TPI', 'b_ellipse', 'k_ellipse', 'upsilon', 'chi_b_guess', 'chi_n_guess','etr_params','mtrx_params', 'mtry_params','tau_payroll', 'tau_bq', 'retire', 'mean_income_data', 'g_n_vector', 'h_wealth', 'p_wealth', 'm_wealth', 'omega', 'g_n_ss', 'omega_SS', 'surv_rate', 'e', 'rho'] variables = {} for key in param_names: variables[key] = globals()[key] for key in variables: globals()[key] = variables[key]
#molecule = extraction.setup_calculations() """ th = np.expand_dims(np.array([1.803-0.05, 1.118-0.05, 1.695-0.02]), 0) extraction.log_likelihood_density(th) sys.exit(0) """ walkers_init = initialize_walkers(data_parameters, extraction.atom_positions) extraction.run_mcmc(walkers_init, data_parameters["run_limit"]) if __name__ == "__main__": data_parameters = get_parameters() q_max = [10] #q_max = [5, 7.5, 10, 12.5, 15, 17.5, 20] #sigmas = np.insert(1./(10**np.arange(11)), 0, 0.163) ston = [100] #ston = [25, 50, 100, 200, 400] lmk_arr = [[100, 100]] lmk_arr = [[25, 12.5], [25, 25], [25, 50], [25, 100], [12.5, 100], [50, 100]] options = [] for bg in ston: for lg in lmk_arr: adm_params = copy(data_parameters["ADM_params"]) adm_params["temperature"] = lg[0]
lx = L[0] ly = L[1] ka = 1. A0 = 1. gamma = 0.04 * ka * A0 # hexagonal network # gamma = 0.1 * ka * A0 # soft network Lambda = 0.12 * ka * (A0**(3/2)) # hexagonal network # # Lambda = -0.85 * ka * A0**(3/2) # soft network lmin = 0.01 delta_t = 0.05 # get parameter dictionary parameters = get_parameters(lx, ly, ka, gamma, Lambda, lmin, delta_t) # get vertices vertices = read_vertices(vertex_file) # get edges edges = read_edges(edge_file) # get polygons poly_indices = read_poly_indices(poly_file) polys = build_polygons(poly_indices, A0) steepest_descent(vertices, edges, polys, parameters, folder)
''' ------------------------------------------------------------------------ Create variables for SS baseline graphs ------------------------------------------------------------------------ ''' SS_FIG_DIR = "OUTPUT" COMPARISON_DIR = "OUTPUT" ss_init = os.path.join(SS_FIG_DIR, "SSinit/ss_init_vars.pkl") variables = pickle.load(open(ss_init, "rb")) for key in variables: globals()[key] = variables[key] #globals().update(ogindia.parameters.get_parameters_from_file()) globals().update(parameters.get_parameters()) param_names = [ 'S', 'J', 'T', 'BW', 'lambdas', 'starting_age', 'ending_age', 'beta', 'sigma', 'alpha', 'nu', 'Z', 'delta', 'E', 'ltilde', 'g_y', 'maxiter', 'mindist_SS', 'mindist_TPI', 'b_ellipse', 'k_ellipse', 'upsilon', 'chi_b_guess', 'chi_n_guess', 'etr_params', 'mtrx_params', 'mtry_params', 'tau_payroll', 'tau_bq', 'retire', 'mean_income_data', 'g_n_vector', 'h_wealth', 'p_wealth', 'm_wealth', 'omega', 'g_n_ss', 'omega_SS', 'surv_rate', 'e', 'rho' ] variables = {} for key in param_names: variables[key] = globals()[key] for key in variables: globals()[key] = variables[key]
simulator.reset(dspt_times) return stop_headways def get_mean_std(headways): return np.around(np.mean(headways), decimals=1), np.around(np.std(headways), decimals=1) if __name__ == "__main__": batch_no = 1 delta_t, sim_duration, dspt_times, \ stop_locs, demand_rates, board_rates, stop_num, demand_start_times, \ link_mean_speeds, link_cv_speeds, link_lengths, link_start_locs, \ cycle_lengths, green_ratios, signal_offsets, signal_locs, examined_signal \ = get_parameters() stop_headways_dict = get_result(batch_no, delta_t, sim_duration, dspt_times, \ stop_locs, demand_rates, board_rates, stop_num, demand_start_times, \ link_mean_speeds, link_cv_speeds, link_lengths, link_start_locs, \ cycle_lengths, green_ratios, signal_offsets, signal_locs) arr_means = [] arr_stds = [] for stop in range(stop_num): arr_mean, arr_std = get_mean_std(stop_headways_dict[stop]) arr_stds.append(arr_std) arr_means.append(arr_mean) print(arr_means)
path, img_size, batch_size, shuffle=False) class_dict = {} device = torch.device(config.device) model = InceptionV3().to(device) data_iter = dataloader.loader() index = get_index() for batch_idx, (images, labels) in enumerate(data_iter): batch = images.to(device) preds = model(batch)[0] preds = preds.cpu().numpy().reshape((preds.shape[0], preds.shape[1])) index.addDataPointBatch( preds, range(batch_idx * batch_size, (batch_idx + 1) * batch_size)) class_dict = save_images(images, labels, batch_idx, batch_size, class_dict, out_dir) create_index(index, os.path.join(out_dir, 'index.bin')) with open(os.path.join(out_dir, 'class_dict.json'), 'w') as fp: json.dump(class_dict, fp) if __name__ == "__main__": config = get_parameters() main(config)
------------------------------------------------------------------------ Create variables for SS baseline graphs ------------------------------------------------------------------------ ''' SS_FIG_DIR = "OUTPUT" COMPARISON_DIR = "OUTPUT" ss_init = os.path.join(SS_FIG_DIR, "SSinit/ss_init_vars.pkl") variables = pickle.load(open(ss_init, "rb")) for key in variables: globals()[key] = variables[key] #globals().update(ogusa.parameters.get_parameters_from_file()) globals().update(parameters.get_parameters()) param_names = ['S', 'J', 'T', 'BW', 'lambdas', 'starting_age', 'ending_age', 'beta', 'sigma', 'alpha', 'nu', 'Z', 'delta', 'E', 'ltilde', 'g_y', 'maxiter', 'mindist_SS', 'mindist_TPI', 'b_ellipse', 'k_ellipse', 'upsilon', 'chi_b_guess', 'chi_n_guess','etr_params','mtrx_params', 'mtry_params','tau_payroll', 'tau_bq', 'retire', 'mean_income_data', 'g_n_vector', 'h_wealth', 'p_wealth', 'm_wealth', 'omega', 'g_n_ss', 'omega_SS', 'surv_rate', 'e', 'rho'] variables = {} for key in param_names: variables[key] = globals()[key] for key in variables: globals()[key] = variables[key]
def set_annotate_interact(year, month, violation_code): # Build parameters params = par_fn.get_parameters() months = params[1] mo = months[month] # Build data project_path, db_path = path.project_path(), path.db_path() db_dir = os.path.join(project_path + db_path) db = create_engine('sqlite:///' + db_dir + 'mke_wibrs_db.db', echo=False) Session = sessionmaker(bind=db) session = Session() query = session.query(mke_wibrs_db).\ filter(mke_wibrs_db.date >= year + mo[0], mke_wibrs_db.date <= year + mo[1], getattr(mke_wibrs_db, 'code_' + violation_code) == 1, mke_wibrs_db.addr != None, mke_wibrs_db.zip_code != None, mke_wibrs_db.ald_dist != None, mke_wibrs_db.mpd_dist != None, mke_wibrs_db.ald_dist != 0, mke_wibrs_db.mpd_dist != 0, mke_wibrs_db.voting_ward != None, mke_wibrs_db.x_lon != None, mke_wibrs_db.y_lat != None).all() incident_number, date, formatted_addr, ald_dist, mpd_dist = [], [], [], [], [] for data in query: incident_number.append(data.incident_number) date.append('{:%Y-%m-%d %H:%M:%S}'.format(data.date)) formatted_addr.append(data.formatted_addr) ald_dist.append(data.ald_dist) mpd_dist.append(data.mpd_dist) session.close() # Format data ald_dist_int, mpd_dist_int = [int(i) for i in ald_dist ], [int(i) for i in mpd_dist] ald_dist_cnt, mpd_dist_cnt = Counter(ald_dist_int), Counter(mpd_dist_int) ald_dist_srt, mpd_dist_srt = sorted(ald_dist_cnt.items()), sorted( mpd_dist_cnt.items()) # Aldermanic districts & violation counts ald_dist = [str(i[0]) for i in ald_dist_srt] ald_dist_cnt = [i[1] for i in ald_dist_srt] # MPD districts & violation counts mpd_dist = [str(i[0]) for i in mpd_dist_srt] mpd_dist_cnt = [i[1] for i in mpd_dist_srt] ald_source = ColumnDataSource( data=dict(ald_dist=ald_dist, ald_dist_cnt=ald_dist_cnt)) mpd_source = ColumnDataSource( data=dict(mpd_dist=mpd_dist, mpd_dist_cnt=mpd_dist_cnt)) ald_tool_tips = [('District', '@ald_dist'), ('Count', '@ald_dist_cnt')] mpd_tool_tips = [('District', '@mpd_dist'), ('Count', '@mpd_dist_cnt')] ald_p = figure(x_range=ald_dist, plot_height=275, plot_width=750, toolbar_location='right', tools='hover', tooltips=ald_tool_tips) mpd_p = figure(x_range=mpd_dist, plot_height=275, plot_width=750, toolbar_location='right', tools='hover', tooltips=mpd_tool_tips) ald_p.vbar(x='ald_dist', top='ald_dist_cnt', color='#0b5394', width=0.8, source=ald_source) mpd_p.vbar(x='mpd_dist', top='mpd_dist_cnt', color='#0b5394', width=0.8, source=mpd_source) ald_p.xgrid.grid_line_color, mpd_p.xgrid.grid_line_color, ald_p.toolbar.logo, mpd_p.toolbar.logo = None, None, None, None ald_p.y_range.start, mpd_p.y_range.start = 0, 0 # Dataset metrics vio_cnt = sum(ald_dist_cnt) return ald_p, mpd_p, vio_cnt
#!/usr/bin/python3 # -*- coding: UTF-8 -* from concurrent.futures import ProcessPoolExecutor import pymysql as pymysql import features as features import parameters if __name__ == '__main__': parameter_info = parameters.get_parameters() for key, value in parameter_info.items(): print("[" + key + "] : " + value) db_source = pymysql.connect(parameter_info['hostname'], parameter_info['username'], parameter_info['password'], parameter_info["database_source"], charset='utf8mb4') cursor_source = db_source.cursor() sql = "SELECT `ID` FROM `{0}`.`{1}` WHERE ".format( parameter_info["database_source"], parameter_info["table_captured"]) where_info = "" types = str(parameter_info["type"]).split(',') for index in range(0, len(types)): where_info += "`TYPE`='{0}'".format(types[index]) if index is not len(types) - 1: where_info += " OR " sql += where_info + ";" print(sql)
training_this_round, testing_this_round, fold_i)) thread_list.append(t) # Starts threads for thread in thread_list: thread.start() for thread in thread_list: thread.join() # início da execução if __name__ == "__main__": # horário de execução, descritor, parâmetros, diretórios, lista de classes e lista de dataset start_algorithm = datetime.now() arguments = get_arguments() parameters = p.get_parameters(arguments['descriptor'], 'part2' in sys.argv, arguments['neurons'], arguments['output']) print(parameters) u.create_directories(['data', 'src', 'output']) dataset = u.get_dataset_list(u.get_classes_list(parameters['workpath']), parameters['workpath']) k_fold(dataset, len(dataset), parameters, start_algorithm) print("Main Start Time: \t\t\t\t\t\t{}".format( start_algorithm.strftime("%Y-%m-%d %H:%M:%S"))) print("Main End Time: \t\t\t\t\t\t{}".format( datetime.now().strftime("%Y-%m-%d %H:%M:%S"))) print("Total time running: \t\t\t\t\t\t{}\n".format(datetime.now() - start_algorithm))
if dump_activations and dump_path is not None: for idx in range(len(models)): save_activations(idx, activations[idx], dump_path) # Remove the hooks (as this was intefering with prediction ensembling) for idx in range(len(forward_hooks)): for hook in forward_hooks[idx]: hook.remove() # print("selective activations returned are", activations) return activations if __name__ == '__main__': args = parameters.get_parameters(options_type='mnist_act', deprecated=True) config = vgg_hyperparams.config model_list = os.listdir(ensemble_dir) num_models = len(model_list) train_loader, test_loader = cifar_train.get_dataset(config) # Load models models = [] for idx in range(num_models): print("Path is ", ensemble_dir) print("loading model with idx {} and checkpoint_type is {}".format( idx, checkpoint_type)) models.append(
def kde_interact(): try: project_path, sessions_path = path.project_path(), path.sessions_path() sessions_dir = os.path.join(project_path + sessions_path) id = [] with open(sessions_dir, 'rt') as sessions_r: active = csv.reader(sessions_r, delimiter=',') for _id in active: if session['username'] in _id[0]: id.append(session['username']) if session['username'] in id: params = par_fn.get_parameters() years, months, codes, kernels, bandwidths, metrics = params[0],\ params[1],\ params[2],\ params[3],\ params[4],\ params[5]\ months, codes, metrics = [month for month in months.keys()],\ [code for code in codes.keys()],\ [metric for metric in metrics.keys()] kde = fn.set_kde_startup() kde_script, kde_div = components(kde) try: session_status = 'App ready.' if len(request.form) != 0: post_year = str(request.form['kde_form_data_year_select']) post_month = str( request.form['kde_form_data_month_select']) post_violation = str( request.form['kde_form_data_code_select']) post_kernel = str( request.form['kde_form_data_kernel_select']) post_bandwidth = str( request.form['kde_form_data_bandwidth_select']) post_metric = str( request.form['kde_form_data_metric_select']) form_input = [ post_year, post_month, post_violation, post_kernel, float(post_bandwidth), post_metric ] codes_post = params[2] post_violation = codes_post[post_violation] kde = fn.set_kde_interact(post_year, post_month, post_violation, post_kernel, post_bandwidth, post_metric) datasets = ds_fn.set_annotate_interact( post_year, post_month, post_violation) ald_dist, mpd_dist = datasets[0], datasets[1] kde_script, kde_div = components(kde) ald_script, ald_div = components(ald_dist) mpd_script, mpd_div = components(mpd_dist) vio_cnt = datasets[2] return render_template('mkecs-kde_result.html', kde_script=kde_script, kde_div=kde_div, ald_script=ald_script, ald_div=ald_div, mpd_script=mpd_script, mpd_div=mpd_div, months=months, years=years, codes=codes, kernels=kernels, bandwidths=bandwidths, metrics=metrics, form_input=form_input, session_status=session_status, session_id=session['username'], vio_cnt=vio_cnt) except: session_status = 'All fields must be selected.' kde = fn.set_kde_startup() kde_script, kde_div = components(kde) return render_template('mkecs-kde.html', kde_script=kde_script, kde_div=kde_div, months=months, years=years, codes=codes, kernels=kernels, bandwidths=bandwidths, metrics=metrics, session_status=session_status, session_id=session['username']) session_status = 'All fields must be selected.' return render_template('mkecs-kde.html', kde_script=kde_script, kde_div=kde_div, months=months, years=years, codes=codes, kernels=kernels, bandwidths=bandwidths, metrics=metrics, session_status=session_status, session_id=session['username']) except: return redirect(url_for('mkecs_kde_session'))
def get(self): return get_parameters()
print(fName) with h5py.File(fName, 'w') as hf: hf.create_dataset("q", data=q_map) hf.create_dataset("atm_diffraction", data=atm_diffraction) hf.create_dataset("mol_diffraction", data=mol_diffraction) hf.create_dataset("mol_diffraction_raw", data=mol_diffraction_raw) hf.create_dataset("mod_diffraction", data=mol_diffraction / atm_diffraction) hf.create_dataset("detector_distance", data=detector_dist) if __name__ == '__main__': FLAGS = parser.parse_args() if FLAGS.molecule is not None: params = get_parameters(FLAGS.run, FLAGS.molecule) else: params = get_parameters(FLAGS.run) if FLAGS.molecule is not None: params.molecule = FLAGS.molecule FLAGS = setup(parser, output_dir=params.simOutputDir) # Flag handling if FLAGS.calculation_type is not None: params.calculation_type = FLAGS.calculation_type if FLAGS.xyz_file is not None: params.xyz_file = FLAGS.xyz_file if FLAGS.output_fileName_suffix is not None: params.output_fileName_suffix = FLAGS.output_fileName_suffix if FLAGS.basis_folder is not None: params.basis_folder = FLAGS.basis_folder
total += target_var.size(0) correct += (predicted == target_var).sum().item() return 100 * correct / total def save_checkpoint(state, is_best, dirpath, epoch): filename = 'checkpoint.{}.ckpt'.format(epoch) checkpoint_path = os.path.join(dirpath, filename) best_path = os.path.join(dirpath, 'best.ckpt') torch.save(state, checkpoint_path) if is_best: shutil.copyfile(checkpoint_path, best_path) print('Best Model Saved: ');print(best_path) def get_current_consistency_weight(epoch): return args.consistency * ramps.sigmoid_rampup(epoch, args.consistency_rampup) if __name__ == '__main__': args = get_parameters() args.device = torch.device( "cuda:%d" % (args.gpu_id) if torch.cuda.is_available() else "cpu") main(args)
def set_kde_interact(year, month, violation_code, kernel, bandwidth, metric): # Build parameters params = par_fn.get_parameters() codes = {'200':'Arson', '13':'Assault Offenses', '220':'Burglary/Breaking & Entering', '290':'Destruction/Damage/Vandalism of Property', '09':'Homicide Offenses', '23f':'Theft from Motor Vehicle', '120':'Robbery', '11_36':'Sex Offenses', '23':'Larceny/Theft Offenses', '240':'Motor Vehicle Theft'} months = params[1] mo, bw = months[month], str(bandwidth).replace('.', '') # Build data project_path, db_path = path.project_path(), path.db_path() db_dir = os.path.join(project_path + db_path) db = create_engine('sqlite:///' + db_dir + 'mke_wibrs_db.db', echo = False) Session = sessionmaker(bind = db) session = Session() query = session.query(mke_wibrs_db).\ filter(mke_wibrs_db.date >= year + mo[0], mke_wibrs_db.date <= year + mo[1], getattr(mke_wibrs_db, 'code_' + violation_code) == 1, mke_wibrs_db.addr != None, mke_wibrs_db.zip_code != None, mke_wibrs_db.ald_dist != None, mke_wibrs_db.mpd_dist != None, mke_wibrs_db.voting_ward != None, mke_wibrs_db.x_lon != None, mke_wibrs_db.y_lat != None).all() x_lon, y_lat, incident_number, date, formatted_addr, ald_dist, mpd_dist = [], [], [], [], [], [], [] for data in query: x_lon.append(data.x_lon) y_lat.append(data.y_lat) incident_number.append(data.incident_number) date.append('{:%Y-%m-%d %H:%M:%S}'.format(data.date)) formatted_addr.append(data.formatted_addr) ald_dist.append(data.ald_dist) mpd_dist.append(data.mpd_dist) session.close() # Build plot & labels tt_src = ColumnDataSource(data = dict(x = x_lon, y = y_lat, i = incident_number, d = date, a = formatted_addr, ald = ald_dist, m = mpd_dist)) x_range, y_range = [-88.080736, -87.839722], [42.917670, 43.19712] p = figure(width = 606, height = 700, x_range = x_range, y_range = y_range, tools = 'save') plot_title_label = Label(x = -87.89, y = 43.187, text = 'Milwaukee, WI', text_font = 'helvetica', text_font_size = '14px', text_color = '#41444B', text_font_style = 'bold') label_src = ColumnDataSource(data = dict(x = [-88.07, -88.07, -88.07, -88.07, -88.07], y = [42.955, 42.948, 42.941, 42.934, 42.927], txt = [month + ' ' + year, codes[violation_code], 'Kernel: ' + kernel, 'Bandwidth: ' + bandwidth, 'Metric: ' + metric])) plot_labels = LabelSet(x = 'x', y = 'y', text = 'txt', text_font = 'helvetica', text_font_size = '8px', text_color = '#41444B', source = label_src) hs_src = ColumnDataSource(data = dict(x = [-87.93, -87.93, -87.93], y = [43.155, 43.148, 43.141], txt = ['Hotspot identification:', '- Click the Point Draw Tool in the upper right corner', '- Drag and drop icons on identified hotspots'])) hs_labels = LabelSet(x = 'x', y = 'y', text = 'txt', text_font = 'helvetica', text_font_size = '12px', text_color = '#41444B', source = hs_src) p.add_layout(plot_title_label), p.add_layout(plot_labels), p.add_layout(hs_labels) p.xgrid.grid_line_color, p.ygrid.grid_line_color, p.toolbar.logo, = None, None, None p.xaxis.visible, p.yaxis.visible = False, False # Retrieve/create KDE f_yr, f_mo, f_vc = str(year), month.lower()[:3], str(violation_code) f_ke, f_bw, f_me = kernel.lower()[:3], str(bandwidth).replace('.', ''), metric.lower()[:3] plot_path_long = path.plot_path_long() plot_file = os.path.isfile(plot_path_long + '%s%s%s%s%s%s.png' % (f_yr, f_mo, f_vc, f_ke, f_bw, f_me)) if plot_file: plot_path_short = path.plot_path_short() p.image_url(url = [plot_path_short + '%s%s%s%s%s%s.png' % (f_yr, f_mo, f_vc, f_ke, f_bw, f_me)], x = x_range[0], y = y_range[1], w = x_range[1] - x_range[0], h = y_range[1] - y_range[0]) else: plot_path_short = path.plot_path_short() kde_bw = float(bandwidth) kde_fn.exe_kde(x_lon, y_lat, year, month, violation_code, kernel, kde_bw, metric) sleep(3) p.image_url(url = [plot_path_short + '%s%s%s%s%s%s.png' % (f_yr, f_mo, f_vc, f_ke, f_bw, f_me)], x = x_range[0], y = y_range[1], w = x_range[1] - x_range[0], h = y_range[1] - y_range[0]) events = p.square(x = 'x', y = 'y', size = 20, color = 'blue', fill_alpha = 0.0, line_color = None, source = tt_src) tt_hover = HoverTool(tooltips = [('WIBRS', '@i'), ('Address','@a'), ('Date', '@d'), ('Ald/MPD', '@ald, @m')], renderers = [events]) p.add_tools(tt_hover) # Hotspot identification hs_src = cds({'x': [-87.9305, -87.911, -87.8915, -87.872, -87.8525], 'y': [43.17, 43.17, 43.17, 43.17, 43.17]}) hs_render = p.circle_cross(x = 'x', y = 'y', size = 40, color = 'green', fill_alpha = 0.25, line_width = 1, source = hs_src) hs_plot = PointDrawTool(renderers = [hs_render], empty_value = None, add = False) p.add_tools(hs_plot) p.toolbar.active_tap = hs_plot return p
def __init__(self,pc_file_u,pc_file_v, covfile, covfile_sublayer=None, pc_size=-1, params={}, preset=None): """ Initialize PCAFlow object. Parameters ---------- pc_file_u, pc_file_v : string Files containing the principal components in horizontal and vertical direction, respectively. These files should be .npy files, in which each row is a flattened principal component (i.e., the total size of these principal component matrices is NUM_PC x (WIDTH*HEIGHT). cov_file : string File containing the covariance matrix of size NUM_PC x NUM_PC for PCA-Flow. covfile_sublayer : string, optional File containing the covariance matrix for the layers (usually biased towards the first PCs). If PCA-Layers is used and this file is not given, use cov_file. pc_size : tuple, optional Size of principal components. Only required if PCs are not of size 512x256 or 1024x436. params : dict, optional Parameters. See parameters.py for documentation of parameters. preset : string Preset with useful parameter values for different datasets. Can be one of 'pcaflow_sintel' 'pcalayers_sintel' 'pcaflow_kitti' 'pcalayers_kitti' """ np.random.seed(1) self.params = defaults.get_parameters(params,preset) cprint('[PCAFlow] Initializing.', self.params) NC = int(self.params['NC']) self.NC = NC pc_u = np.load(pc_file_u) pc_v = np.load(pc_file_v) cov_matrix = np.load(covfile).astype('float32') if covfile_sublayer is not None: cov_matrix_sublayer = np.load(covfile_sublayer).astype('float32') else: cov_matrix_sublayer = None pc_w = 0 pc_h = 0 if pc_size==-1: # Try to guess principal component dimensions if pc_u.shape[1] == 1024*436: cprint('[PCAFLOW] Using PC dimensionality 1024 x 436', self.params) pc_w = 1024 pc_h = 436 elif pc_v.shape[1] == 512*256: cprint('[PCAFLOW] Using PC dimensionality 512 x 256', self.params) pc_w = 512 pc_h = 256 else: print('[PCAFLOW] *** ERROR *** ') print('[PCAFLOW] Could not guess dimensionality of principal components.') print('[PCAFLOW] Please provide as parameter.') sys.exit(1) self.PC = [] # Smooth principal components. self.pc_u = self.filter_pcs(pc_u,(pc_w,pc_h)).astype('float32') self.pc_v = self.filter_pcs(pc_v,(pc_w,pc_h)).astype('float32') self.cov_matrix = cov_matrix self.pc_w = pc_w self.pc_h = pc_h self.reshape_features=True ############################### # Feature matcher ############################### if self.params['features'].lower() == 'libviso' and libviso_available: self.feature_matcher = FeatureMatcherLibviso(self.params) elif self.params['features'].lower() == 'orb': self.feature_matcher = FeatureMatcherORB(self.params) elif self.params['features'].lower() == 'fast': self.feature_matcher = FeatureMatcherFast(self.params) elif self.params['features'].lower() == 'akaze' or not libviso_available: self.feature_matcher = FeatureMatcherAKAZE(self.params) else: print('[PCAFLOW] *** ERROR ***') print('[PCAFLOW] Unknown feature type {}. Please use "libviso" or "fast".'.format(self.params['features'])) sys.exit(1) if self.params['n_models'] <= 1: ############################## # Solver for PCA-Flow ############################## self.solver = RobustQuadraticSolver(self.pc_u, self.pc_v, self.cov_matrix, pc_size=(pc_w,pc_h), params=self.params) else: ############################## # Solver for PCA-Layers ############################## self.solver = EMSolver(self.pc_u, self.pc_v, self.cov_matrix, pc_size = (pc_w,pc_h), params=self.params, cov_matrix_sublayer=cov_matrix_sublayer) self.images = deque(maxlen=2) cprint('[PCAFLOW] Finished initializing.',self.params)
import parameters import config import market import energy_source import mpc_solver import matplotlib.pyplot as plt import json import numpy as np from time import sleep if True: data = parameters.get_parameters() config = config.Config(**data['config']) energy_sources = [energy_source.EnergySource(**kwargs) for kwargs in data['energy_sources']] for ess in energy_sources: ess.tuning_parameter_fit() markets = [market.Market(**kwargs) for kwargs in data['markets']] algo2 = mpc_solver.MPCSolver(config=config, markets=markets, energy_sources=energy_sources) results = algo2.solve([[1.80, 1.81] for i in range(len(markets))], ['free', 'free', 'free']) with open('t.json', 'w') as f: json.dump(results, f) with open('t.json', 'r') as f: results = json.load(f) for key in results[0].keys(): values = [] for record in results: values.append(record[key])
parameters["database_distribution"], source_id, col_names, col_values, ks_ipd[1], kld_ipd, ttest_ipd[1], whitney_ipd[1], ansari_ipd[1], wasserstein_ipd, energy_ipd, ks_count[1], kld_count, ttest_count[1], whitney_count[1], ansari_count[1], wasserstein_count, energy_count, sourceID, paraID) # print(sql) cursor_distribution.execute(sql) db_distribution.commit() cursor_distribution.close() db_distribution.close() if __name__ == '__main__': print("Start Test") parameters = parameters.get_parameters() db_distribution = pymysql.connect(parameters['hostname'], parameters['username'], parameters['password'], parameters["database_distribution"], charset='utf8mb4') cursor_distribution = db_distribution.cursor() sql = "SELECT `IPDDIS`,`COUNTDIS` FROM `{0}`.`distribution_31` " \ "WHERE `PARAID`='0' AND `SOURCEID`='31' LIMIT 1;".format(parameters["database_distribution"]) cursor_distribution.execute(sql) cursor_result = cursor_distribution.fetchone() referenceIPDValues_excellent = dict_to_dis_list( json_map_to_dict(cursor_result[0])) referenceCountValues_excellent = dict_to_dis_list( json_map_to_dict(cursor_result[1]))
import patterns import initialisation import iteration import correlations import matplotlib as mpl import matplotlib.pyplot as plt if os.environ.get('DISPLAY', '') == '': print('no display found. Using non-interactive Agg backend') mpl.use('Agg') # plt.ion() dt, tSim, N, S, p, num_fact, p_fact, dzeta, a_pf, eps, cm, a, U, T, w, \ tau_1, tau_2, tau_3_A, tau_3_B, g_A, beta, g, t_0, tau, cue_ind, \ random_seed = get_parameters() cue_ind = 0 rd.seed(random_seed) lambdas = np.nan * np.ones((int(N * a), int(N * a))) max_C1 = 2 * int(N * a * a / S) max_C2 = int(2 * N * a * a * (S - 1) / S) n_C1 = min(max_C1 + 1, 25) n_C2 = min(max_C2 + 1, 25) ksi_i_mu, delta__ksi_i_mu__k = patterns.get_uncorrelated() for C2 in np.linspace(0, 20, 21, dtype=int): for C1 in np.linspace(0, 10, 11, dtype=int):
def run(): if sys.platform == 'win32': bar_value = '#' else: bar_value = '█' start = datetime.now() path = './data/database.db' conn = sqlite3.connect(path) c = conn.cursor() c.execute( '''CREATE TABLE IF NOT EXISTS treinamento (image_name TEXT PRIMARY KEY, HOG BLOB, LBP BLOB)''' ) c.execute( '''CREATE TABLE IF NOT EXISTS testes (image_name TEXT PRIMARY KEY, HOG BLOB , LBP BLOB)''' ) hog = p.get_parameters('HOG', True) lbp = p.get_parameters('LBP', True) try: values = {} values['HOG'] = imagelib.getHog('./data/img_test.png', hog['descriptor_param_1'], hog['descriptor_param_2'], hog['descriptor_param_3']) values['LBP'] = imagelib.getLBP('./data/img_test.png', lbp['descriptor_param_1'], lbp['descriptor_param_2']) c.execute('INSERT OR IGNORE INTO treinamento VALUES (?, ?, ?)', ['img_test.png', values['HOG'], values['LBP']]) except Exception: raise dataset = u.get_dataset_list( u.get_classes_list('./data/dataset2/treinamento'), './data/dataset2/treinamento') count = 1 for line in dataset: widgets = [ 'Training Letter: {} | '.format(u.get_letter(line[0], True)), progressbar.Percentage(), ' (', progressbar.Counter(), ' of ', str(len(line)), ') ', progressbar.Bar(bar_value), ' ', progressbar.ETA() ] with progressbar.ProgressBar(widgets=widgets, max_value=len(line)) as bar: for i, name in enumerate(line): values = {} try: values['HOG'] = imagelib.getHog(hog['workpath'] + name, hog['descriptor_param_1'], hog['descriptor_param_2'], hog['descriptor_param_3']) values['LBP'] = imagelib.getLBP(lbp['workpath'] + name, lbp['descriptor_param_1'], lbp['descriptor_param_2']) c.execute( 'INSERT OR IGNORE INTO treinamento VALUES (?, ?, ?)', [name, values['HOG'], values['LBP']]) except Exception: raise count += 1 bar.update(i) conn.commit() dataset = u.get_dataset_list(u.get_classes_list('./data/dataset2/testes'), './data/dataset2/testes') for line in dataset: widgets = [ 'Testing Letter: {} | '.format(u.get_letter(line[0], True)), progressbar.Percentage(), ' (', progressbar.Counter(), ' of ', str(len(line)), ') ', progressbar.Bar(bar_value), ' ', progressbar.ETA() ] with progressbar.ProgressBar(widgets=widgets, max_value=len(line)) as bar: for i, name in enumerate(line): values = {} try: values['HOG'] = imagelib.getHog(hog['testpath'] + name, hog['descriptor_param_1'], hog['descriptor_param_2'], hog['descriptor_param_3']) values['LBP'] = imagelib.getLBP(lbp['testpath'] + name, lbp['descriptor_param_1'], lbp['descriptor_param_2']) c.execute('INSERT OR IGNORE INTO testes VALUES (?, ?, ?)', [name, values['HOG'], values['LBP']]) except Exception: raise count += 1 bar.update(i) conn.commit() c.close() end = datetime.now() print('\nTotal of images described: {} + test image.'.format( str(count - 1).zfill(5))) print('Database in {}\n'.format(path)) print("Start Time: {}".format(start.strftime("%d/%m/%Y %H:%M:%S"))) print("End Time: {}".format(end.strftime("%d/%m/%Y %H:%M:%S"))) print('Total Elapsed Time: {}\n'.format(end - start))
''' SS_FIG_DIR = "../OUTPUT_BASELINE/sigma3.0" COMPARISON_DIR = "../OUTPUT_BASELINE/sigma3.0" ss_init = os.path.join(SS_FIG_DIR, "SS/SS_vars.pkl") variables = pickle.load(open(ss_init, "rb")) for key in variables: globals()[key] = variables[key] # params_given = os.path.join(SS_FIG_DIR, "Saved_moments/params_given.pkl") # variables = pickle.load(open(params_given, "rb")) # for key in variables: # globals()[key] = variables[key] #globals().update(ogusa.parameters.get_parameters_from_file()) globals().update(parameters.get_parameters(True, {}, {}, {})) param_names = [ 'S', 'J', 'T', 'BW', 'lambdas', 'starting_age', 'ending_age', 'beta', 'sigma', 'alpha', 'nu', 'Z', 'delta', 'E', 'ltilde', 'g_y', 'maxiter', 'mindist_SS', 'mindist_TPI', 'b_ellipse', 'k_ellipse', 'upsilon', 'chi_b_guess', 'chi_n_guess', 'etr_params', 'mtrx_params', 'mtry_params', 'tau_payroll', 'tau_bq', 'retire', 'mean_income_data', 'g_n_vector', 'h_wealth', 'p_wealth', 'm_wealth', 'omega', 'g_n_ss', 'omega_SS', 'surv_rate', 'e', 'rho' ] variables = {} for key in param_names: variables[key] = globals()[key] for key in variables: globals()[key] = variables[key]