def valid_generator(batch_size, viz=False): directory = 'dataset/valid/' valid_x_full = np.load(directory + 'valid_x.npy') valid_y_prob_full = np.load(directory + 'valid_y_prob.npy') valid_y_keys_full = np.load(directory + 'valid_y_keys.npy') dataset_size = valid_y_prob_full.shape[0] indices = batch_indices(batch_size=batch_size, dataset_size=dataset_size) print('Validation Dataset Size: {0}'.format(dataset_size)) while True: for index in indices: # load from dataset valid_x = valid_x_full[index[0]:index[1]] valid_y_prob = valid_y_prob_full[index[0]:index[1]] valid_y_keys = valid_y_keys_full[index[0]:index[1]] if viz: visualize(valid_x[-1], valid_y_prob[-1], valid_y_keys[-1]) # normalizing the image and the keypoints valid_x = valid_x / 255.0 valid_y_prob = np.squeeze(valid_y_prob) valid_y_keys = valid_y_keys / 128.0 # creating ensembles of the keypoints valid_y_keys = np.reshape(valid_y_keys, (valid_y_keys.shape[0], 1, 10)) valid_y_keys = np.repeat(valid_y_keys, 10, axis=1) valid_y = [valid_y_prob, valid_y_keys] yield valid_x, valid_y
def visualize(): # load global params load_config() # load bow features X_bof, Y_sift, X_data = joblib.load(os.path.join(feat_dir, 'bow_feats.pkl')) vis.visualize(X_bof, Y_sift, X_data, 'BOW feature visualization', True)
def generate(individual, n_population=2, n_generation=10, logs=True): best = individual best_individuals = [best] global LOGS LOGS = logs for i in range(n_generation): population = generate_population(best, n_population) best, score, categories = select_best(population) best_individuals.append(best) visualize(best, name=str(i + 1)) if logs: print(f'\nGeneration {i + 1}. Best score: {score}.') print(f'Color score: {categories["color"]}.') print(f'Points distribution score: {categories["points"]}.') print(f'Polylines length score: {categories["polylines"]}.') print(f'Polylines smoothing score: {categories["smoothness"]}.') print('\n') write(best, avg_scores, globals()) return best, best_individuals
def reductionKNNAlgorithm(dataset, k=1, r=1, w='eq', v='maj', show=False, reduction_alg=None): """ Execute a 10-fold kNN. :param dataset: either "adult" or "satimage" :param k: the number of nearest neighbours to take into account when voting :param show: boolean, whether to show a 2-plot of the results :return: None """ df=pd.DataFrame(columns=['dataset','k','r','w','v','reduction','acc','eff','storage']) read = read_vowel_fold if dataset == 'vowel' else read_satimage_fold for i in range(10): x_train, y_train, x_test, y_test = read(i) if reduction_alg: n0=x_train.shape[0] x_train, y_train = reduction_alg(x_train, y_train, k, r, w, v) storage=x_train.shape[0]/n0 else: storage=1 # Call here the KNN indexes, y_pred, eff, acc = kNNAlgorithm(x_train,y_train, x_test,y_test, k, r, w, v) print(f"{dataset} fold {i}, k={k}, r={r}, w={w}, v={v}: Accuracy {acc}, Time {eff}") df=df.append(pd.DataFrame([[dataset,k,r,w,v,reduction_alg,acc,eff,storage]],columns=['dataset','k','r','w','v','reduction','acc','eff','storage'])) if show: visualize(x_test, y_test, y_pred) return df
def _train_epoch(self, epoch): """ Training logic for an epoch :param epoch: Integer, current training epoch. :return: A log that contains average loss and metric in this epoch. """ self.model.train() self.train_metrics.reset() for batch_idx, batch in tqdm(enumerate(self.train_loader)): original_labels, masked_labels, masked_token_ids, masked_indices = MLM_load_batch( batch, self.config) actual_batch_size = original_labels.shape[0] self.optimizer.zero_grad() # https://discuss.pytorch.org/t/selecting-element-on-dimension-from-list-of-indexes/36319/2 yhat = self.model( masked_labels) # (actual_batch_size, seq_length, vocab_size) yhat = yhat[torch.arange(actual_batch_size), masked_indices] # (actual_batch_size, vocab_size) loss = self.criterion(yhat, masked_token_ids) self.writer.add_scalar("Loss/train", loss, self.train_iter_global) acc = metrics.accuracy(yhat, masked_token_ids) self.writer.add_scalar("Accuracy/train", acc * 1.0 / actual_batch_size, self.train_iter_global) self.train_iter_global += 1 self.train_metrics.update("loss", loss.item()) for met in self.metric_ftns: self.train_metrics.update(met.__name__, met(yhat, masked_token_ids)) if batch_idx % self.log_step == 0: self.logger.info('Train Epoch: {}-{} Loss: {:.6f}'.format( epoch, self._progress(batch_idx, self.train_loader), loss.item())) loss.backward() self.optimizer.step() log = self.train_metrics.result() if self.config["do_validation"]: val_log = self._val_epoch(epoch) log.update(**{'val_' + k: v for k, v in val_log.items()}) self.writer.flush() with torch.no_grad(): # visualize embedding for a sample word by BERT as of this epoch and save as image visualize(self.model.bert, epoch=epoch, location=os.path.dirname(CUR_DIR) + "/image") if self.lr_scheduler is not None: self.lr_scheduler.step() return log
def main(args): if not os.path.exists(args.save_path): os.mkdir(args.save_path) net = UNet(n_channels=3, n_classes=1) checkpoint = flow.load(args.pretrained_path) net.load_state_dict(checkpoint) net.to("cuda") x_test_dir, y_test_dir = get_datadir_path(args, split="test") test_dataset = Dataset( x_test_dir, y_test_dir, augmentation=get_test_augmentation(), ) print("Begin Testing...") for i, (image, mask) in enumerate(tqdm(test_dataset)): show_image = image with flow.no_grad(): image = image / 255.0 image = image.astype(np.float32) image = flow.tensor(image, dtype=flow.float32) image = image.permute(2, 0, 1) image = image.to("cuda") pred = net(image.unsqueeze(0).to("cuda")) pred = pred.numpy() pred = pred > 0.5 save_picture_name = os.path.join(args.save_path, "test_image_" + str(i)) visualize( save_picture_name, image=show_image, GT=mask[0, :, :], Pred=pred[0, 0, :, :] )
def upload(self, fileToUpload, n_voices, keydiff_threshold, submit): name = fileToUpload.filename + '-' + str(n_voices) + '-' + str( keydiff_threshold) dir_prefix = f'src/website/html5up-dimension/products/' + name with open("input.wav", "wb") as fout: fout.write(fileToUpload.file.read()) print('processing audio') if not os.path.exists(dir_prefix + "-output.mid"): process_audio.wav2midi( "input.wav", dir_prefix + "-output.mid", { 'n_peaks': int(n_voices), 'keydiff_threshold': int(keydiff_threshold) }) print('generating pdf') if not os.path.exists(dir_prefix + "-output.pdf"): musescore_call.generate_pdf(dir_prefix + "-output.mid", dir_prefix + "-output.pdf") print('generating mp3') if not os.path.exists(dir_prefix + "-output.mp3"): musescore_call.generate_mp3(dir_prefix + "-output.mid", dir_prefix + "-output.mp3") print('generating video') if not os.path.exists(dir_prefix + "-output.mp4"): visualize.visualize(dir_prefix + "-output.mid", dir_prefix + "-output.mp4") with open("src/website/html5up-dimension/pianotalks.html") as f: return f.read().replace("OUTPUT_FILENAME", name)
def selection(): imgsrc = 'static/images/image.png' first = request.form['first'] second = request.form['second'] third = request.form['third'] visualize.visualize(first,second,third)[0].savefig(imgsrc) return render_template('draw.html', imgsrc = imgsrc)
def run(self): max_gen = 0 while max_gen < 10: for i in self.current_generation: i.play() self.previous_generation = copy.copy(self.current_generation) self.current_generation = [] self.previous_generation.sort(key=lambda x: x.fitness, reverse=True) self.previous_generation = self.previous_generation[:settings. num_par_snakes] if settings.visualize: visualize.visualize(self.previous_generation[0]) print("Gen:" + str(self.num_gen)) avg = 0 for i in self.previous_generation: avg += i.fitness / (settings.num_par_snakes + settings.num_snakes_gen) print("Average Fitness: " + str(avg)) print("Best Fitness: " + str(self.previous_generation[0].fitness)) most_fit_apple = len( self.previous_generation[0].eaten_apple_locations) print("Most Fit Apples Eaten: " + str(most_fit_apple)) if most_fit_apple >= 98: max_gen += 1 parents = self.selection() for i in range(0, len(parents) - 1, 2): self.crossover(parents[i], parents[i + 1]) self.mutate() for i in self.previous_generation: self.current_generation.append( s.snake(i.network.weights, i.network.bias)) self.num_gen += 1 print("it worked?")
def test(seg): visualize.visualize(seg, "Before normalization to 40dB") print("Normalizing to 40dB SPL...") print(" SPL before:", seg.spl, "dB") res = seg.normalize_spl_by_average(db=40) print(" SPL after:", res.spl, "dB") visualize.visualize(res, "After normalization to 40dB") return res
def visualize_epoch(job): epoch_id = job[0] machine_steering = job[1] visualize.visualize(epoch_id, machine_steering, params.out_dir, verbose=True, frame_count_limit=None)
def test(seg): print("Removing silence...") seg = seg.filter_silence() outname_silence = "results/nosilence.wav" seg.export(outname_silence, format="wav") visualize.visualize(seg[:min(visualize.VIS_MS, len(seg))], title="After Silence Removal") print("After removal:", outname_silence) return seg
def daily_returns(df, graph_name, returns_col): # Daily percentage returns date_col = 'Date' daily_returns_col = 'Daily Returns' df[daily_returns_col] = df[returns_col].pct_change(1) filename = '%s_daily_returns' %graph_name.lower() title = '%s Daily Returns' %graph_name visualize.visualize(plot_direc, df, date_col, [daily_returns_col], filename, title=title, ylabel=daily_returns_col, xlabel=date_col) return df
def train_generator(batch_size, is_augment=True, viz=False): if is_augment: batch_size = int(batch_size / 2) # load dataset directory = 'dataset/train/' train_x_full = np.load(directory + 'train_x.npy') train_y_prob_full = np.load(directory + 'train_y_prob.npy') train_y_keys_full = np.load(directory + 'train_y_keys.npy') dataset_size = train_y_prob_full.shape[0] indices = batch_indices(batch_size=batch_size, dataset_size=dataset_size) print('Training Dataset Size: {0}'.format(dataset_size)) while True: for index in indices: # load from dataset train_x = train_x_full[index[0]:index[1]] train_y_prob = train_y_prob_full[index[0]:index[1]] train_y_keys = train_y_keys_full[index[0]:index[1]] if viz: visualize(train_x[-1], train_y_prob[-1], train_y_keys[-1]) # augment dataset and append to the batch train_x_aug, train_y_keys_aug = augment(train_x, train_y_prob, train_y_keys) train_x = np.append(train_x, train_x_aug, axis=0) train_y_prob = np.append(train_y_prob, train_y_prob, axis=0) train_y_keys = np.append(train_y_keys, train_y_keys_aug, axis=0) if viz: visualize(train_x[-1], train_y_prob[-1], train_y_keys[-1]) # normalizing the image and the keypoints train_x = train_x / 255.0 train_y_prob = np.squeeze(train_y_prob) train_y_keys = train_y_keys / 128.0 # creating ensembles of the keypoints train_y_keys = np.reshape(train_y_keys, (train_y_keys.shape[0], 1, 10)) train_y_keys = np.repeat(train_y_keys, 10, axis=1) # random shuffling over the batch seed = random.randint(0, 1000) np.random.seed(seed) np.random.shuffle(train_x) np.random.seed(seed) np.random.shuffle(train_y_prob) np.random.seed(seed) np.random.shuffle(train_y_keys) train_y = [train_y_prob, train_y_keys] yield train_x, train_y
def process_data(method, features): classifier = create_classifier(method) visualize(classifier, training_set, data, "diabetes", features) verification_score = calculate_score(classifier, validation_set, features, "diabetes") training_score = calculate_score(classifier, training_set, features, "diabetes") plt.savefig('static/img/visualize.png') plt.clf() return verification_score, training_score
def cumulative_daily_returns(df, graph_name, returns_col): # Cumulative daily returns date_col = 'Date' daily_returns_col = 'Cumulative Daily Returns' initial_price = df[returns_col][0] df[daily_returns_col] = pd.rolling_apply(df[returns_col], 1, lambda x: x / initial_price) filename = '%s_cumulative_daily_returns' %graph_name.lower() title = '%s Cumulative Daily Returns' %graph_name visualize.visualize(plot_direc, df, date_col, [daily_returns_col], filename, title=title, ylabel=daily_returns_col, xlabel=date_col) return df
def simulate(params): # Get the selected distribution dist = dict(g.get_distributions())[params.distribution] wire_drop_counts = range(params.wire_range[0], params.wire_range[1], params.step) #lg = g.get_exponential_decay_length_generator(math.e, 2, max_length, min_length) lengths = dist( *(params.length_range + params.distribution_parameters) ) positions = g.get_uniform_position_generator(params.grid_width) angles = g.get_uniform_angle_generator() # Setup graph animator if requested. after_add = None if params.anitmate_graph_creation: animation = v.GraphCreationAnimator() after_add = lambda *args: animation.add_graph(*args) wire_factory = f.WireFactory(lengths, positions, angles) factory = f.NocFactory(wire_factory, after_add=after_add) # Generate the NoC trials = [] for trial in range(0, params.trials): results = [] for drop_count in wire_drop_counts: noc = factory.create(drop_count) noc.create_graph() # Save the image of the NoC if requested if params.graph_save_path: s = "{0}/{1}_trial{2}_count{3}.png" path = s.format( params.graph_save_path, params.distribution, trial, drop_count) v.save_noc_image(noc, path) result = a.get_stats(noc, drop_count, trial) results.append(result) trials.append(results) if params.anitmate_graph_creation: animation.create_animation("graph.mp4") # Save results if requested if params.output_file != None: for trial in trials: io.export_to_csv(trial, params.output_file, mode="a+") # Display the results if params.visualize: v.visualize(trials)
def run_simulation(path): import main sim = Simulation.from_path(path) with unittest.mock.patch("main.pyautogui", sim): board = main.solve_live_game() visualize.visualize(board) sim.screenshot().show() assert board.is_solved()
def main(): config = configparser.ConfigParser() if len(config.read("config")) == 0: print("Configuration file was not found!\n") return section_name = "PARAMS" params = config[section_name] train_file = str(params["train_file"]) test_ratio = float(params["test_ratio"]) k = int(params["k"]) mesh_step = float(params["mesh_step"]) do_manhattan = config.getboolean(section_name, "use_manhattan") do_czebyszew = config.getboolean(section_name, "use_Czebyszew") do_validation = config.getboolean(section_name, "do_validation") do_visulization = config.getboolean(section_name, "do_visulization") print("train file {},\ntest ratio {},\nk parameter {}".format( train_file, test_ratio, k)) train_data, test_data = load_data(train_file, test_ratio) num_of_classes = max(test_data, key=operator.itemgetter(-1))[-1] start = time.time() print("Length of test data: {}, train data: {}".format( len(test_data), len(train_data))) k_vals = [k] if do_czebyszew: final_metric = czebyszew.Czebyszew.calc_dist elif do_manhattan: final_metric = manhattan.Manhattan.calc_dist else: final_metric = euclidean.Euclidean.calc_dist result = None for i in k_vals: start = time.time() result = test(train_data, test_data, i, final_metric) end = time.time() - start print(';' + str(end)) best_result = result end = time.time() - start print('end time' + str(end)) if do_visulization: # this will draw classes and plot testing points V.visualize(best_result[0], train_data, test_data, k, num_of_classes, mesh_step, do_manhattan)
def main(): try: args = get_args() data = preprocess(args) if args.visualize_data: visualize(data) sys.exit(1) train_set, test_set = split(data) num_examples = train_set.shape[0] num_features = train_set.shape[1] - 1 if args.mini_batch: batch_size = 32 # or 64 epochs = 1500 else: batch_size = num_examples epochs = 30000 nn = NeuralNetwork(num_features, batch_size, epochs) if args.train: nn.train(data, train_set, test_set, num_examples, args.quiet) if args.evaluation: y_pred = probability_to_class( nn.output.T) get_validation_metrics(y_pred[:, 0], nn.y.T[:, 0]) # mini-batch learning is noisy, so we don't plot it if not args.mini_batch: plot_learning(nn.train_losses, nn.test_losses) # save network params if args.save_model: W1, W2, W3, W4 = nn.weights1.tolist(), nn.weights2.tolist(), nn.weights3.tolist(), nn.weights4.tolist() B1, B2, B3, B4 = nn.bias1.tolist(), nn.bias2.tolist(), nn.bias3.tolist(), nn.bias4.tolist() model = dict(weights1=W1, weights2=W2, weights3=W3, weights4=W4, bias1=B1, bias2=B2, bias3=B3, bias4=B4) with open("model.json", "w") as f: json.dump(model, f, separators=(',', ':'), indent=4) if args.predict and (args.predict == "model.json"): try: with open(args.predict) as file: model = json.load(file) except: error_exit("please provide a valid model") nn.load_model(model) nn.predict(test_set, epochs) except: pass
def long_short_ratio(df, graph_name, long_col, short_col): date_col = 'Date' long_short_ratio_col = 'Long Short Ratio' df[long_short_ratio_col] = df[long_col] / df[short_col] filename = '%s_long_short_ratio' %graph_name.lower() title = '%s Long Short Ratio' %graph_name visualize.visualize(plot_direc, df, date_col, [long_short_ratio_col], filename, title=title, ylabel=long_short_ratio_col, xlabel=date_col) return df df['Long Short Ratio'] = df[long_col] / df[short_col] return df
def visualize_participant_production(dataset, dir_path): for participant in dataset: if participant['test_type'] == 'production' and participant[ 'status'] == 'finished': partition = np.zeros(64, dtype=int) for answer, response in zip(participant['test_sequence'], participant['test_responses']): partition[answer] = response partition = partition.reshape((8, 8)) figure_path = dir_path + '%s/%s.pdf' % (participant['condition'], participant['user_id']) visualize.visualize(partition, figure_path)
def main(): print("Getting paths and labels for all train and test data") train_image_paths, test_image_paths, train_labels, test_labels = \ get_image_paths(DATA_PATH, CATEGORIES, NUM_TRAIN_PER_CAT) if FEATURE == 'tiny_image': train_image_feats = get_tiny_images(train_image_paths) test_image_feats = get_tiny_images(test_image_paths) elif FEATURE == 'bag_of_sift': if os.path.isfile('vocab.pkl') is False: print('No existing visual word vocabulary found. Computing one from training images\n') vocab_size = 400 vocab = build_vocabulary(train_image_paths, vocab_size) with open('vocab.pkl', 'wb') as handle: pickle.dump(vocab, handle, protocol=pickle.HIGHEST_PROTOCOL) if os.path.isfile('train_image_feats.pkl') is False: train_image_feats = get_bags_of_sifts(train_image_paths); with open('train_image_feats.pkl', 'wb') as handle: pickle.dump(train_image_feats, handle, protocol=pickle.HIGHEST_PROTOCOL) else: with open('train_image_feats.pkl', 'rb') as handle: train_image_feats = pickle.load(handle) if os.path.isfile('test_image_feats.pkl') is False: test_image_feats = get_bags_of_sifts(test_image_paths); with open('test_image_feats.pkl', 'wb') as handle: pickle.dump(test_image_feats, handle, protocol=pickle.HIGHEST_PROTOCOL) else: with open('test_image_feats.pkl', 'rb') as handle: test_image_feats = pickle.load(handle) else: raise NameError('Unknown feature type') if CLASSIFIER == 'nearest_neighbor': predicted_categories = nearest_neighbor_classify(train_image_feats, train_labels, test_image_feats) elif CLASSIFIER == 'support_vector_machine': predicted_categories = svm_classify(train_image_feats, train_labels, test_image_feats) else: raise NameError('Unknown classifier type') accuracy = float(len([x for x in zip(test_labels,predicted_categories) if x[0]== x[1]]))/float(len(test_labels)) print("Accuracy = ", accuracy) test_labels_ids = [CATE2ID[x] for x in test_labels] predicted_categories_ids = [CATE2ID[x] for x in predicted_categories] train_labels_ids = [CATE2ID[x] for x in train_labels] build_confusion_mtx(test_labels_ids, predicted_categories_ids, ABBR_CATEGORIES) visualize(CATEGORIES, test_image_paths, test_labels_ids, predicted_categories_ids, train_image_paths, train_labels_ids)
def main(): args = parse_arg() configs = [make_config(k, embd=args.embd, tsne=args.tsne, small=args.small, shuffled_class=args.shuffled_class) for k in args.datasets] if args.make_graph: visualize(configs[0], k_nearest=args.k_nearest, M_sample=args.M) valss = [test_job(config=k, k_nearest=args.k_nearest, M_sample=args.M) for k in tqdm(configs)] plot_with_err(*valss)
def correlations_vs_shifted_returns(df, feature_col, returns_col): x = range(1, 200) pearsons = [] distances = [] for n in x: temp = shift_returns(df, returns_col, n) temp = temp[:-n] pearsons.append(pearson_correlation(temp, feature_col, returns_col)) distances.append(distance_correlation(temp, feature_col, returns_col)) filename = 'correlations_vs_%s' %'_'.join((feature_col.lower()).split(' ')) title = 'Correlations vs Days Shifted for %s' %(feature_col) new_df = pd.DataFrame({'Days Shifted': x, 'Pearson Correlations': pearsons, 'Distance Correlations': distances}) visualize.visualize(plot_direc, new_df, 'Days Shifted', ['Pearson Correlations', 'Distance Correlations'], filename, title=title, ylabel='Correlation', xlabel='Days Shifted')
def test(seg): print("Removing silence...") result = seg.filter_silence() outname_silence = "results/nosilence.wav" result.export(outname_silence, format="wav") visualize.visualize(result[:min(visualize.VIS_MS, len(result))], title="After Silence Removal") print("After removal:", outname_silence) # Now try again, but with massive threshold for silence removal # This will strip almost every sample in the file, leaving a practically empty # WAV file, which Pydub chokes on. _ = seg.filter_silence(threshold_percentage=99.9) return result
def visualize_participant_comprehension(dataset, dir_path): for participant in dataset: if participant['test_type'] == 'comprehension' and participant[ 'status'] == 'finished': partition = np.zeros((4, 8, 8), dtype=float) for answer, response in zip(participant['test_sequence'], participant['test_responses']): row = response // 8 col = response % 8 partition[answer, row, col] += 1 figure_path = dir_path + '%s/%s.pdf' % (participant['condition'], participant['user_id']) for cat in range(4): partition[cat] = partition[cat] / partition[cat].max() visualize.visualize(partition, figure_path)
def main(data_path, rootNode): MAX_LEVEL = 10 folder = create_dir(data_path, rootNode) print(folder) # used_word = ["information_retrieval", "information", "model", "method"] # used_word = ["information_retrieval"] used_word = [] for level in range(MAX_LEVEL): print('\n================================== Running level ', level, ' ==================================\n') iteration(folder, rootNode, used_word) print(folder) visualize(folder)
def run(output_dir, input_dir): click.echo("output directory: {}".format(output_dir)) click.echo("input directory: {}".format(input_dir)) # find files that are Iguana result files files = list( filter( lambda path: path.is_file() and path.suffix == ".nt" and not path. name.startswith("cleaned_"), [ Path(os.path.join(input_dir, path)) for path in os.listdir(input_dir) ])) click.echo("\nFiles for conversion: \n{}".format("\n".join( [file.name for file in files]))) output_files = list() click.echo("\nConverted files:") for file in files: for output_file in result2rdf.convert_result_file( file.name, input_dir, output_dir): click.echo("{}.csv".format(Path(output_file).name)) output_files.append(output_file) click.echo("\nConcatenating all output files ... ") concat_output_file_path = os.path.join(output_dir, "all_results") with open(concat_output_file_path + ".csv", 'w') as output_file: csv_writer = csv.DictWriter(output_file, fieldnames=result2rdf.fieldnames, quoting=csv.QUOTE_NONNUMERIC) csv_writer.writeheader() for input_file_path in output_files: with open(input_file_path + ".csv", 'r') as input_file: csv_reader = csv.DictReader(input_file, ) for row in csv_reader: csv_writer.writerow(row) with open(concat_output_file_path + ".json", 'w') as output_file: entries = list() for input_file_path in output_files: with open(input_file_path + ".json", 'r') as input_file: json_obj = json.load(input_file) entries.append(json_obj) output_file.write( json.dumps({"benchmarks": entries}, sort_keys=True, indent=4)) click.echo("Done\n") click.echo("Generating plots ...") visualize(os.path.join(output_dir, "all_results.json"), os.path.join(output_dir, "all_results.csv")) click.echo("Done")
def choose(): """ Renders a form for the user to pick a visualization style, also handles the request and displays the visualization """ if request.method == 'POST': global visChoice visChoice = request.form['visChoice'] print visChoice # try: global ical global dateRange vis.visualize(ical, visChoice, dateRange=dateRange) # except: # print 'didnt output vis' + str(datetime.datetime.now()) # theFile= jsonify("vis.json") return redirect(url_for('visualize')) return render_template('choose.html')
def plot_spectro(loa_omegas, loa_omega_names, version_string, expl_string, exclude_eu=False): #Plot spectroscopic data for [O/H], [Fe/H], [Eu/H] vis_obj = visualize(loa_omegas, loa_omega_names, num_yaxes=3, loa_abu=[], loa_spectro_abu=["[O/H]", "[Fe/H]", "[Eu/H]"]) vis_obj.add_time_relabu("[O/H]", index_yaxis=0) vis_obj.add_time_relabu("[Fe/H]", index_yaxis=1) vis_obj.add_time_relabu("[Eu/H]", index_yaxis=2) vis_obj.finalize(show=False, save="data/star_parameters_%s_n%d" % (version_string, num_steps)) #save data from spectroscopic plots for i, spectro_string in enumerate(["[O/H]", "[Fe/H]", "[Eu/H]"]): save_obj = save_data(loa_omegas) save_obj.make_filenames("data/star_parameters_%s_%d_n%d" % (version_string, i, num_steps)) save_obj.make_explanatory_file( "spectroscopic arrays of %s \n%s" % (spectro_string, expl_string), ["time"] + loa_omega_names, "Applying star-parameters to 'Omega' with mass-parameters determined" ) save_obj.make_numpy_file(["time", spectro_string])
def main(): # number of measurements N = 1000 # means of random sample m = np.zeros(N) # sigmas s = np.ones(N) # generate random data R = generate_data(m, s, 3) print(R) visualize(R)
def update_gui_tree(): from visualize import visualize if 'gui_tree' in doc: doc['gui_tree'].deparent() # Should instead find the right index doc['overlay'].append( Node("group", children=list(visualize(doc[doc['selection.root']]))))
def process_data(): """Run all processing (analysis, graph generation) on macrogenetic data""" raw_graph = graph.import_graph() links_analysis = inscription_order.analyse_graph() links_graphs = visualize.visualize() _write_index_html(links_graphs + links_analysis)
def parse_command_line(): parser = argparse.ArgumentParser( description="""Train, validate, and test a face detection classifier that will determine if two faces are the same or different.""") parser.add_argument("-p", "--prepare-data", help="Prepare training and validation data.", action="store_true") parser.add_argument("-t", "--train", help="""Train classifier. Use --graph to generate quality graphs""", action="store_true") parser.add_argument("-g", "--graph", help="Generate training graphs.", action="store_true") parser.add_argument("--weights", help="""The trained model weights to use; if not provided defaults to the network that was just trained""", type=str, default=None) parser.add_argument("--note", help="Adds extra note onto generated quality graph.", type=str) parser.add_argument("-s", "--is_same", help="""Determines if the two images provided are the same or different. Provide relative paths to both images.""", nargs=2, type=str) parser.add_argument("--visualize", help="""Writes out various visualizations of the facial images.""", action="store_true") args = vars(parser.parse_args()) if os.environ.get("CAFFE_HOME") == None: print "You must set CAFFE_HOME to point to where Caffe is installed. Example:" print "export CAFFE_HOME=/usr/local/caffe" exit(1) # Ensure the random number generator always starts from the same place for consistent tests. random.seed(0) webface = WebFace() if args["prepare_data"] == True: webface.load_data() webface.pair_data() if args["visualize"] == True: # TODO: Adapt this to WebFace, not just LFW. visualize() if args["train"] == True: train(args["graph"], data=webface, weight_file=args["weights"], note=args["note"]) if args["is_same"] != None: # TODO: Fill this out once we have a threshold and neural network trained. images = args["is_same"] predict(images[0], images[1])
def main(): # creates a list of student and course objects in preparation file lists = preparation.main() student_list = lists[0] course_list = lists[1] session_list = lists[2] room_list = lists[3] score_max = 0 # searches 5 times for a local maximum and visualizes the best score for i in range(5): print i # executes simulated annealing local_max_schedule = simulated_annealing(student_list, course_list, session_list, room_list) schedule_room_list = local_max_schedule[0] schedule_student_list = local_max_schedule[1] score_list = score_function.main(schedule_room_list, schedule_student_list, course_list) score_local_max = score_list[0] # when new hillclimber score is better if score_local_max > score_max: max_schedule = local_max_schedule score_max = score_local_max data = max_schedule[2] # writes data of schedule to csv with open("data_simulated_annealing.csv", "wb") as f: writer = csv.writer(f) writer.writerow(["Iteration", "Score", "malus_conflict", "malus_capacity", "malus_spread", "bonus_spread"]) for row in data: writer.writerow(row) # visualizes room schedules schedule_room_list = max_schedule[0] schedule_student_list = max_schedule[1] for schedule_room in schedule_room_list: visualize.visualize(schedule_room) # visualize students schedules for schedule_student in schedule_student_list[151:152]: visualize.visualize(schedule_student) for schedule_student in schedule_student_list[155:156]: visualize.visualize(schedule_student)
# start training if load: saver.restore(sess, './model') for step in range(50000): batch_x1, batch_y1 = mnist.train.next_batch(128) batch_x2, batch_y2 = mnist.train.next_batch(128) batch_y = (batch_y1 == batch_y2).astype('float') _, loss_v = sess.run([train_step, siamese.loss], feed_dict={ siamese.x1: batch_x1, siamese.x2: batch_x2, siamese.y_: batch_y}) if np.isnan(loss_v): print('Model diverged with loss = NaN') quit() if step % 10 == 0: print ('step %d: loss %.3f' % (step, loss_v)) if step % 1000 == 0 and step > 0: saver.save(sess, './model') embed = siamese.o1.eval({siamese.x1: mnist.test.images}) embed.tofile('embed.txt') # visualize result x_test = mnist.test.images.reshape([-1, 28, 28]) y_test = mnist.test.labels visualize.visualize(embed, x_test, y_test)
auth = lg_authority.AuthRoot() auth__doc = "The object that serves authentication pages" @cherrypy.expose def index(self): output = "" output += getIndexContent() output = getPage(output, '') return output if __name__ == '__main__': #cherrypy.config.update({'server.socket_port':index_port}) cherrypy.config.update(cherry_settings) index = index() index.upload = upload.upload() index.manage = manage.manage() index.modify = modify.modify() index.download = download.download() index.learn = learn.learn() index.support = support.support() index.visualize = visualize.visualize() #index.dashboard = dashboard.dashboard() cherrypy.quickstart(index)
def visualize(self, filename="graph.png", include_args=True, transitive=False): from visualize import visualize visualize(self.graph, filename, include_args, transitive)
def main(source_data='.jp2', time_range=TimeRange('2011/10/01 09:45:00', '2011/10/01 10:15:59'), algorithm='hough', feed_directory='~/Data/eitwave/jp2/20111001_jp2/', use_pickle=None,diff_type='running',data_savedir=None): ''' This is the main executable for the Automated EUV Wave Analysis and Reduction (AWARE) code. The procedure is as follows: - Query the HEK to find whether any flares were observed in SDO/AIA 211A during the input time range - If yes, then read in (or download) a sequence of solar images corresponding to the time range - Transform these images from Helioprojective Coordinates to Heliographic Coordinates, with the origin centered at the flare origin - Create a sequence of difference maps from these transformed maps - Use a threshold method to create a binary map from the the difference maps. - Apply the Hough Transform to the binary map to search for strong lines in the image - Use the results of the Hough Transform to detect whether an EUV wave is present - Fit an appropriate function (e.g. Gaussian) to the detected wavefront as a function of longitude - Record fit results and return data products Parameters ---------- source_data : string description of the type of data being input. Allowed is '.jp2', 'fits', or 'test'. will look for helioviewer JP2 files, FITS files, or load the test data respectively time_range : a TimeRange object time range within which to search for EUV waves feed_directory : string A directory containing data files to be analysed. If set, AWARE will assume data files are already download and will search in this directory instead. Assumes that all files in the directory with the appropriate extension (e.g. .jp2, .fits) are relevant to the flare detection. use_pickle : string BUGGED - currently not supported, always set to None diff_type : string The type of image differencing to use. Allowed values are 'running' or 'base'. Default is 'running' Will perform either running differencing or base differencing on the image sequence. data_savedir : string directory in which to save downloaded jp2 files from Helioviewer. If None, then AWARE will construct a directory based on the start time of the query. Returns ------- Outputs a pickle file containing the following data products (in order): 1) a list of maps modelling the detected wavefront, transformed back to original HPC coordinates ''' if feed_directory != None: feed_directory = os.path.expanduser(feed_directory) #Check which type of data is being analysed, and establish the directory to store downloaded files, #if appropriate if source_data == 'test': maps = test_wave2d() elif source_data == '.jp2' and data_savedir == None: data_savedir = '~/aware_data/jp2/' + time_range.start().strftime('%Y%m%d_%H%M') elif source_data == '.fits' and data_savedir == None: data_savedir = '~/aware_data/fits/' + time_range.start().strftime('%Y%m%d_%H%M') if not os.path.exists(os.path.expanduser(data_savedir)): os.makedirs(os.path.expanduser(data_savedir)) # Query the HEK to see whether there were any flares during the time range specified # Concentrate on the AIA 211A channel as it has clearest observations of global waves client = hek.HEKClient() hek_result = client.query(hek.attrs.Time(time_range.t1, time_range.t2), hek.attrs.EventType('FL'),hek.attrs.OBS.ChannelID == '211') if hek_result is None: # if no flares found, no analysis possible. Return print 'No flares found in HEK database during specified time range.' print 'No analysis possible. Returning.' return None # Otherwise, we have found at least one flare print('Number of flares found = ' + str(len(hek_result))) #assume the first result of the HEK query has the correct information for flare in hek_result[0:1]: if feed_directory is None: print('Acquiring data for flare') filelist = aware_utils.acquire_data(data_savedir, source_data, flare) else: # Assumes that the necessary files are already present filelist = aware_utils.listdir_fullpath(feed_directory, filetype = source_data) #filter to only grab the data files with the source_data extn in the directory #this looks like duplication of listdir_fullpath files_tmp = [] for f in filelist: if f.endswith(source_data): files_tmp.append(f) files = files_tmp # reduce the number of files to those that happen after the flare has # started files = [] for f in files_tmp: fhv = f.split(os.sep)[-1] if aware_utils.hv_filename2datetime(fhv) > \ parse_time(flare['event_starttime']): files.append(f) print('Number of files :' + str(len(files))) if len(files) == 0: print('No files found. Returning.') return None # Define the transform parameters params = aware_utils.params(flare) # read in files and accumulate them if use_pickle != None: # load in a pickle file of the data pfile = open(feed_directory + use_pickle, 'rb') a = pickle.load(pfile) maps = a[0] new_maps = a[1] diffs = a[2] pfile.close() else: maps = aware_utils.accumulate(files[6:30], accum=1, nsuper=4, verbose=True) #temporary fix for exposure control and S/N changes long_maps = [] for m in maps: if m.exposure_time > 2.0: long_maps.append(m) maps=long_maps # Unravel the maps new_maps = aware_utils.map_unravel(maps, params, verbose=True) #return new_maps #sometimes unravelling maps leads to slight variations in the unraveled #image dimensions. check dimensions of maps and resample to dimensions #of first image in sequence if need be. #new_maps[0].peek() new_maps = aware_utils.check_dims(new_maps) # calculate the differences if diff_type == 'base': diffs=aware_utils.map_basediff(new_maps) else: diffs = aware_utils.map_diff(new_maps) #generate persistence maps - currently bugged, so skip this step #persistence_maps = eitwaveutils.map_persistence(diffs) persistence_maps=[] #determine the threshold to apply to the difference maps. #diffs > diff_thresh will be 1, otherwise 0. threshold_maps = aware_utils.map_threshold(new_maps, factor=0.2) #return threshold_maps # transform difference maps into binary maps binary_maps = aware_utils.map_binary(diffs, threshold_maps) if algorithm == 'hough': # detection based on the hough transform detection = aware_utils.hough_detect(binary_maps, vote_thresh=10) elif algorithm == 'prob_hough': # detection based on the probabilistic hough transform. Takes the # keywords of the probabilistic hough transform - see the documentation # of skimage.transform.probabilistic_hough (scikit-image.org) detection = aware_utils.prob_hough_detect(binary_maps, threshold=10) # Remove areas that are too small or that don't have enough detections detection = aware_utils.cleanup(detection, size_thresh=50, inv_thresh=8) detection_maps = copy.deepcopy(binary_maps) for i in range(0,len(detection)): detection_maps[i].data = detection[i] #If there is anything left in 'detection', fit a function to the original #diffmaps in the region defined by 'detection'. Currently fits a #Gaussian in the y-direction for each x #use 'detection' to guess starting fit parameters. #get just the positive elements of the difference map. Perform fitting on #these positive diffmaps. posdiffs = copy.deepcopy(diffs) for i in range(0, len(diffs)): temp = diffs[i].data < 0 posdiffs[i].data[temp] = 0 #fit a function to the difference maps in the cases where there has been a #detection fitparams, wavefront = aware_utils.fit_wavefront(posdiffs, detection) #transform the detected model wavefront back into heliocentric coordinates so it can be overlayed wavefront_hc = aware_utils.map_reravel(wavefront,params,verbose=True) #strip out the velocity information from the wavefront fitting velocity = aware_utils.wavefront_velocity(fitparams) #strip out the position and width information from the wavefront fitting pos_width = aware_utils.wavefront_position_and_width(fitparams) #now save products we have created in a pickle file for future reference #Will save output in ~/aware_results extn=time_range.start().strftime('%Y%m%d_%H%M') save_path=os.path.expanduser('~/aware_results/') save_file='aware_results_' + extn + '.pickle' if not os.path.exists(save_path): os.makedirs(save_path) output=open(save_path + save_file,'wb') print 'Saving result products to: '+ save_path + save_file pickle.dump(wavefront_hc,output) output.close() #visualize the model wavefront visualize(wavefront_hc) return maps, new_maps, diffs, threshold_maps, binary_maps, detection_maps, wavefront, velocity, pos_width, persistence_maps, wavefront_hc
"hpcx_min": -1228.8, "hpcx_max": 1228.8, "hpcx_bin": 2.4, "hpcy_min": -1228.8, "hpcy_max": 1228.8, "hpcy_bin": 2.4 } #wave_maps = wave2d.simulate(params) wave_maps = wave2d.simulate(params, verbose = True) #To get simulated HG' maps (centered at wave epicenter): #wave_maps_raw = wave2d.simulate_raw(params) #wave_maps_raw_noise = wave2d.add_noise(params, wave_maps_raw) visualize(wave_maps) """ import util new_wave_maps = [] for wave in wave_maps: print("Unraveling map at "+str(wave.date)) new_wave_maps += [util.map_hpc_to_hg_rotate(wave, epi_lon = 45., epi_lat = 30., xbin = 5, ybin = 0.2)] from matplotlib import colors wave_maps_raw = wave2d.simulate_raw(params) wave_maps_transformed = wave2d.transform(params, wave_maps_raw, verbose = True)
def user_interface(): """ Prompts for stock data input and offers options to perform calculations on data: best six, worst six, all months' average stock prices with best and worst visually indicated, and comparing two company's standard deviation of monthly average price. :inputs: company_name: String. Name of a stock company. :inputs: company_file: String. Input JSON file name and file location of the stock company relative to mining.py :inputs: option: integer between 1 and 5 :inputs: second_company_name: String. Name of a second stock company. :inputs: second_company_file: String. Input JSON file name and file location of the second stock company relative to mining.py :returns: If option 1 is selected returns the top six months of average stock price in a list of tuples, in the format('YYYY/DD', 111.11) If option 2 is selected returns the top six months of average stock price in a list of tuples, in the format('YYYY/DD', 111.11). If option 3 is selected returns first company's monthly average stock price appears on a separate line in the format "YYYY/MM 111.11" with " <<< One of the Best Months" to the right if it is one of the top six months, or " <<< One of the Worst Months" to the right if it is one of the bottom six months. If option 4 is selected prints one of three options: 1. stock_one_name + " has the highest standard deviation!" or 2. stock_one_name + " and " + stock_two_name + " have the same standard deviation!" or 3. stock_two_name + " has the highest standard deviation!" If an integer outside of 1 to 6 is entered for option, a prompt prints: """ company_name = input("Company name: ") while True: try: company_file = input("Company stock data file " "(i.e., data\GOOG.json): ") read_stock_data(company_name, company_file) break except: print("Invalid Input.") visit = True while visit: print("-"*50) print("Choose your option # (i.e., 1)") print("1. Display the best six months' average stock prices") print("2. Display the worst six months' average stock prices") print("3. Display all months' average stock prices with best and " + "worst six") print("4. Display comparison of two companies' " "monthly average price " + "standard deviation") print("5. Exit") option = (input("Choose: ")) if option == "1": print(six_best_months()) elif option == "2": print(six_worst_months()) elif option == "3": visualize(company_name, company_file) elif option == "4": second_company_name = input("Second Company name " "(i.e., data\GOOG.json): ") while True: try: second_company_file = input("Second Company file: ") print(compare_two_stocks(company_name, company_file, second_company_name, second_company_file)) break except: print("Invalid Input.") elif option == "5": visit = False else: print("Invalid option selected, please choose again from the \ following options: ")
p.add_argument('--usage', nargs='+', default=[],help='search for usages in all project files') p.add_argument('--graph', nargs='*', default=None, help='generate a project dependency graph') p.add_argument('--pyplot', action='store_true', help='use pyplot for the project dependency graph') args = p.parse_args() if not (args.grep or args.defn or args.usage) and args.graph is None: p.error('None of the options specified, nothing to do.') if args.pyplot and not args.graph: p.error('--pyplot specified without --graph') return args def find_files(rootdir): ff = {} for dirname,subdirs,files in os.walk(rootdir): # TODO: have a directory blacklist argument if '.svn' in subdirs: subdirs.remove('.svn') for f in files: # TODO: have a filename criteria argument if re.search('\.(m|c(pp)?|h(pp)?)$',f): ff[f.lower()] = os.path.join(dirname,f) return ff if __name__ == '__main__': args = parse_args() files = find_files(args.project_root) search_patterns(files, args) if args.graph is not None: visualize(files, args)
tmp_file = open(tmp_file_name, 'w') tmp_file.write(input_data) tmp_file.close() # Runs the command: ./tsp.bin tmp_file_name process = Popen(['./tsp.bin', tmp_file_name], stdout=PIPE) (stdout, stderr) = process.communicate() # removes the temporay file os.remove(tmp_file_name) return stdout.strip() import sys if __name__ == '__main__': if len(sys.argv) > 1: file_location = sys.argv[1].strip() input_data_file = open(file_location, 'r') input_data = ''.join(input_data_file.readlines()) input_data_file.close() toPrint = solve_it(input_data) visualize(input_data, toPrint.split('\n')[1].split(' ')) print "Solution:" print toPrint else: print 'This test requires an input file. Please select one from the data directory. (i.e. python solver.py ./data/tsp_51_1)'
cap = cv2.VideoCapture(vid_path) machine_steering = [] timeArr = [] print 'performing inference...' time_start = time.time() for frame_id in xrange(frame_count): ret, img = cap.read() assert ret img = preprocess.preprocess(img) frameTime = time.time() #Get the start time of the angle calculation deg = model.y.eval(feed_dict={model.x: [img], model.keep_prob: 1.0})[0][0] timeArr.append(time.time() - frameTime) #Add angle calculation time to time array machine_steering.append(deg) cap.release() fps = frame_count / (time.time() - time_start) print 'completed inference, total frames: {}, average fps: {} Hz'.format(frame_count, round(fps, 1)) print 'performing visualization...' startTime = time.time() visualize.visualize(epoch_id, machine_steering, params.out_dir, timeArr, verbose=True, frame_count_limit=None) print "Visualization took a total of %i seconds" % (time.time() - startTime)
#!/usr/bin/env python # -*- coding: utf-8 -*- import numpy as np from scipy.io import loadmat import sys import shelve sys.path.append('../') from encoder import Sparse import encoder import visualize if __name__ == '__main__': # create new sparse enconder s = Sparse(nodes = 25) # load data data_mat = loadmat('patch') d = data_mat['patches'] # train sparse encoder encoder.fit(s, d, epochs = 600) filename = 'persistence.data' f = shelve.open(filename) f['autoencoder'] = s f.close() visualize.visualize(s.theta1)
def onReadCmd(self, event): """ reads text from InputBox, passes it asa command to Liggghts, updates the display panel, and finally clears InputBox""" command = self.InputTxt.GetValue() if len(command.split()) == 1: if command == 'clc': self.clearDisplayPanel() elif command == 'whos': for item in self.loadedVars: self.UpdateDisplayPanel(item, type(self.loadedVars[item])) elif command == 'visualize': self.onVisualize() else: self.UpdateDisplayPanel('Unknown command') else: method, var = command.split(' ', 1) if method == 'plot': if var not in self.loadedVars: self.UpdateDisplayPanel('Error: could not find {}'.format(var)) else: self.UpdateDisplayPanel('Generating plot ...') try: if var == 'mesh': visualize.visualize(meshFname=self.loadedVars[var]) else: if var == 'mesh and particles': if var not in self.loadedVars: self.UpdateDisplayPanel('Error: no particles loaded') else: try: visualize.visualize(meshFname=self.loadedVars['mesh'], dumpFname=self.loadedVars['mesh and particles']) except: self.UpdateDisplayPanel('Plot failed with unexpected error: {}'.format(sys.exc_info()[0])) else: pass # make 2D plot except: pass elif method == 'run': if self.__selEngine__: # try to call a DEM-engine method based on user-supplied cmds try: self._module.command('{}'.format(var)) except: self.UpdateDisplayPanel('Unexpected error: {}'.format(sys.exc_info()[0])) else: self.UpdateDisplayPanel('No engine selected. Make sure an available DEM engine is installed.') elif method == 'unix': try: output = os.popen(var).read() self.UpdateDisplayPanel(output) except: self.UpdateDisplayPanel('Unexpected error: {}'.format(sys.exc_info()[0])) else: self.UpdateDisplayPanel('Unknown command') self.InputTxt.Clear()
data = pickle.load(open(os.path.join(dir, pkl_fn), "rb")) face, eye0, eye1 = pickle.load(open(os.path.join(dir, clm_pkl_fn), "rb")) offset = [0, -2, 10, 1] eyeball_offset_clm = face.get_rotation_transform().dot(np.array(offset)) eyeball_3d_pos_clm = (np.array(face.pts_3d[42])+np.array(face.pts_3d[45]))/2.0 + eyeball_offset_clm[:3] true_iris_centre_3d = np.mean(data["ldmks_iris_3d"], axis=0)*100 cam_mat = np.array([[749.9999, 0.0000, 400.0000], [0.0000, 749.9999, 300.0000], [0.0000, 0.0000, 1.0000]]) coord_swap = np.array([[1,0,0],[0,-1,0],[0,0,-1]]) visualize.visualize(face.pts_3d) visualize.visualize([coord_swap.dot(np.array(data["eye_centre_3d"])*100)]) visualize.visualize([eyeball_3d_pos_clm], radius=12) # VISUALIZE IN 2D BEFORE OPTIMIZING for pt in face.pts_2d: cv2.circle(img, tuple([int(p) for p in pt]), 3, 0, -1) cv2.circle(img, tuple([int(p) for p in pt]), 2, (255, 255, 255), -1) cv2.polylines(img, np.array([data["ldmks_lids_2d"]], int), True, 255) cv2.imshow("TEST", img) cv2.waitKey(1) # (e_x, e_y), (e_w, e_h), e_tht = cv2.fitEllipse(np.array(data["ldmks_iris_2d"], dtype=int))
#in the accumulator #indices=((transform == transform.max())+(transform2 == transform2.max())).nonzero() indices = ((transform> votethresh)+(transform2 > votethresh)).nonzero() distances = d[indices[0]] theta = theta[indices[1]] n =len(indices[1]) print("Found " + str(n) + " lines.") # Perform the inverse transform to get a series of rectangular # images that show where0.581673377128 the wavefront is. invTransform = sunpy.make_map(np.zeros(imgShape),input_maps[i+1]._original_header) #invTransform.data = np.zeros(imgShape) for i in range(0,n): nextLine = htLine( distances[i],theta[i], np.zeros(shape=imgShape) ) invTransform = invTransform + nextLine # Dump the inverse transform back into a series of maps detection.append(invTransform) visualize(detection)
# if you just want to load a previously trainmodel? new = True model_ckpt = 'model.ckpt' if os.path.isfile(model_ckpt): input_var = None while input_var not in ['yes', 'no']: input_var = raw_input("We found model.ckpt file. Do you want to load it [yes/no]?") if input_var == 'yes': new = False # start training if new: # visualize result x_test = mnist.test.images.reshape([-1, 28, 28]) embed = siamese.o1.eval({siamese.x1: mnist.test.images}) visualize.visualize(embed, x_test, 'pretrain.png') for step in range(100): batch_x1, batch_y1 = mnist.train.next_batch(128) batch_x2, batch_y2 = mnist.train.next_batch(128) batch_y = (batch_y1 == batch_y2).astype('float') _, loss_v = sess.run([train_step, siamese.loss], feed_dict={ siamese.x1: batch_x1, siamese.x2: batch_x2, siamese.y_: batch_y}) if np.isnan(loss_v): print('Model diverged with loss = NaN') quit()
print res3 x3 = np.where(a[0]!=0, 1, a[0]) y3 = np.where(b[0]!=0, 1, b[0]) N = (y3.sum()) z3 = np.multiply(x3, y3) n = (z3.sum()) res4 = n/N print res4 t = (res+res2+res3+res4) f = t/4 print f '''visualize(detection) visualize(detection2) visualize(diffs) visualize(diffs2)'''
# Let Python search for modules in the parent directory import os, sys CURRENT_DIR = os.path.dirname(os.path.abspath(__file__)) PARENT_DIR = os.path.dirname(CURRENT_DIR) sys.path.append(PARENT_DIR) from missionplanner import plan_complete_coverage_mission import testfields import visualize if __name__ == '__main__': testfield = testfields.test1 # choose any test available in # testfields.py visualization_data, mission = plan_complete_coverage_mission( testfield) visualize.visualize(visualization_data);
n =len(indices[1]) print("Found " + str(n) + " lines.") # Perform the inverse transform to get a series of rectangular # images that show where the wavefront is. invTransform = sunpy.map.BaseMap(input_maps[i+1]) invTransform.data = np.zeros(imgShape) for i in range(0,n): nextLine = htLine( distances[i],theta[i], np.zeros(shape=imgShape) ) invTransform = invTransform + nextLine # Dump the inverse transform back into a series of maps detection.append(invTransform) visualize(diffs) visualize(detection) from matplotlib import cm from matplotlib import colors #wmap = sunpy.make_map(input_maps[max_steps/2], wave_maps[0], type = "composite") #wmap.set_colors(1, cm.Reds) #wmap.set_alpha(1,0.1) #wmap.set_norm(1, colors.Normalize(0.1,1)) #wmap.show() #pmap = sunpy.make_map(detection[max_steps/2],input_maps[max_steps/2], type ="composite") #pmap.set_alpha(1,0.6) #pmap.set_colors(0, cm.Blues) #pmap.set_colors(1, cm.Reds)
def main(source_data='.fits', time_range=TimeRange('2011/10/01 09:45:00', '2011/10/01 10:15:59'), algorithm='hough', feed_directory='/home/hayesla/fits/data/', use_pickle=None,diff_type='running'): ''' source_data { jp2 | fits | test } look for helioviewer JP2 files, FITS files, or load the test data respectively time_range : a TimeRange object time range that is searched for EUV waves feed_directory If set to a string, look in this directory for the jp2 files. Assumes that all the JP2 files are relevant to the flare detection. algorithm: { 'hough' : 'phough' } algorithm used to find the wave ''' if feed_directory != None: feed_directory = os.path.expanduser(feed_directory) if source_data == 'test': # where to store those data maps = test_wave2d() elif source_data == '.fits': # where to store those data data_storage = "/home/hayesla/fits/" if not os.path.exists(os.path.expanduser(data_storage)): os.makedirs(os.path.expanduser(data_storage)) # Query the HEK for flare information we need client = hek.HEKClient() hek_result = client.query(hek.attrs.Time(time_range.t1, time_range.t2), hek.attrs.EventType('FL')) #hek.attrs.FRM.Name == '') if hek_result is None: # no flares, no analysis possible return None # Flares! print('Number of flares found = ' + str(len(hek_result))) for flare in hek_result[10:11]: if feed_directory is None: print('Acquiring data for flare') filelist = eitwaveutils.acquire_data(data_storage, source_data, flare) else: # Assumes that the necessary files are already present filelist = eitwaveutils.listdir_fullpath(feed_directory, filetype ='fits') #filter to only grab the data files with the source_data extn in the directory files_tmp = [] for f in filelist: if f.endswith(source_data): files_tmp.append(f) files = files_tmp # reduce the number of files to those that happen after the flare has # started #files = [] #for f in files_tmp: # fhv = f.split(os.sep)[-1] # if eitwaveutils.hv_filename2datetime(fhv) > \ # parse_time(flare['event_starttime']): # files.append(f) #print('Number of files :' + str(len(files))) #if len(files) == 0: #print('No files found. Returning.') #return None # Define the transform parameters # params = eitwaveutils.params(flare='test') params = eitwaveutils.params(flare) # read in files and accumulate them if use_pickle != None: # load in a pickle file of the data pfile = open(feed_directory + use_pickle, 'rb') a = pickle.load(pfile) maps = a[0] new_maps = a[1] diffs = a[2] pfile.close() else: maps = eitwaveutils.accumulate(files[0:30], accum=1, nsuper=4, verbose=True) #temporary fix for exposure control and S/N changes long_maps = [] for m in maps: if m.exposure_time > 2.0: long_maps.append(m) maps=long_maps for i in range(len(maps)): maps[i] = np.sqrt((maps[i] + abs(maps[i].min())+1)) # Unravel the maps new_maps = eitwaveutils.map_unravel(maps, params, verbose=True) #sometimes unravelling maps leads to slight variations in the unraveled #image dimensions. check dimensions of maps and resample to dimensions #of first image in sequence if need be. new_maps = eitwaveutils.check_dims(new_maps) # calculate the differences if diff_type == 'base': diffs=eitwaveutils.map_basediff(new_maps) else: diffs = eitwaveutils.map_diff(new_maps) # save the outpout #output = open(feed_directory + 'maps.pkl', 'wb') ##pickle.dump([maps, new_maps, diffs], output, protocol=0) #output.close() # Unravel the maps #new_maps = eitwaveutils.map_unravel(maps, params, verbose=True) #sometimes unravelling maps leads to slight variations in the unraveled #image dimensions. check dimensions of maps and resample to dimensions #of first image in sequence if need be. #new_maps = eitwaveutils.check_dims(new_maps) # calculate the differences #diffs = eitwaveutils.map_diff(new_maps) #generate persistence maps #persistence_maps = eitwaveutils.map_persistence(diffs) #determine the threshold to apply to the difference maps. #diffs > diff_thresh will be True, otherwise False. threshold_maps = eitwaveutils.map_threshold(new_maps, factor=0.2) # transform difference maps into binary maps binary_maps = eitwaveutils.map_binary(diffs, threshold_maps) if algorithm == 'hough': # detection based on the hough transform detection = eitwaveutils.hough_detect(binary_maps, vote_thresh=10) elif algorithm == 'prob_hough': # detection based on the probabilistic hough transform. Takes the # keywords of the probabilistic hough transform - see the documentation # of skimage.transform.probabilistic_hough (scikit-image.org) detection = eitwaveutils.prob_hough_detect(binary_maps, threshold=10) # Remove areas that are too small or that don't have enough detections detection = eitwaveutils.cleanup(detection, size_thresh=50, inv_thresh=8) #If there is anything left in 'detection', fit a function to the original #diffmaps in the region defined by 'detection'. Simplest case: fit a #Gaussian in the y-direction for some x or range of x. #eitwaveutils.fit_wavefront should probably take the arguments of fitfunc. #use 'detection' to guess starting fit parameters? #get just the positive elements of the difference map. Perform fitting on #these positive diffmaps. '''posdiffs = copy.deepcopy(diffs) for i in range(0, len(diffs)): temp = diffs[i] < 0 posdiffs[i][temp] = 0 #fit a function to the difference maps in the cases where there has been a #detection wavefront = eitwaveutils.fit_wavefront(posdiffs, detection) #strip out the velocity information from the wavefront fitting velocity = eitwaveutils.wavefront_velocity(wavefront[0]) #strip out the position and width information from the wavefront fitting pos_width = eitwaveutils.wavefront_position_and_width(wavefront[0])''' visualize(detection) return maps, new_maps, diffs, threshold_maps, binary_maps, detection
#HPC grid, probably would only want to change the bin sizes "hpcx_min": -1228.8, "hpcx_max": 1228.8, "hpcx_bin": 2.4, "hpcy_min": -1228.8, "hpcy_max": 1228.8, "hpcy_bin": 2.4 } wave_maps = wave2d.simulate(params, verbose = True) #To get simulated HG' maps (centered at wave epicenter): wave_maps_raw = wave2d.simulate_raw(params) wave_maps_raw_noise = wave2d.add_noise(params, wave_maps_raw) visualize(wave_maps) import util new_wave_maps = [] for wave in wave_maps: print("Unraveling map at "+str(wave.date)) new_wave_maps += [util.map_hpc_to_hg_rotate(wave, epi_lon = params.get('epi_lon'), epi_lat = params.get('epi_lat'), xbin = 5, ybin = 0.2)] input_maps = new_wave_maps #wave_maps = wave2d.simulate(params, verbose = True) #visualize(wave_maps) # # Use Albert's wavemaps to test the hough transform as a means