def sanity_check(model, test_loader, use_gpu=True): """ TODO: add documentation here """ device = get_device(use_gpu) model.to(device) model.eval() original_loss_list = [] final_loss_list = [] for features, labels in test_loader: # move tensors to GPU if CUDA is available features, labels = features.to(device).float(), labels.to( device).float() # forward pass: compute predicted outputs by passing inputs to the model output = model(features) original_loss = [ utils.angular_error( features.data[0, index * 3:index * 3 + 3].detach().numpy(), np.transpose(labels.detach().numpy()))[0] for index in range(len(features[0]) // 3) ] final_loss = utils.angular_error(output.detach().numpy(), np.transpose( labels.detach().numpy()))[0][0] original_loss_list.append(original_loss[-1].item()) final_loss_list.append(final_loss) print("Original:") utils.print_stats(original_loss_list) print("Final:") utils.print_stats(final_loss_list)
def check_unet(weights): dataset = Dataset() model = model_unet(INPUT_SHAPE) model.load_weights(weights) batch_size = 16 for batch_x, batch_y in dataset.generate_validation(batch_size=batch_size): print(batch_x.shape, batch_y.shape) with utils.timeit_context('predict 16 images'): prediction = model.predict_on_batch(batch_x) for i in range(batch_size): # plt.imshow(unprocess_input(batch_x[i])) # plt.imshow(prediction[i, :, :, 0], alpha=0.75) img = batch_x[i].astype(np.float32) mask = prediction[i, :, :, 0] utils.print_stats('img', img) utils.print_stats('mask', mask) img[:, :, 0] *= mask img[:, :, 1] *= mask img[:, :, 2] *= mask img = unprocess_input(img) plt.imshow(img) plt.show()
def check_watershed_direction_and_energy(): data = dataset.UVectorNetDataset(fold=-1, batch_size=4, output_watershed=True) for mask_samples in data.masks.values(): crops = [] for mask, crop_offset in mask_samples: cfg = dataset.SampleCfg(sample_id=0, src_center_x=128, src_center_y=128, scale_x=1, scale_y=1) tform = cfg.transform() crop = utils.transform_crop(mask, crop_offset, tform, output_shape=(dataset.CROP_SIZE, dataset.CROP_SIZE)) crops.append(crop) wshed = dataset.watershed_direction_and_energy( crops, output_mask_area=data.output_mask_area) plt.subplot(3, 3, 1) plt.title('mask') plt.imshow(np.sum(crops, axis=0)) for i in range(wshed.shape[2]): plt.subplot(3, 3, i + 2) plt.title('mask') utils.print_stats(f'res {i}', wshed[:, :, i]) plt.imshow(wshed[:, :, i]) plt.show()
def test(model, loss, loader, xp, args): if not len(loader): return 0 model.eval() metrics = xp.get_metric(tag=loader.tag, name='parent') timer = xp.get_metric(tag=loader.tag, name='timer') metrics.reset() if args.multiple_crops: epoch_test_multiple_crops(model, loader, xp, args.cuda) else: epoch_test(model, loader, xp, args.cuda) # measure elapsed time timer.update() xp.log_with_tag(loader.tag) if loader.tag == 'val': xp.Acc1_Val_Best.update(float(xp.acc1_val)).log() xp.Acck_Val_Best.update(float(xp.acck_val)).log() if args.verbosity: print_stats(xp, loader.tag) if args.eval: dump_results(xp, args)
def non_interactive_demo(model, args): renderer = create_renderer() show_window = not args.no_show for rec in tqdm(model.images_list): log.info("Starting inference for %s", rec['img_name']) image = rec['img'] distribution, targets = model.infer_sync(image) prob = calculate_probability(distribution) log.info("Confidence score is %s", prob) if prob >= args.conf_thresh ** len(distribution): phrase = model.vocab.construct_phrase(targets) if args.output_file: with open(args.output_file, 'a') as output_file: output_file.write(rec['img_name'] + '\t' + phrase + '\n') else: print("\n\tImage name: {}\n\tFormula: {}\n".format(rec['img_name'], phrase)) if renderer is not None: rendered_formula, _ = renderer.render(phrase) if rendered_formula is not None and show_window: cv.imshow("Predicted formula", rendered_formula) cv.waitKey(0) else: log.info("Confidence score is low. The formula was not recognized.") if args.perf_counts: log.info("Encoder performance statistics") print_stats(model.exec_net_encoder) log.info("Decoder performance statistics") print_stats(model.exec_net_decoder)
def finish(buckets, num_bytes, X, V, row_reorder, col_reorder, filename, recons): compressed_X = _reconstruct(X, buckets, row_reorder, col_reorder, recons) # Print stats and send to file. utils.print_stats(X, compressed_X, num_bytes) utils.to_file(filename, V, compressed_X)
def find_init_h0(input_file, dataset_info, n_iters): A = utils.read_adj_sparse_matrix(input_file, comment=dataset_info['comment'], sep=dataset_info['sep'], min_node=dataset_info['min_node']) print(A.dtype) print("File read") components = utils.find_connected_components(A) largest = components[np.argmax([len(i) for i in components])] A = A[largest, :][:, largest] S_max = [] for i in range(n_iters): # TIMBAL # We discard all connected components but the largest # Subsample => Whether or not to sample random graphs. Recommended when the graph is large # Max_removals => Max number of removed vertices per iteration # samples => Randomly sampled graphs. Must be a factor of the number of threads used # avg_size => Approximate desired size of randomly sampled graph start_time = time.time() print("TIMBAL started") S = timbal.process(A, max_removals=dataset_info['max_removals'], samples=dataset_info['samples'], avg_size=dataset_info['avg_size'], subsample=dataset_info['subsample']) print("TIMBAL finished") print("Time taken to find h0: ", str(time.time() - start_time) + " s") if ((len(S_max) < len(S)) and (utils.is_balanced(A[S, :][:, S]))): S_max = S As = A[S_max, :][:, S_max] print("\nH") utils.print_stats(A) print("\nS(H)") utils.print_stats(As) return (A, S_max)
def predict(model_name, epoch_num, img_dir, sample_ids, run=None): model = EmbeddingsModel() model = model.cuda() run_str = '' if run is None or run == '' else f'_{run}' checkpoints_dir = f'../output/checkpoints_3/{model_name}{run_str}' checkpoint = torch.load(f'{checkpoints_dir}/{epoch_num:03}.pt') model.load_state_dict(checkpoint['model_state_dict']) dataset = TripletDatasetPredict(sample_ids=sample_ids, img_dir=img_dir, transform=transforms.Compose([ transforms.ToTensor(), transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]) ]), crop_size=256) data_loader_update_train = DataLoader( dataset, shuffle=False, num_workers=8, batch_size=64) results = [] results_idx = [] with torch.set_grad_enabled(False): for data in tqdm(data_loader_update_train): img = data['img'].cuda() samples_idx = data['idx'] embeddings = model(img).detach().cpu().numpy() results.append(embeddings) results_idx.append(samples_idx) # for image_id in tqdm(sample_ids): # images = [] # for color in ['red', 'green', 'blue']: # try: # img = cv2.imread(f'{img_dir}/{image_id}_{color}.png', cv2.IMREAD_UNCHANGED) # img = cv2.resize(img, (256, 256), interpolation=cv2.INTER_AREA).astype("uint8") # images.append(img) # except: # print(f'failed to open {img_dir}/{image_id}_{color}.png') # raise # # images = np.stack(images, axis=0).astype(np.float32) / 255.0 # images = torch.from_numpy(images).cuda() # images = normalize(images) # images = torch.unsqueeze(images, 0) # embeddings = model(images) # embeddings = embeddings.detach().cpu().numpy() # # print(image_id, embeddings.flatten()) # results.append(embeddings) results = np.concatenate(results, axis=0) results_idx = np.concatenate(results_idx, axis=0) utils.print_stats('results_idx diff', np.diff(results_idx)) return results
def check_dataset_aug(): with utils.timeit_context('load ds'): ds = ClassificationDataset(fold=0, is_training=True, img_aug_level=20, geometry_aug_level=10) while True: sample = ds[1] utils.print_stats('img', sample['img']) plt.imshow(np.moveaxis(sample['img'], 0, 2)[:, :, :3]) plt.show()
def run_train(config, data, model_file, db=None): net_model = get_net_model(config) data.load() training_data = data.get_training_data(config['validation_split'], additional_layers_conf=net_model.get_additional_layers_conf()) model = net_model.get_model(config, data) model.summary() gc.collect() tr = train_model(model, config, training_data) training_data = None gc.collect() test_data = data.get_validation_test_data(config['validation_split'], additional_layers_conf=net_model.get_additional_layers_conf()) x_test = net_model.get_x_val_test(test_data) y_test = test_data['y_val_test'] y_pred = model.predict(x_test) y_pred = get_y_pred(config, y_pred) p_r_f, acc, p_r_f_avg_micro, p_r_f_avg_weighted = print_stats(y_test, y_pred, data.labels_index, config['binary']) if model_file: data.save_model(model, model_file) not_matched = test_data['not_matched'] if len(not_matched): print() print ('Computing metrics with not matched true candidates') y_test = y_test.tolist() y_pred = y_pred.tolist() for label in not_matched: for _ in range(not_matched[label]): y_test.append(data.get_label_from_idx(data.labels_index[label])) y_pred.append(data.get_label_from_idx(data.labels_index[data.irrelevant_class])) p_r_f, acc, p_r_f_avg_micro, p_r_f_avg_weighted = print_stats(np.array(y_test), np.array(y_pred), data.labels_index, config['binary']) sys.stdout = mystdout = StringIO() model.summary() sys.stdout = sys.__stdout__ model_summary = mystdout.getvalue() if db: db.init() db.save_result(config, acc, p_r_f, model_summary, data.num_classes, tr["class_weights"], labels_index=data.labels_index, model_id=model_file, p_r_f_avg_micro = p_r_f_avg_micro, p_r_f_avg_weighted = p_r_f_avg_weighted)
def trainP(net, disc, optim_g, optim_d, device, data_loader, start_step, current_epoch, epochs=1, train_disc=True, step_update=100, batch_size=1): losses = [] epoch_times = [] vgg = [ VGGFeatureExtractor().float().cuda(), VGGFeatureExtractor(pool_layer_num=36).float().cuda() ] for i, (images, targets, bicub) in enumerate(data_loader): optim_g.zero_grad() images = images.to(device) targets = targets.to(device) bicub = bicub.to(device) images = images.view((-1, 3, 32, 32)) targets = targets.view((-1, 3, 128, 128)) bicub = bicub.view((-1, 3, 128, 128)) images = images.squeeze(0) targets = targets.squeeze(0) bicub = bicub.squeeze(0) output = net(images.float()) output = torch.add(output, bicub).clamp(0, 1) output = output.to(device) loss = LossP(device, vgg[0](output.float()), vgg[0](targets.float()), vgg[1](output.float()), vgg[1](targets.float())) losses.append(loss.detach().item()) loss.backward() optim_g.step() if i % step_update == 0 and i is not 0: end_step = time.perf_counter() epoch_times.append(end_step - start_step) print_stats(False, current_epoch, epochs, len(data_loader), i, losses, [], [], [], [], step_update) time_stats(epoch_times, end_step, start_step, len(data_loader), i) losses = [] start_step = time.perf_counter()
def spam_image(): spam_count = utils.get_spam_count() print('\n** Make sure the image is copied to your clipboard **') utils.prompt_confirmation_and_countdown() image_count = 0 start_time = time.time() for _ in range(spam_count): pyautogui.hotkey('command', 'v') pyautogui.press('enter') image_count += 1 utils.check_and_print_progress(image_count) utils.print_stats(start_time, image_count)
def multiple_demo(load_fnames, save_fname="multi", reg_a=10., reg_d=10.): # assignment: section 6 bfm = h5py.File(MODELS_PATH + BFM_FNAME, "r") bfm_params, color, triangles = utils.read_bfm(bfm) lms = utils.read_landmarks(MODELS_PATH + LM_FNAME) # landmark annotations N = len(load_fnames) # number of images to be loaded imgs_real = [utils.read_image(IMAGE_PATH + fname) for fname in load_fnames] # load all images hs = [np.size(img, 0) for img in imgs_real] # store all heights ws = [np.size(img, 1) for img in imgs_real] # store all widths lms_real = [torch.from_numpy(detect_landmark(img)) for img in imgs_real] # detect all ground truth landmarks lms_real_flip = [utils.flip_ycoords(lms_real[i], H=hs[i]) for i in range(N)] # flip y axis alpha, deltas, rotations, translations, loss = multiple.estimate_params((bfm_params, color, triangles), lms, lms_real_flip, hs=hs, ws=ws, reg_a=reg_a, reg_d=reg_d) utils.save_loss(loss, save_fname=save_fname + "_loss.pdf") # save results for each image for i in range(N): print(load_fnames[i] + ":") # print stats for each image (alpha is the same for each img) utils.print_stats(alpha, deltas[i], rotations[i], translations[i]) G = morph.compute_G(bfm_params, alpha=alpha, delta=deltas[i]) G_transformed = pinhole.transform(G, rotations[i], translations[i]) G_pinhole = pinhole.camera_model(G, rotations[i], translations[i], h=hs[i], w=ws[i]) color = texture.get_color(imgs_real[i], G_pinhole[:, :2]) print("Rendering...") img_pred = utils.get_image(G_pinhole, color, triangles, h=hs[i], w=ws[i]) utils.show_face(img_pred) utils.flip_y() plt.savefig(PINHOLE_PATH + save_fname + str(i) + ".pdf") plt.close() save_obj(OBJ_3D_PATH + save_fname + str(i) + "_3d.obj", G_transformed, color, triangles) save_obj(OBJ_2D_PATH + save_fname + str(i) + "_2d.obj", G_pinhole, color, triangles) lm_pred_flip = utils.get_landmarks(G_pinhole[:, :2], lms) lm_pred = utils.flip_ycoords(lm_pred_flip, H=hs[i]) utils.show_face(imgs_real[i], white_background=False) utils.show_landmarks(lms_real[i], indices=False, label="ground-truth") try: utils.show_landmarks(lm_pred, indices=False, label="model") except TypeError: print("... unable to show predicted landmarks") plt.savefig(PINHOLE_PATH + save_fname + str(i) + "_lm.pdf") plt.close()
def spam_message(): message = input('What message would you like to send?: ') spam_count = utils.get_spam_count() utils.prompt_confirmation_and_countdown() start_time = time.time() message_count = 0 for _ in range(spam_count): pyautogui.write(message) pyautogui.press('enter') message_count += 1 utils.check_and_print_progress(message_count) utils.print_stats(start_time, message_count)
def spam_text_file(): file_path = utils.choose_file() file = open(file_path, 'r') utils.prompt_confirmation_and_countdown() word_count = 0 start_time = time.time() for line in file: for word in line.split(): pyautogui.write(word) pyautogui.press('enter') word_count += 1 utils.check_and_print_progress(word_count) file.close() utils.print_stats(start_time, word_count)
def _stats(self, accum_recount): print('=========================================') print(f'Max p-value: {self.max_p_value:.5f}') print(f'Polled ballots: {self.m}') W = [ winner for winner in self.party_members if winner in self.winning_candidates ] L = [ loser for loser in self.party_members if loser not in self.winning_candidates ] utils.print_stats(accum_recount, W, L) print('=========================================')
def train(model, loss, optimizer, loader, xp, args): if args.use_dali: loader.reset() loader_len = loader._size // loader.batch_size else: loader_len = len(loader) if not loader_len: return 0 model.train() xp.Parent_Train.reset() for batch_idx, batch in tqdm(enumerate(loader), desc='Train Epoch', leave=False, total=loader_len): if args.use_dali: data = batch[0]['data'] target = batch[0]['label'].squeeze().cuda().long() else: data, target = batch data, target = data_to_var(data, target, args.cuda) output = model(data) obj = loss(output, target) if obj.item() != obj.item(): print('NaN Erorr') import sys sys.exit(-1) optimizer.zero_grad() obj.backward() optimizer.step() prec1 = accuracy(output.data, target.data, topk=1) preck = accuracy(output.data, target.data, topk=xp.config['topk']) xp.Parent_Train.update(loss=obj.data.item(), acck=preck, acc1=prec1, n=data.size(0)) # compute objective function (including regularization) obj = xp.Loss_Train.get() + regularization(model, xp.mu) xp.Obj_Train.update(obj) # measure elapsed time xp.Timer_Train.update() xp.log_with_tag('train') if args.verbosity: print_stats(xp, 'train')
def train(): notcars = glob.glob('data/non-vehicles/*/*.png') cars = glob.glob('data/vehicles/*/*.png') print_stats(cars, notcars) features_car = [] for car in cars: img = read_image(car) img_processed = process_image(img) features_car.append(extract_features(img_processed, parameters)) features_notcar = [] for notcar in notcars: img = read_image(notcar) img_processed = process_image(img) # png features_notcar.append(extract_features(img_processed, parameters)) features = np.vstack((features_car, features_notcar)) # Fit a per-column scaler scaler = StandardScaler().fit(features) # Apply the scaler to X features_scaled = scaler.transform(features) # Define the labels vector labels = np.hstack( (np.ones(len(features_car)), np.zeros(len(features_notcar)))) # Split up data into randomized training and test sets rand_state = np.random.randint(0, 100) out = train_test_split(features_scaled, labels, test_size=0.2, random_state=rand_state) features_train, features_test, labels_train, labels_test = out # Initialize support vector machine object clf = SVC(kernel='linear', C=0.00001) # Check the training time for the SVC t = time.time() clf.fit(features_train, labels_train) print('{0:2.2f} seconds to train SVC...'.format(time.time() - t)) # Accuracy score accuracy = clf.score(features_test, labels_test) print('Test Accuracy of SVC = {0:2.4f}'.format(accuracy)) classifier = Classifier(clf, scaler) joblib.dump(classifier, 'classifier.pkl') return classifier
def main(): # Prepare args parser = utils.create_arg_parser() args = parser.parse_args() ok = '{}OK{}'.format(colors.GREEN, colors.ENDC) # Create mirror, base_dir and contents_filename. print('Initial setup ... ', end='') mirror = utils.create_ftp_mirror(args.country) base_dir = utils.create_base_dir(args.dist, args.comp) contents_filename = utils.create_contents_filename(args.architecture) print(ok) # Establish ftp connection. print('Establishing FTP connection ... ', end='') ftp = utils.create_ftp_connection(mirror, base_dir) print(ok) # Download Contents-xxx.gz file. print('Downloading {}{}{} ... '.format(colors.WARNING, contents_filename, colors.ENDC), end='') utils.download_contents(ftp, contents_filename) print(ok) # Destroy ftp connection as we no longer need it. print('Closing FTP connection ... ', end='') utils.destroy_ftp_connection(ftp) print(ok) # Create a Counter object to store statistics. package_entries = Counter() # Begin parsing. print('Parsing ... ', end='') with utils.gunzip_file(contents_filename) as contents_file: for line in contents_file.readlines(): utils.parse_line_and_update_entries(line, package_entries) print(ok) # Get Top N packages by number of associated files. print('Analyzing data ... ', end='') top_packages = utils.get_top_packages(args.top, package_entries) print(ok) # Print stats. utils.print_stats(top_packages)
def test_agent(env, agent, run=0, episodes=5, time_steps=500, initial_state=None, initial_noise=None, render=True, deterministic=True): stats = EpisodeStats(episode_lengths=np.zeros(episodes), episode_rewards=np.zeros(episodes), episode_loss=np.zeros(episodes)) print_header(3, 'Testing') for e in range(episodes): s = env.reset(initial_state=initial_state, noise_amplitude=initial_noise) for t in range(time_steps): if render: env.render() a = agent.get_action(s, deterministic=deterministic) s, r, d, _ = env.step(tn(a)) stats.episode_rewards[e] += r stats.episode_lengths[e] = t if d: break pr_stats = { 'run': run, 'steps': int(stats.episode_lengths[e] + 1), 'episode': e + 1, 'episodes': episodes, 'reward': stats.episode_rewards[e] } print_stats(pr_stats) if render: env.viewer.close() return stats
def run_cv(config, data, model_file=None): data.load() fold_idx = 0 cv_results = [] acc_list = [] for fold_data in data.get_cv_folds_data(config['validation_split']): print("Running Fold", fold_idx + 1) fold_idx += 1 model = branched_bi_gru_lstm.get_model(config, data) # model.summary() train_model(model, config, fold_data) x_test = branched_bi_gru_lstm.get_cv_x_test(fold_data) y_pred = model.predict(x_test) y_pred = get_y_pred(config, y_pred) p_r_f, acc = print_stats(fold_data['y_test'], y_pred, data.labels_index, config['binary']) acc_list.append(acc) cv_results.append(p_r_f) if model_file: data.save_model(model, model_file + "_fold_" + str(fold_idx)) cv_prf = merge_several_folds_results(cv_results, fold_idx) print_p_r_f(cv_prf, data.labels_index)
def texture_demo(load_fname="yke_neutral.jpeg", save_fname="texture", reg_a=10., reg_d=10., idx=0): # assignment: section 5 bfm = h5py.File(MODELS_PATH + BFM_FNAME, "r") bfm_params, color, triangles = utils.read_bfm(bfm) lms = utils.read_landmarks(MODELS_PATH + LM_FNAME) # landmark annotations img_real = utils.read_image(IMAGE_PATH + load_fname) # load image of face we want to reconstruct h, w, _ = np.shape(img_real) lm_real = torch.from_numpy(detect_landmark(img_real)) # detect ground-truth landmarks lm_real_flip = utils.flip_ycoords(lm_real, H=h) # flip y axis because img is upside down compared to pinhole output alpha, delta, rotation, translation, loss = latent.estimate_params((bfm_params, color, triangles), lms, lm_real_flip, h=h, w=w, reg_a=reg_a, reg_d=reg_d) utils.print_stats(alpha, delta, rotation, translation) # latent params statistics utils.save_loss(loss, save_fname=save_fname + str(idx) + "_loss.pdf") G = morph.compute_G(bfm_params, alpha=alpha, delta=delta) G_pinhole = pinhole.camera_model(G, rotation, translation, h=h, w=w) color = texture.get_color(img_real, G_pinhole[:, :2]) # obtain vertex colors from provided image save_obj(OBJ_3D_PATH + save_fname + str(idx) + "_3d.obj", G, color, triangles) save_obj(OBJ_2D_PATH + save_fname + str(idx) + "_2d.obj", G_pinhole, color, triangles) print("Rendering...") img_pred = utils.get_image(G_pinhole, color, triangles, h=h, w=w) utils.show_face(img_pred) utils.flip_y() plt.savefig(PINHOLE_PATH + save_fname + str(idx) + ".pdf") plt.close() lm_pred_flip = utils.get_landmarks(G_pinhole[:, :2], lms) lm_pred = utils.flip_ycoords(lm_pred_flip, H=h) utils.show_face(img_real, white_background=False) utils.show_landmarks(lm_real, indices=False, label="ground-truth") try: utils.show_landmarks(lm_pred, indices=False, label="model") except TypeError: print("... unable to show predicted landmarks") plt.savefig(PINHOLE_PATH + save_fname + str(idx) + "_lm.pdf") plt.close()
def get_votes(images_path, correct_illums, debug=False): # TODO: add doc here # constants GREY_WORLD = 'grey_world' MAX_RGB = 'max_rgb' GREY_EDGE = 'grey_edge' # keys lists keys = [GREY_WORLD, MAX_RGB, GREY_EDGE] # voters lists voters = { GREY_WORLD: lambda x: grey_edge(x, njet=0, mink_norm=1, sigma=0), MAX_RGB: lambda x: grey_edge(x, njet=0, mink_norm=-1, sigma=0), GREY_EDGE: lambda x: grey_edge(x, njet=1, mink_norm=5, sigma=2) } # illum lists illums = {GREY_WORLD: [], MAX_RGB: [], GREY_EDGE: []} # error lists errors = {GREY_WORLD: [], MAX_RGB: [], GREY_EDGE: []} # estimate illuminations and calculate error for index, (image_path, correct_illum) in enumerate(zip(images_path, correct_illums)): image = skio.imread(image_path) if debug: print(image_path) print('illumination: ' + str(correct_illum)) for key in keys: estim_illum = voters[key](image) illums[key].append(estim_illum) errors[key].append(utils.angular_error(estim_illum, correct_illum)) if debug: print(key + ": " + str(estim_illum) + " error: " + str(errors[key][-1])) if debug: print() else: utils.clear_screen() print(str(index + 1) + ' / ' + str(len(images_path))) for key in keys: print(key + ":") utils.print_stats(errors[key]) return illums, errors
def spam_image_url(): image_url = input('Image URL: ') image_name = image_url.split('/')[-1] image_path = IMAGE_DOWNLOAD_FOLDER + image_name urllib.request.urlretrieve(image_url, image_path) print(f'Downloaded image {image_name} as {image_path}') utils.copy_image_to_clipboard(image_path) spam_count = utils.get_spam_count() utils.prompt_confirmation_and_countdown() image_count = 0 start_time = time.time() for _ in range(spam_count): pyautogui.hotkey('command', 'v') pyautogui.press('enter') image_count += 1 utils.check_and_print_progress(image_count) utils.print_stats(start_time, image_count)
def check_augmentations(): with utils.timeit_context('load ds'): ds = NihDataset(fold=0, is_training=True, img_size=512) sample_num = 2 ds.is_training = False plt.imshow(ds[sample_num]['img']) plt.figure() ds.is_training = True for i in range(100): sample = ds[sample_num] utils.print_stats('img', sample['img']) plt.imshow(sample['img']) plt.show()
def print_stat(twitter_name): """ compute the figures if needed, then serves it """ token = datetime.datetime.now().strftime("%Y%m%d") events = get_tweets_for_user(target_user='******'+twitter_name, n_rounds=16) figure_path = op.join(project_path, 'figures', '%s_%s.png' % (twitter_name, token)) print "Entering stat method" returned_path = print_stats(events, figure_path=figure_path) print returned_path return send_file(figure_path, mimetype='image/png')
def main(num_haps, num_snps, Ne=1e5, length=5e3, recombination_rate=2e-8, mutation_rate=2e-8, random_seed=None, flip=False, verbose=False, print_matrices=False): true_haplotypes = np.array([[]]) while true_haplotypes.shape[1] < num_snps: if verbose: print('simulating...') true_haplotypes = simulate_haplotype_matrix( num_haps, num_snps, Ne=Ne, length=length, recombination_rate=recombination_rate, mutation_rate=mutation_rate, random_seed=random_seed) if flip: random.seed(a=random_seed) column_list = random.choices([0, 1], k=num_haps) true_haplotypes = flip_columns(column_list, true_haplotypes) write_vcf_from_haplotype_matrix(INPUT_VCF, true_haplotypes, phased=False) phased_haplotypes = beagle_phase(BEAGLE_JAR_PATH, INPUT_VCF, BEAGLE_OUTPUT_PATH) # constructed for statistics later, not used by beagle, which works directly on vcf genotypes = compress_to_genotype_matrix(true_haplotypes) unphased_haplotypes = get_incomplete_phasing_matrix(genotypes) # TODO nthomas: add some facility to contribute reference haplotypes to beagle if verbose: print_stats(true_haplotypes, unphased_haplotypes, phased_haplotypes, print_matrices=print_matrices) return true_haplotypes, phased_haplotypes
def run(YEAR, DAY, p1_fn, p2_fn, cmds, FILE=None): target = get_target(YEAR, DAY) fmt_str = '%(asctime)-15s %(filename)8s:%(lineno)-3d %(message)s' log.basicConfig(level=log.DEBUG, format=fmt_str) force = 'force_fetch' in cmds v = fetch(YEAR, DAY, log, wait_until_date=target, force=force) if FILE != None: writeInputToFolder(FILE, v) if 'print_stats' in cmds: print_stats(v) if 'run1' in cmds: res = p1_fn(v) print('part_1: {}'.format(res)) if 'submit1' in cmds: submit(YEAR, DAY, 1, res) if 'run2' in cmds: res = p2_fn(v) print('part_2: {}'.format(res)) if 'submit2' in cmds: submit(YEAR, DAY, 2, res)
def main(num_haps, num_snps, num_ref=0, Ne=1e5, length=5e3, recombination_rate=2e-8, mutation_rate=2e-8, random_seed=None, flip=False, verbose=False, print_matrices=False): # simulate with msprime true_haplotypes = np.array([[]]) while true_haplotypes.shape[1] < num_snps: if verbose: print('simulating...') true_haplotypes = simulate_haplotype_matrix(num_haps, num_snps, Ne=Ne, length=length, recombination_rate=recombination_rate, mutation_rate=mutation_rate, random_seed=random_seed) if flip: random.seed(a=random_seed) column_list = random.choices([0, 1], k=num_haps) true_haplotypes = flip_columns(column_list, true_haplotypes) genotypes = compress_to_genotype_matrix(true_haplotypes) unphased_haplotypes = get_incomplete_phasing_matrix(genotypes) unphased_haplotypes = np.vstack([unphased_haplotypes, true_haplotypes[0:num_ref]]) true_with_ref = np.vstack([true_haplotypes, true_haplotypes[0:num_ref]]) phased_haplotypes = phission_phase(unphased_haplotypes) if verbose: print_stats(true_with_ref, unphased_haplotypes, phased_haplotypes, print_matrices=print_matrices) return true_haplotypes, phased_haplotypes
def check_color_conv(): data = dataset.UVectorNetDataset(fold=-1, batch_size=1, output_watershed=True, use_colors=False) from skimage.color import rgb2hed import skimage.exposure import PIL.Image for sample_id, img in data.images.items(): print(sample_id) fn = '/home/dmytro/ml/kaggle/2018_data_science_bowl/input/stage1_test/0f1f896d9ae5a04752d3239c690402c022db4d72c0d2c087d73380896f72c466/images/0f1f896d9ae5a04752d3239c690402c022db4d72c0d2c087d73380896f72c466.png' img = PIL.Image.open(fn) img = np.array(img)[:, :, :3] plt.subplot(1, 3, 1) utils.print_stats('img', img) plt.imshow(img) plt.subplot(1, 3, 2) preprocessed = data.preprocess(img)[:, :, 0] utils.print_stats('preprocessed', preprocessed) plt.imshow(preprocessed) plt.subplot(1, 3, 3) img_hed = rgb2hed(img / 255.0 * 2 - 1)[:, :, 0] utils.print_stats('img_hed', img_hed) # hed_scaled = skimage.exposure.rescale_intensity(img_hed[:, :, 0]*-1, out_range=(0.0, 1.0)) # utils.print_stats('img_hed_scaled', img_hed) plt.imshow(img_hed * -1) plt.show()
def process_po(ref_messages, po, glob_stats, do_stats, do_messages): print("Checking {}...".format(po)) ret = 0 messages, states, stats = utils.parse_messages(po) if do_messages: t = print_diff(ref_messages, messages, states) if t: ret = t if do_stats: print("\tStats:") t = utils.print_stats(stats, glob_stats, prefix=" ") if t: ret = t if states["is_broken"]: print("\tERROR! This .po is broken!") ret = 1 return ret
def main(): import argparse parser = argparse.ArgumentParser(description="" \ "Merge one or more .po files into the first dest one.\n" \ "If a msgkey (msgctxt, msgid) is present in more than " \ "one merged po, the one in the first file wins, unless " \ "it’s marked as fuzzy and one later is not.\n" \ "The fuzzy flag is removed if necessary.\n" \ "All other comments are never modified.\n" \ "Commented messages in dst will always remain " \ "commented, and commented messages are never merged " \ "from sources.") parser.add_argument('-s', '--stats', action="store_true", help="Show statistics info.") parser.add_argument('-r', '--replace', action="store_true", help="Replace existing messages of same \"level\" already in dest po.") parser.add_argument('dst', metavar='dst.po', help="The dest po into which merge the others.") parser.add_argument('src', metavar='src.po', nargs='+', help="The po's to merge into the dst.po one.") args = parser.parse_args() ret = 0 done_msgkeys = set() done_fuzzy_msgkeys = set() nbr_merged = 0 nbr_replaced = 0 nbr_added = 0 nbr_unfuzzied = 0 dst_messages, dst_states, dst_stats = utils.parse_messages(args.dst) if dst_states["is_broken"]: print("Dest po is BROKEN, aborting.") return 1 if args.stats: print("Dest po, before merging:") utils.print_stats(dst_stats, prefix="\t") # If we don’t want to replace existing valid translations, pre-populate # done_msgkeys and done_fuzzy_msgkeys. if not args.replace: done_msgkeys = dst_states["trans_msg"].copy() done_fuzzy_msgkeys = dst_states["fuzzy_msg"].copy() for po in args.src: messages, states, stats = utils.parse_messages(po) if states["is_broken"]: print("\tSrc po {} is BROKEN, skipping.".format(po)) ret = 1 continue print("\tMerging {}...".format(po)) if args.stats: print("\t\tMerged po stats:") utils.print_stats(stats, prefix="\t\t\t") for msgkey, val in messages.items(): msgctxt, msgid = msgkey # This msgkey has already been completely merged, or is a commented one, # or the new message is commented, skip it. if msgkey in (done_msgkeys | dst_states["comm_msg"] | states["comm_msg"]): continue is_ttip = utils.is_tooltip(msgid) # New messages does not yet exists in dest. if msgkey not in dst_messages: dst_messages[msgkey] = messages[msgkey] if msgkey in states["fuzzy_msg"]: done_fuzzy_msgkeys.add(msgkey) dst_states["fuzzy_msg"].add(msgkey) elif msgkey in states["trans_msg"]: done_msgkeys.add(msgkey) dst_states["trans_msg"].add(msgkey) dst_stats["trans_msg"] += 1 if is_ttip: dst_stats["trans_ttips"] += 1 nbr_added += 1 dst_stats["tot_msg"] += 1 if is_ttip: dst_stats["tot_ttips"] += 1 # From now on, the new messages is already in dst. # New message is neither translated nor fuzzy, skip it. elif msgkey not in (states["trans_msg"] | states["fuzzy_msg"]): continue # From now on, the new message is either translated or fuzzy! # The new message is translated. elif msgkey in states["trans_msg"]: dst_messages[msgkey]["msgstr_lines"] = messages[msgkey]["msgstr_lines"] done_msgkeys.add(msgkey) done_fuzzy_msgkeys.discard(msgkey) if msgkey in dst_states["fuzzy_msg"]: dst_states["fuzzy_msg"].remove(msgkey) nbr_unfuzzied += 1 if msgkey not in dst_states["trans_msg"]: dst_states["trans_msg"].add(msgkey) dst_stats["trans_msg"] += 1 if is_ttip: dst_stats["trans_ttips"] += 1 else: nbr_replaced += 1 nbr_merged += 1 # The new message is fuzzy, org one is fuzzy too, # and this msgkey has not yet been merged. elif msgkey not in (dst_states["trans_msg"] | done_fuzzy_msgkeys): dst_messages[msgkey]["msgstr_lines"] = messages[msgkey]["msgstr_lines"] done_fuzzy_msgkeys.add(msgkey) dst_states["fuzzy_msg"].add(msgkey) nbr_merged += 1 nbr_replaced += 1 utils.write_messages(args.dst, dst_messages, dst_states["comm_msg"], dst_states["fuzzy_msg"]) print("Merged completed. {} messages were merged (among which {} were replaced), " \ "{} were added, {} were \"un-fuzzied\"." \ "".format(nbr_merged, nbr_replaced, nbr_added, nbr_unfuzzied)) if args.stats: print("Final merged po stats:") utils.print_stats(dst_stats, prefix="\t") return ret
if not batch_size: filename = paths.SERIALIZED_DIR + "test_batch.bin" print "Writing file", filename local.serialized.tofile(filename) elif local.serialized.any(): write_batch() all_data = {} for sequence in TRAINING_DATA: sequence.download() data = sequence.read() if not all_data: all_data = data else: for key, value in all_data.iteritems(): all_data[key] = np.concatenate( (all_data[key], data[key]), axis=0) if os.path.exists(paths.SERIALIZED_DIR): shutil.rmtree(paths.SERIALIZED_DIR) utils.print_stats(all_data, ['train', 'test']) print "Serializing.." utils.create_dirs_if_not_exists(paths.SERIALIZED_DIR) serialize(all_data['train_dataset'], all_data['train_labels'], batch_size=2000) serialize(all_data['test_dataset'], all_data['test_labels'])
datasets = [ ('HeLa2D', 'hela', True, 'slf7dna', False), ('RT-widefield-no-origin', 'labeled-widefield', True, 'field-dna+', False), ('RT-widefield', 'labeled-widefield', True, 'field-dna+', True), ('RT-confocal-no-origin', 'labeled-confocal', True, 'field-dna+', False), ('RT-confocal', 'labeled-confocal', True, 'field-dna+', True), ('LOCATE-transfected', 'SubCellLoc/Transfected', True, 'field-dna+', False), ('LOCATE-endogenous', 'SubCellLoc/Endogenous', True, 'field-dna+', False), ('binucleate', 'binucleate', False, 'field+', False), ('CHO', 'cho', False, 'field+', False), ('Terminal Bulb', 'terminalbulb', False, 'field+', False), ('RNAi', 'rnai', False, 'field+', False), ] print_stats(datasets) run_twice = ['RT-widefield'] compares = [] rt_compares = [] sizes = {} for name,directory,has_dna,base,use_origins in datasets: images = CachedFunction(load_directory,'../data/'+directory) sizes[name] = len(images) learner = precluster_learner_plus_features(kfrac=4) origins = None if use_origins: origins = [im.origin for im in images] labels = [im.label for im in images] surfs = ['surf']
'duration': self.duration, 'average speed': self.average_speed, 'top speed': self.top_speed, 'energy': self.energy, 'energy rate': self.energy_rate, 'peak output power': self.peak_output_power, 'average motor power': self.average_motor_power, 'peak regen power': self.peak_regen_power, 'steepest incline': self.steepest_incline, 'steepest decline': self.steepest_decline, } if __name__ == '__main__': if len(sys.argv) > 1: commute = Commute(Car(), sys.argv[1:]) for track in commute.tracks: print 'Track', track.filename print_stats(track.stats) print print 'Total commute' print_stats(commute.stats) else: sys.stderr.write('Usage: %s file.gps [...]\n' % sys.argv[0]) sys.exit(1)