def worker_main(creation_queue, pipe, worker_id): '''The code for a worker. First the worker helps fill the database with players. Then the worker will simulate its players.''' # We need a try block to cleanly exit when ctrl-c is pressed. try: # Setup our reporting db = Database() reporter = Reporter(pipe) # We are responsible for the following range of players. players = range(worker_id * PLAYERS_PER_WORKER, (worker_id + 1) * PLAYERS_PER_WORKER) # Add all our players to the db. for player in players: db.add_player(player) # Mark our worker as done initializing. creation_queue.get() creation_queue.task_done() # Wait for all workers to be done. creation_queue.join() # Now simulate actions. while True: simulate_play(db, random.choice(players)) reporter.mark_event() except KeyboardInterrupt: pass
def testGetConnection(self): report=Reporter(config=reportConfig) res = report.reportGet('[{"data":"get"}]') logging.debug("ok"+str(res))
elif triplet_method == "batch_hardv2": loss_fn = losses.BatchHardTripletLoss_v2(margin=margin, squared=False, soft_margin=soft_margin) elif triplet_method == "batch_all": loss_fn = losses.BatchAllTripletLoss(margin=margin, squared=False, soft_margin=soft_margin) # rt = '/nfs/nas4/marzieh/marzieh/puf/ckpt/batch_hardv2/' # model_filename = rt + 'Run004,modelTriplet,Epoch_345,acc_0.999688.tar' # model_filename = Reporter(ckpt_root=os.path.join(ROOT_DIR, 'ckpt'), # exp=triplet_method, monitor='acc').select_best(run=run_name).selected_ckpt model_filename = Reporter( ckpt_root=os.path.join(ROOT_DIR, 'ckpt'), exp=triplet_method, monitor='acc001').select_best(run='X' + run_name).selected_ckpt print(model_filename) model = models.modelTriplet(embedding_dimension=emb_dim, model_architecture=model_name, pooling=pooling) model.to(device) model.load_state_dict(torch.load(model_filename)['model_state_dict']) print('Evaluating model on test data') batch_all = losses.BatchAllTripletLoss(margin=margin, squared=False, soft_margin=soft_margin) t = 0 nonzeros = 0 triplet_loss_sum = 0
elif triplet_method == "batch_all": loss_fn = losses.BatchAllTripletLoss(margin=margin, squared=False, soft_margin=soft_margin) model = models.modelTriplet(embedding_dimension=emb_dim, model_architecture=model_name, pretrained=False, pooling=pooling) model.to(device) # -------------------------------------------------------------------------------------- # Resume training if start is False # -------------------------------------------------------------------------------------- if not start: reporter = Reporter(ckpt_root=os.path.join(ROOT_DIR, 'ckpt'), exp=triplet_method, monitor='acc') last_model_filename = reporter.select_last(run=run_name).selected_ckpt last_epoch = int(reporter.select_last(run=run_name).last_epoch) loss0 = reporter.select_last(run=run_name).last_loss loss0 = float(loss0[:-4]) model.load_state_dict( torch.load(last_model_filename)['model_state_dict']) else: last_epoch = -1 loss0 = 0 optimizer_model = torch.optim.Adam(model.parameters(), lr=lr) path_ckpt = '{}/ckpt/{}'.format(ROOT_DIR, triplet_method) # learning embedding checkpointer. ckpter = CheckPoint(model=model,
#!/usr/bin/env python3 from utils import Reporter if __name__ == '__main__': reporter = Reporter() reporter.report_test_success()
#!/usr/bin/env python3 from utils import Reporter if __name__ == '__main__': reporter = Reporter() reporter.report_test_failure()
#!/usr/bin/env python3 from utils import Reporter if __name__ == '__main__': reporter = Reporter() reporter.report_release_success()
#!/usr/bin/env python3 from utils import Reporter if __name__ == '__main__': reporter = Reporter() reporter.report_release_failure()
def train(): # Essential train elements image_dataset = GANImageDataset(dataset_path) image_dataiter = DataIterator(dataset=image_dataset, batch_size=8, shuffle=True) net_d = Discriminator().to(device) net_g = Generator().to(device) lr = 1e-4 optimizer_g = torch.optim.Adam(net_d.parameters(), lr=lr, betas=(0, 0.9)) optimizer_d = torch.optim.Adam(net_g.parameters(), lr=lr, betas=(0, 0.9)) one = torch.FloatTensor([1]).to(device) mone = one * -1 # Record items keys_d = ["fake", "real", "gp", "d_loss"] reporter_d = Reporter(keys_d, t="discriminator", interval=1, file_path=None) keys_g = ["g_loss"] reporter_g = Reporter(keys_g, t="generator", interval=1, file_path=None) writer = SummaryWriter() fixed_noise = gen_rand_noise() with torch.no_grad(): fixed_noisev = fixed_noise # Training process for iteration in range(ITERS): # 1. Update netD(discriminator) for p in net_d.parameters(): # reset requires_grad p.requires_grad_( True) # they are set to False below in netG update for i in range(CRITIC_ITERS): with timer("Critic iter: " + str(iteration) + "/" + str(i)): net_d.zero_grad() noise = gen_rand_noise() with torch.no_grad(): noisev = noise fake_data = net_g(noisev).detach() real_data = next(image_dataiter) # Discriminator loss define d_fake = net_d( fake_data).mean() # Expection of discriminating fake data d_real = net_d( real_data).mean() # Expection of discriminating real data gradient_penalty = calc_gradient_penalty( net_d, real_data, fake_data) d_loss = d_fake - d_real + gradient_penalty # Update parameters d_loss.backward(one) optimizer_d.step() # Record "fake", "real", "gp", "d_loss" record_d = [ d_fake.data, d_real.data, gradient_penalty.data, d_loss.data ] record_d = [float(r) for r in record_d] reporter_d.recive(record_d) if i == CRITIC_ITERS - 1: reporter_d.report(iteration) writer.add_scalar("data/d_loss", d_loss, iteration) writer.add_scalar("data/gradient_penalty", gradient_penalty, iteration) # 2. Update netG(generator) for p in net_d.parameters(): p.requires_grad_(False) for i in range(GENER_ITERS): with timer("Generator iters: " + str(iteration) + "/" + str(i)): net_g.zero_grad() noise = gen_rand_noise() noise.requires_grad_(True) fake_data = net_g(noise) # Generator loss define g_loss = net_d(fake_data).mean() # Update parameters g_loss.backward(mone) optimizer_g.step() # Record "g_loss" record_g = [g_loss.data] record_g = [float(r) for r in record_g] reporter_g.recive(record_g) if i == GENER_ITERS - 1: reporter_g.report(iteration) writer.add_scalar("data/d_loss", -g_loss, iteration) # Save intermediate models and images if iteration % 200 == 199: v = (iteration + 1) / 200 gen_images = net_g( fixed_noisev) * 0.5 + 0.5 # Scale from -1 ~ 1 to 0 ~ 1 torchvision.utils.save_image(gen_images, os.path.join( IMAGES_PATH, "samples_{}.png".format(v)), nrow=BATCH_SIZE, padding=2) grid_images = torchvision.utils.make_grid(gen_images, nrow=8, padding=2) writer.add_image('images', grid_images, iteration) torch.save( net_d, os.path.join(MODEL_PATH, "discriminator_{}.pkg".format(v))) torch.save(net_g, os.path.join(MODEL_PATH, "generator_{}.pkg".format(v)))
def extract_features(): torch.multiprocessing.set_start_method('spawn', force=True) ROOT_DIR = os.path.dirname(os.path.abspath(__file__)) device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') with open(r'{}/args.yaml'.format(ROOT_DIR)) as file: args_list = yaml.load(file, Loader=yaml.FullLoader) emb_dim = args_list['emb_dim'] model_name = args_list['model_name'] triplet_method = args_list['triplet_method'] run_name = args_list['run_name'] num_workers = args_list['num_workers'] n_classes_test = args_list['n_classes_test'] n_samples_test = args_list['n_samples_test'] data_type = args_list['data'] parser = argparse.ArgumentParser() parser.add_argument('--triplet_method', '--tm', type=str, default=triplet_method, help='triplet method (default: "batch_hard")') parser.add_argument('--run_name', '--rn', type=str, default=run_name, help='The name for this run (default: "Run01-hardv2")') parser.add_argument('--data_type', '--data', type=str, default=data_type, help='the data source') args = parser.parse_args() triplet_method = args.triplet_method run_name = args.run_name data_type = args.data_type pooling = False with open('{}/dataset.json'.format(ROOT_DIR), 'r') as fp: dataset = json.load(fp) partition = dataset['partition'] labels = dataset['labels'] # 'data_c1','data_c2','data_complex_c1','data_complex_c2','data_resized','data_complex_resized' # 'data_c1v2','data_c2v2','data_complex_c1v2','data_complex_c2v2','data_resizedv2','data_complex_resizedv2' if data_type in [ 'data_resizedv2', 'data_complex_resizedv2', 'data_c1v2', 'data_complex_c1v2', 'data_c2v2', 'data_complex_c2v2' ]: pooling = True n_classes_test = 8 data_x = partition['train'] + partition['validation'] + partition['test'] data_y = {**labels['train'], **labels['validation'], **labels['test']} test_dataset = PairLoader(data_x, data_y, data_source=data_type) test_batch_sampler = BalanceBatchSampler(dataset=test_dataset, n_classes=n_classes_test, n_samples=n_samples_test) test_loader = DataLoader(test_dataset, batch_sampler=test_batch_sampler, num_workers=num_workers) model_filename = Reporter( ckpt_root=os.path.join(ROOT_DIR, 'ckpt'), exp=triplet_method, monitor='acc').select_best(run=run_name).selected_ckpt print(model_filename) model = models.modelTriplet(embedding_dimension=emb_dim, model_architecture=model_name, pooling=pooling) model.to(device) model.load_state_dict(torch.load(model_filename)['model_state_dict']) model.eval() with torch.no_grad(): emb, tg = [], [] for batch_idx, (data, target) in enumerate(test_loader): embedding = model(data).cpu().numpy() target = target.cpu().numpy() emb.append(embedding) tg.append(target) features = {'emb': emb, 'tg': tg} with open('features_1.json', 'w') as f_out: json.dump(features, f_out, cls=NumpyEncoder)