def main(): # Establish gmail connection creds = None # The file token.pickle stores the user's access and refresh tokens, and is # created automatically when the authorization flow completes for the first # time. if os.path.exists('token.pickle'): with open('token.pickle', 'rb') as token: creds = pickle.load(token) # If there are no (valid) credentials available, let the user log in. if not creds or not creds.valid: if creds and creds.expired and creds.refresh_token: creds.refresh(Request()) else: flow = InstalledAppFlow.from_client_secrets_file( 'credentials.json', SCOPES) creds = flow.run_local_server(port=0) # Save the credentials for the next run with open('token.pickle', 'wb') as token: pickle.dump(creds, token) service = build('gmail', 'v1', credentials=creds) print('{} Establishing params'.format(getTime())) params = set_params() params['service'] = service params['user_id'] = 'me' printGreen('{} Successfully established params'.format(getTime())) listener = HaroListener(params) listener.messageLoop()
def main(): """Main workflow""" args = utils.build_args(argparse.ArgumentParser()) utils.init_logger(args.model_file) assert torch.cuda.is_available() torch.cuda.set_device(args.gpuid) utils.init_random(args.seed) utils.set_params(args) logger.info("Config:\n%s", pformat(vars(args))) fields = utils.build_fields() logger.info("Fields: %s", fields.keys()) logger.info("Load %s", args.train_file) train_data = LMDataset(fields, args.train_file, args.sent_length_trunc) logger.info("Training sentences: %d", len(train_data)) logger.info("Load %s", args.valid_file) val_data = LMDataset(fields, args.valid_file, args.sent_length_trunc) logger.info("Validation sentences: %d", len(val_data)) fields["sent"].build_vocab(train_data) train_iter = utils.build_dataset_iter(train_data, args) val_iter = utils.build_dataset_iter(val_data, args, train=False) if args.resume and os.path.isfile(args.checkpoint_file): logger.info("Resume training") logger.info("Load checkpoint %s", args.checkpoint_file) checkpoint = torch.load(args.checkpoint_file, map_location=lambda storage, loc: storage) es_stats = checkpoint["es_stats"] args = utils.set_args(args, checkpoint) else: checkpoint = None es_stats = ESStatistics(args) model = utils.build_model(fields, args, checkpoint) logger.info("Model:\n%s", model) optimizer = utils.build_optimizer(model, args, checkpoint) try_train_val(fields, model, optimizer, train_iter, val_iter, es_stats, args)
def main() -> None: auth_response = send_auth_request(GRID_ADDRESS, NAME, VERSION) worker_id = auth_response["data"]["worker_id"] cycle_response = send_cycle_request(GRID_ADDRESS, NAME, VERSION, worker_id) request_key = cycle_response["data"]["request_key"] model_id = cycle_response["data"]["model_id"] client_config = cycle_response["data"]["client_config"] alpha = client_config["alpha"] gamma = client_config["gamma"] min_epsilon = client_config["min_epsilon"] epsilon_reduction = client_config["epsilon_reduction"] n_train_iterations = client_config["n_train_iterations"] n_test_iterations = client_config["n_test_iterations"] downloaded_params = get_model_params(GRID_ADDRESS, worker_id, request_key, model_id) local_agent = QLearningAgent( input_width=INPUT_WIDTH, output_width=OUTPUT_WIDTH, hidden_width=HIDDEN_WIDTH, alpha=alpha, gamma=gamma, min_epsilon=min_epsilon, epsilon_reduction=epsilon_reduction, ) set_params(local_agent, downloaded_params) _, pre_rets = run_epoch(n_test_iterations, local_agent, train=False) print(f"Pre-training performance: {sum(pre_rets) / n_test_iterations}") trained_params, _ = run_epoch(n_train_iterations, local_agent, train=True) _, post_rets = run_epoch(n_test_iterations, local_agent, train=False) print(f"Post-training performance: {sum(post_rets) / n_test_iterations}") diff = calculate_diff(downloaded_params, trained_params) send_diff_report(GRID_ADDRESS, worker_id, request_key, diff) new_model_params = retrieve_model_params(GRID_ADDRESS, NAME, VERSION) set_params(local_agent, new_model_params) _, updated_rets = run_epoch(n_test_iterations, local_agent, train=False) print( f"Updated model performance: {sum(updated_rets) / n_test_iterations}")
def main(): """Main workflow""" args = utils.build_gen_args(argparse.ArgumentParser()) utils.init_logger(args.out_file) logger.info("Config:\n%s", pformat(vars(args))) assert torch.cuda.is_available() torch.cuda.set_device(args.gpuid) utils.init_random(args.seed) logger.info("Load parameters from '%s'", args.model_file) params = torch.load(args.model_file, map_location=lambda storage, loc: storage) utils.set_params(params["args"]) fields = utils.load_fields_from_vocab(params["vocab"]) logger.info("Fields: %s", fields.keys()) model = utils.build_test_model(fields, params) sent_idx = [i for i in range(args.num_sentences)] num_batches = math.ceil(float(args.num_sentences) / args.batch_size) samples = [] with torch.no_grad(): for i in range(num_batches): running_batch_size = len( sent_idx[i * args.batch_size : (i + 1) * args.batch_size] ) samples.append( model.generate( running_batch_size, args.max_sent_length, args.temperature ) ) samples = torch.cat(samples, 0) save(samples, fields, args.out_file)
def run(mode): # modeの選択 → hra+1 valid_modes = ['dqn', 'dqn+1', 'hra', 'hra+1', 'all'] assert mode in valid_modes if mode in ['all']: modes = valid_modes[:-1] else: modes = [mode] dir_path = os.path.dirname(os.path.realpath(__file__)) cfg_file = os.path.join(dir_path, 'config.yaml') params = yaml.safe_load(open(cfg_file, 'r')) for m in modes: params = set_params(params, m) worker(params) # if __name__ == '__main__': # run(mode='hra+1')
def run(mode, options): valid_modes = ['dqn', 'dqn+1', 'hra', 'hra+1', 'all'] assert mode in valid_modes if mode in ['all']: modes = valid_modes[:-1] else: modes = [mode] dir_path = os.path.dirname(os.path.realpath(__file__)) cfg_file = os.path.join(dir_path, 'config.yaml') params = yaml.safe_load(open(cfg_file, 'r')) # replacing params with command line options for opt in options: assert opt[0] in params dtype = type(params[opt[0]]) if dtype == bool: new_opt = False if opt[1] != 'True' else True else: new_opt = dtype(opt[1]) params[opt[0]] = new_opt for m in modes: params = set_params(params, m) worker(params)
create_metric_holders, wg_eval_acc_metrics_update_i, \ sg_eval_acc_metrics_update_i, save_eval_metrics randseed = int(time.time()) print("random seed: ", randseed) random.seed(randseed) npr.seed(randseed) ed.set_seed(randseed) tf.set_random_seed(randseed) if __name__ == '__main__': parser = create_argparser() args = parser.parse_args() all_params = set_params(args) DATA_DIR, CAUSEFIT_DIR, OUT_DATA_DIR, \ outdim, caudim, thold, M, n_iter, binary, \ pri_U, pri_V, alpha = all_params print("setting params....") print("data/cause/out directories", DATA_DIR, CAUSEFIT_DIR, OUT_DATA_DIR) print("relevance thold", thold) print("batch size", M, "n_iter", n_iter) print("outdim", outdim) print("caudim", caudim) print("prior sd on U", pri_U, "prior sd on V", pri_V) print("alpha", alpha) unique_uid = list() with open(os.path.join(DATA_DIR, 'unique_uid.txt'), 'r') as f:
def set_params(self): utils.set_params(self.params_file, self.p) self.p = utils.get_params(self.params_file)
def main(): """Main workflow""" args = utils.build_test_args(argparse.ArgumentParser()) suff = ".test" if args.report_iw_nll: if ( args.num_iw_samples > args.iw_batch_size and args.num_iw_samples % args.iw_batch_size != 0 ): raise RuntimeError("Expected num_iw_samples divisible by iw_batch_size") suff = ".test.iw" + str(args.num_iw_samples) utils.init_logger(args.model_file + suff) logger.info("Config:\n%s", pformat(vars(args))) assert torch.cuda.is_available() torch.cuda.set_device(args.gpuid) utils.init_random(args.seed) logger.info("Load parameters from '%s'", args.model_file) params = torch.load(args.model_file, map_location=lambda storage, loc: storage) utils.set_params(params["args"]) fields = utils.load_fields_from_vocab(params["vocab"]) logger.info("Fields: %s", fields.keys()) model = utils.build_test_model(fields, params) logger.info("Model:\n%s", model) logger.info("Load %s", args.test_file) test_data = LMDataset(fields, args.test_file, args.sent_length_trunc) logger.info("Test sentences: %d", len(test_data)) test_iter = utils.OrderedIterator( dataset=test_data, batch_size=args.batch_size, device=params["args"].device, train=False, shuffle=False, repeat=False, sort=False, sort_within_batch=True, ) if model.encoder is None: args.report_iw_nll = False logger.info("Force report_iw_nll to False") start_time = time.time() logger.info("Start testing") if args.report_iw_nll: if args.num_iw_samples <= args.iw_batch_size: n_iw_iter = 1 else: n_iw_iter = args.num_iw_samples // args.iw_batch_size args.num_iw_samples = args.iw_batch_size test_stats = report_iw_nll(model, test_iter, n_iw_iter, args.num_iw_samples) logger.info( "Results: test nll %.2f | test ppl %.2f", test_stats.nll(), test_stats.ppl() ) else: test_stats = validate(model, test_iter) logger.info( "Results: test nll %.2f | test kl %.2f | test ppl %.2f", test_stats.nll(), test_stats.kl(), test_stats.ppl(), ) logger.info("End of testing: time %.1f min", (time.time() - start_time) / 60)
import argparse from utils import set_params, set_download_dir, format_datetime, scrape_images, extract_average_color, classify_images, df_to_csv, df_to_excel, label_images if __name__ == "__main__": parser = argparse.ArgumentParser() profile, num, startdate, enddate, mode, season = set_params(parser) #Output directory for Scraped Images output = set_download_dir(startdate, enddate) format1 = '%Y-%m-%d' startdate, enddate = format_datetime(format1, startdate, enddate) # Mode for scraping images and saving them in output folder if mode == '0': print("starting download.....") scrape_images(profile, num, startdate, enddate, output) print("Download Complete!") # Mode for detecting color and exporting dataframe as .csv elif mode == '1': classify_images(output, '.jpg','color') # Mode for labelling the images elif mode == '2': label_images(output, '.jpg', season) #elif mode == '3':