def test_load_cfg(self): """ Check if the two config files are present and loaded with the load_cfg method """ config_file = utils.load_cfg("config.yml") config_quick_debug_file = utils.load_cfg("config_quick_debug.yml") assert config_file and config_quick_debug_file
def main(): cfg = load_cfg() # We're making random actions in selfplay and multiagent so seed the RNGs. seed = cfg["Environment"]["Random_seed"] random.seed(seed) np.random.seed(seed) self_play = cfg["Training"]["Self_play"] if self_play: scores = selfplay(cfg) else: scores = multiagent(cfg)
def run(): config = utils.load_cfg('conf.cfg') template_path = config.get('image', 'filename') markers = {'school': config.getint('markers', 'school'), 'year': config.getint('markers', 'year'), 'level': config.getint('markers', 'level'), 'award': config.getint('markers', 'award'), 'recipient': config.getint('markers', 'recipient')} font = {'color': config.get('font', 'color'), 'name': config.get('font', 'name'), 'size': config.getint('font', 'size')} images_per_pdf = config.getint('pdf', 'images_per_pdf') if images_per_pdf > 6: # 6 is the maximum allowed number of images per pdf exit() csv_file = config.get('csv', 'filename') cc = CsvReader(csv_file) csv_data = cc.read() school = 'John Scottus School' year = '2016' image_folder = utils.create_folder('images') pdf_folder = utils.create_folder('pdfs') count = 0 images = [] draw_tool = ImageWriter(font) for row in csv_data: im = draw_tool.open_image(template_path) im = draw_tool.write_text(im, markers['school'], school) im = draw_tool.write_text(im, markers['year'], year) im = draw_tool.write_text(im, markers['level'], row['Level']) im = draw_tool.write_text(im, markers['award'], row['Award']) im = draw_tool.write_text(im, markers['recipient'], row['Recipient']) im_path = image_folder + '/' + 'image_' + utils.timestamp() + '.png' draw_tool.save_image(im, im_path) count += 1 images.append(im_path) if count % images_per_pdf == 0: create_pdf(pdf_folder, images) count = 0 images = [] create_pdf(pdf_folder, images)
def get_config_from_file(args): if args.file: logger.info(f"Using provided config file: {args.file}") config_filepath = args.file else: logger.info(f"Using default config file: {DEFAULT_CONFIG_FILE}") config_filepath = DEFAULT_CONFIG_FILE config = utils.load_cfg(config_filepath) config = utils.init_result_folder(config_filepath, config) return config
## Functions to Load, Preprocessing, and Handling Data. ## All the functions here need to be aproved by and documented by notebooks Delivered-*.ipynb import pandas as pd import os, glob from utils import load_cfg #In order to use yaml confg file, we'll make this functions. config = load_cfg('conf.yaml') conf_preproc = config['Preprocessing'] def read_train_data(): """ Function to read and concat training data that is split across multiples csv. """ print(os.getcwd()) os.chdir("project-template/input") print(os.getcwd()) extension = 'csv' train_filenames = [ i for i in glob.glob('*.{}'.format(extension)) if "train" in i ] concat_df = pd.concat([pd.read_csv(file) for file in train_filenames]) return concat_df def preprocessing_data(df): """ A simple preprocessing function. """
args = parser.parse_args() return args.d, args.c or 'config.json' if __name__ == '__main__': date_str, cfg_file = init_args() # 如果不给日期,视为前一天 if not date_str: date_str = datetime.date.today() - datetime.timedelta(days=1) date_str = date_str.strftime('%Y%m%d') _commands = cfg.get("job", {}).get("commands", []) commands = [ _command.format(date_str) if date_str else _command.format('') for _command in _commands ] global_cfg = load_cfg(cfg_file) if not global_cfg: sys.exit(0) servers = global_cfg.get("servers", []) _servers = [(server.split(' ')[0], server.split(' ')[1], server.split(' ')[2]) for server in servers if len(server.split(' ')) == 3] servers = _servers if _servers else servers if not commands or not servers: logger.error("miss required configuration") sys.exit(0) if not date_str: date_str = datetime.date.today() - datetime.timedelta(days=1) vv = count(commands[0], servers)
import numpy as np # import copy from model import Actor, Critic import torch import torch.nn as nn import torch.nn.functional as F import torch.optim as optim from model import Actor, Critic from noise import OUNoise from replay_buffer import ReplayBuffer from utils import load_cfg, device cfg = load_cfg() # Define global configuration variables BUFFER_SIZE = cfg["Agent"]["Buffer_size"] BATCH_SIZE = cfg["Agent"]["Batch_size"] GAMMA = cfg["Agent"]["Gamma"] TAU = cfg["Agent"]["Tau"] LR_ACTOR = cfg["Agent"]["Lr_actor"] LR_CRITIC = cfg["Agent"]["Lr_critic"] WEIGHT_DECAY = cfg["Agent"]["Weight_decay"] NOISE_DECAY = cfg["Agent"]["Noise_decay"] UPDATE_EVERY = cfg["Agent"]["Update_every"] LEARNING_STEPS = cfg["Agent"]["Learning_steps"] NORM_REWARDS = cfg["Agent"]["Norm_rewards"]
parser = argparse.ArgumentParser() parser.add_argument('--cfg', type=str, default='', help="path to a cfg file") parser.add_argument( '--model_name', default='tmp', help='Weights and samples will be saved under experiments/model_name') parser.add_argument("--save_iter", type=int, default=1000, help="interval between image sampling") parser.add_argument("--epochs", type=int, default=15, help="number of epochs of training") parser.add_argument("--batch_size", type=int, default=16, help="size of the batches") args = parser.parse_args() # get cfg cfg = load_cfg(args.cfg) if args.cfg else get_cfg_defaults() # add additional arguments in the argparser and in the function below cfg = merge_args_and_cfg(args, cfg) print(cfg) main(cfg)
def test_load_cfg(self): config = utils.load_cfg('test/conf.cfg') self.assertIsNotNone(config)
parser.add_argument('-f', help='filename e.g.tomcat1-httpClient.log', required=True) parser.add_argument('-c', help='config file') args = parser.parse_args() return args.k, args.f, args.c or 'config.json' def count(command, servers): job = Count(command, Dispatcher(servers)) job.do() return job.occurrences if __name__ == '__main__': keyword, log_file, cfg_file = init_args() cfg = load_cfg(cfg_file) if not cfg: sys.exit(0) servers = cfg.get("servers", []) _servers = [(server.split(' ')[0], server.split(' ')[1], server.split(' ')[2]) for server in servers if len(server.split(' ')) == 3] servers = _servers if _servers else servers if not servers: logger.error("miss required configuration") sys.exit(0) command = "zgrep -c '{0}' {1}".format(keyword, log_file) job = Count(command, Dispatcher(servers)) job.do()
def selfplay(cfg): cfg = load_cfg() db = get_db() env, agent = get_agent_unity(cfg) # agent = load_pretrained(agent) return ddpg_selfplay(env, agent, cfg, db)
def multiagent(cfg): cfg = load_cfg() db = get_db() env, agent1, agent2 = get_agents_unity(cfg) return ddpg_multiagent(env, agent1, agent2, cfg, db)
def main(yaml_filepath): cfg = u.load_cfg(yaml_filepath) pp = pprint.PrettyPrinter(indent=2) pp.pprint(cfg) model_name = cfg['model']['name'] logger = logging.getLogger(model_name) log_path = cfg['training']['artifacts_path'] + '/' + \ cfg['dataset']['name'] + '/' + model_name + '/' + \ str(datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')).replace(' ', '/') + '/' print(log_path) if not os.path.exists(log_path): os.makedirs(log_path) os.makedirs(log_path + 'save/') copyfile(yaml_filepath, log_path + os.path.basename(os.path.normpath(yaml_filepath))) hdlr = logging.FileHandler(log_path + model_name + '.log') formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s') hdlr.setFormatter(formatter) logger.addHandler(hdlr) logger.setLevel(logging.INFO) logger.info("Loading data...") data_path = cfg['dataset']['data_path'] + '/' areas_tr = cfg['training']['areas'] areas_va = cfg['validation']['areas'] label_to_idx = { '<UNK>': 0, 'beam': 1, 'board': 2, 'bookcase': 3, 'ceiling': 4, 'chair': 5, 'clutter': 6, 'column': 7, 'door': 8, 'floor': 9, 'sofa': 10, 'table': 11, 'wall': 12, 'window': 13 } idx_to_label = { 0: '<UNK>', 1: 'beam', 2: 'board', 3: 'bookcase', 4: 'ceiling', 5: 'chair', 6: 'clutter', 7: 'column', 8: 'door', 9: 'floor', 10: 'sofa', 11: 'table', 12: 'wall', 13: 'window' } batch_size_tr = int(cfg['training']['batch_size']) batch_size_va = int(cfg['validation']['batch_size']) workers_tr = int(cfg['training']['num_workers']) workers_va = int(cfg['validation']['num_workers']) rate_tr = float(cfg['training']['rate']) rate_va = float(cfg['validation']['rate']) flip_prob = float(cfg['data_augmentation']['flip_prob']) crop_size = int(cfg['data_augmentation']['crop_size']) dataset_module = importlib.import_module(cfg['dataset']['module_name'], cfg['dataset']['script_path']) dataset_tr = dataset_module.Dataset(data_path, areas_tr, rate=rate_tr, flip_prob=flip_prob, crop_type='Random', crop_size=crop_size) dataloader_tr = DataLoader(dataset_tr, batch_size=batch_size_tr, shuffle=True, num_workers=workers_tr, drop_last=False, pin_memory=True) dataset_va = dataset_module.Dataset(data_path, areas_va, rate=rate_va, flip_prob=0.0, crop_type='Center', crop_size=crop_size) dataloader_va = DataLoader(dataset_va, batch_size=batch_size_va, shuffle=False, num_workers=workers_va, drop_last=False, pin_memory=True) cv2.setNumThreads( workers_tr ) # Necessary for num_workers > 0 in DataLoader, otherwise freeze logger.info("Preparing model...") # without beam and column: [0.,0.,1.,1.,1.,1.,1.,0.,1.,1.,1.,1.,1.,1.] class_weights = cfg['training']['class_weights'] nclasses = len(class_weights) num_epochs = int(cfg['training']['epochs']) use_gnn = bool(cfg['model']['use_gnn']) gnn_iterations = int(cfg['gnn']['iterations']) gnn_k = int(cfg['gnn']['k']) mlp_num_layers = int(cfg['gnn']['mlp_num_layers']) model_module = importlib.import_module(cfg['model']['module_name'], cfg['model']['script_path']) use_half_precision = bool(cfg['model']['use_half_precision']) model = model_module.Model(nclasses, mlp_num_layers) if use_half_precision: model.half() for layer in model.modules(): if isinstance(layer, nn.BatchNorm2d): layer.float() use_bootstrap_loss = bool(cfg['loss']['use_bootstrap_loss']) bootstrap_rate = float(cfg['loss']['bootstrap_rate']) loss = nn.NLLLoss(reduce=not use_bootstrap_loss, weight=torch.FloatTensor(class_weights)) softmax = nn.Softmax(dim=1) log_softmax = nn.LogSoftmax(dim=1) model.cuda() loss.cuda() softmax.cuda() log_softmax.cuda() opt_mode = 'half_precision_optimizer' if use_half_precision else 'single_precision_optimizer' base_initial_lr = float((cfg[opt_mode]['base_initial_lr'])) gnn_initial_lr = float(cfg[opt_mode]['gnn_initial_lr']) betas = cfg[opt_mode]['betas'] eps = float(cfg[opt_mode]['eps']) weight_decay = float(cfg[opt_mode]['weight_decay']) amsgrad = bool(cfg[opt_mode]['amsgrad']) lr_schedule_type = cfg['schedule']['lr_schedule_type'] lr_decay = float(cfg['schedule']['lr_decay']) lr_patience = int(cfg['schedule']['lr_patience']) # optimizer = torch.optim.Adam(model.parameters(), lr=initial_lr, betas=betas, eps=eps, # weight_decay=weight_decay, amsgrad=amsgrad) optimizer = torch.optim.Adam([{ 'params': model.decoder.parameters() }, { 'params': model.gnn.parameters(), 'lr': gnn_initial_lr }], lr=base_initial_lr, betas=betas, eps=eps, weight_decay=weight_decay, amsgrad=amsgrad) if lr_schedule_type == 'exp': lambda1 = lambda epoch: pow((1 - ((epoch - 1) / num_epochs)), lr_decay) scheduler = torch.optim.lr_scheduler.LambdaLR(optimizer, lr_lambda=lambda1) elif lr_schedule_type == 'plateau': scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau( optimizer, factor=lr_decay, patience=lr_patience) else: print('bad scheduler') exit(1) model_parameters = filter(lambda p: p.requires_grad, model.parameters()) params = sum([np.prod(p.size()) for p in model_parameters]) logger.info("Number of trainable parameters: %d", params) def get_current_learning_rates(): learning_rates = [] for param_group in optimizer.param_groups: learning_rates.append(param_group['lr']) return learning_rates def eval_set(dataloader): model.eval() with torch.no_grad(): loss_sum = 0.0 confusion_matrix = torch.cuda.FloatTensor(np.zeros(14**2)) start_time = time.time() for batch_idx, rgbd_label_xy in enumerate(dataloader): sys.stdout.write('\rEvaluating test set... {}/{}'.format( batch_idx + 1, len(dataloader))) x = rgbd_label_xy[0].cuda(async=True) xy = rgbd_label_xy[2].cuda(async=True) if use_half_precision: x = x.half() xy = xy.half() else: x = x.float() xy = xy.float() input = x.permute(0, 3, 1, 2).contiguous() xy = xy.permute(0, 3, 1, 2).contiguous() target = rgbd_label_xy[1].cuda(async=True).long() output = model(input, gnn_iterations=gnn_iterations, k=gnn_k, xy=xy, use_gnn=use_gnn, use_half_precision=use_half_precision) if use_bootstrap_loss: loss_per_pixel = loss.forward(log_softmax(output.float()), target) topk, indices = torch.topk( loss_per_pixel.view(output.size()[0], -1), int((crop_size**2) * bootstrap_rate)) loss_ = torch.mean(topk) else: loss_ = loss.forward(log_softmax(output.float()), target) loss_sum += loss_ pred = output.permute(0, 2, 3, 1).contiguous() pred = pred.view(-1, nclasses) pred = softmax(pred) pred_max_val, pred_arg_max = pred.max(1) pairs = target.view(-1) * 14 + pred_arg_max.view(-1) for i in range(14**2): cumu = pairs.eq(i).float().sum() confusion_matrix[i] += cumu.item() sys.stdout.write(" - Eval time: {:.2f}s \n".format(time.time() - start_time)) loss_sum /= len(dataloader) confusion_matrix = confusion_matrix.cpu().numpy().reshape((14, 14)) class_iou = np.zeros(14) # we ignore void values confusion_matrix[0, :] = np.zeros(14) confusion_matrix[:, 0] = np.zeros(14) for i in range(1, 14): class_iou[i] = confusion_matrix[i, i] / ( np.sum(confusion_matrix[i, :]) + np.sum(confusion_matrix[:, i]) - confusion_matrix[i, i]) return loss_sum.item(), class_iou, confusion_matrix eval_loss = None class_iou = None confusion_matrix = None model_to_load = None logger.info("num_epochs: %d", num_epochs) batch_loss_interval = cfg['training']['batch_loss_interval'] train_losses = [] eval_losses = [] if model_to_load: logger.info("Loading old model...") model.load_state_dict(torch.load(model_to_load)) else: logger.info("Starting training from scratch...") for epoch in range(1, num_epochs + 1): batch_loss_avg = 0 if lr_schedule_type == 'exp': scheduler.step(epoch) for batch_idx, rgbd_label_xy in enumerate( tqdm(dataloader_tr, smoothing=0.99)): x = rgbd_label_xy[0].cuda(async=True) xy = rgbd_label_xy[2].cuda(async=True) if use_half_precision: x = x.half() xy = xy.half() else: x = x.float() xy = xy.float() input = x.permute(0, 3, 1, 2).contiguous() xy = xy.permute(0, 3, 1, 2).contiguous() target = rgbd_label_xy[1].cuda(async=True).long() optimizer.zero_grad() model.train() output = model(input, gnn_iterations=gnn_iterations, k=gnn_k, xy=xy, use_gnn=use_gnn, use_half_precision=use_half_precision) if use_bootstrap_loss: loss_per_pixel = loss.forward(log_softmax(output.float()), target) topk, indices = torch.topk( loss_per_pixel.view(output.size()[0], -1), int((crop_size**2) * bootstrap_rate)) loss_ = torch.mean(topk) else: loss_ = loss.forward(log_softmax(output.float()), target) loss_.backward() optimizer.step() batch_loss_avg += loss_.item() if batch_idx % batch_loss_interval == 0 and batch_idx > 0: batch_loss_avg /= batch_loss_interval train_losses.append(batch_loss_avg) logger.info("e%db%d Batch loss average: %s", epoch, batch_idx, batch_loss_avg) batch_loss_avg = 0 batch_idx = len(dataloader_tr) logger.info("e%db%d Saving model...", epoch, batch_idx) torch.save( model.state_dict(), log_path + '/save/' + model_name + '_' + str(epoch) + '_' + str(batch_idx) + '.pth') eval_loss, class_iou, confusion_matrix = eval_set(dataloader_va) eval_losses.append(eval_loss) if lr_schedule_type == 'plateau': scheduler.step(eval_loss) logger.info("e%db%d Def learning rate: %s", epoch, batch_idx, get_current_learning_rates()[0]) logger.info("e%db%d GNN learning rate: %s", epoch, batch_idx, get_current_learning_rates()[1]) logger.info("e%db%d Eval loss: %s", epoch, batch_idx, eval_loss) logger.info("e%db%d Class IoU:", epoch, batch_idx) for cl in range(14): logger.info("%+10s: %-10s" % (idx_to_label[cl], class_iou[cl])) logger.info("Mean IoU: %s", np.mean(class_iou[1:])) logger.info("e%db%d Confusion matrix:", epoch, batch_idx) logger.info(confusion_matrix)