def test_load_cfg(self):
"""
Check if the two config files are present
and loaded with the load_cfg method
"""
config_file = utils.load_cfg("config.yml")
config_quick_debug_file = utils.load_cfg("config_quick_debug.yml")
assert config_file and config_quick_debug_file
def main():
cfg = load_cfg()
# We're making random actions in selfplay and multiagent so seed the RNGs.
seed = cfg["Environment"]["Random_seed"]
random.seed(seed)
np.random.seed(seed)
self_play = cfg["Training"]["Self_play"]
if self_play:
scores = selfplay(cfg)
else:
scores = multiagent(cfg)
def run():
config = utils.load_cfg('conf.cfg')
template_path = config.get('image', 'filename')
markers = {'school': config.getint('markers', 'school'),
'year': config.getint('markers', 'year'),
'level': config.getint('markers', 'level'),
'award': config.getint('markers', 'award'),
'recipient': config.getint('markers', 'recipient')}
font = {'color': config.get('font', 'color'),
'name': config.get('font', 'name'),
'size': config.getint('font', 'size')}
images_per_pdf = config.getint('pdf', 'images_per_pdf')
if images_per_pdf > 6:
# 6 is the maximum allowed number of images per pdf
exit()
csv_file = config.get('csv', 'filename')
cc = CsvReader(csv_file)
csv_data = cc.read()
school = 'John Scottus School'
year = '2016'
image_folder = utils.create_folder('images')
pdf_folder = utils.create_folder('pdfs')
count = 0
images = []
draw_tool = ImageWriter(font)
for row in csv_data:
im = draw_tool.open_image(template_path)
im = draw_tool.write_text(im, markers['school'], school)
im = draw_tool.write_text(im, markers['year'], year)
im = draw_tool.write_text(im, markers['level'], row['Level'])
im = draw_tool.write_text(im, markers['award'], row['Award'])
im = draw_tool.write_text(im, markers['recipient'], row['Recipient'])
im_path = image_folder + '/' + 'image_' + utils.timestamp() + '.png'
draw_tool.save_image(im, im_path)
count += 1
images.append(im_path)
if count % images_per_pdf == 0:
create_pdf(pdf_folder, images)
count = 0
images = []
create_pdf(pdf_folder, images)
def get_config_from_file(args):
if args.file:
logger.info(f"Using provided config file: {args.file}")
config_filepath = args.file
else:
logger.info(f"Using default config file: {DEFAULT_CONFIG_FILE}")
config_filepath = DEFAULT_CONFIG_FILE
config = utils.load_cfg(config_filepath)
config = utils.init_result_folder(config_filepath, config)
return config
## Functions to Load, Preprocessing, and Handling Data.
## All the functions here need to be aproved by and documented by notebooks Delivered-*.ipynb
import pandas as pd
import os, glob
from utils import load_cfg
#In order to use yaml confg file, we'll make this functions.
config = load_cfg('conf.yaml')
conf_preproc = config['Preprocessing']
def read_train_data():
"""
Function to read and concat training data that is split across multiples csv.
"""
print(os.getcwd())
os.chdir("project-template/input")
print(os.getcwd())
extension = 'csv'
train_filenames = [
i for i in glob.glob('*.{}'.format(extension)) if "train" in i
]
concat_df = pd.concat([pd.read_csv(file) for file in train_filenames])
return concat_df
def preprocessing_data(df):
"""
A simple preprocessing function.
"""
args = parser.parse_args()
return args.d, args.c or 'config.json'
if __name__ == '__main__':
date_str, cfg_file = init_args()
# 如果不给日期，视为前一天
if not date_str:
date_str = datetime.date.today() - datetime.timedelta(days=1)
date_str = date_str.strftime('%Y%m%d')
_commands = cfg.get("job", {}).get("commands", [])
commands = [
_command.format(date_str) if date_str else _command.format('')
for _command in _commands
]
global_cfg = load_cfg(cfg_file)
if not global_cfg:
sys.exit(0)
servers = global_cfg.get("servers", [])
_servers = [(server.split(' ')[0], server.split(' ')[1],
server.split(' ')[2]) for server in servers
if len(server.split(' ')) == 3]
servers = _servers if _servers else servers
if not commands or not servers:
logger.error("miss required configuration")
sys.exit(0)
if not date_str:
date_str = datetime.date.today() - datetime.timedelta(days=1)
vv = count(commands[0], servers)
import numpy as np # import copy from model import Actor, Critic import torch import torch.nn as nn import torch.nn.functional as F import torch.optim as optim from model import Actor, Critic from noise import OUNoise from replay_buffer import ReplayBuffer from utils import load_cfg, device cfg = load_cfg() # Define global configuration variables BUFFER_SIZE = cfg["Agent"]["Buffer_size"] BATCH_SIZE = cfg["Agent"]["Batch_size"] GAMMA = cfg["Agent"]["Gamma"] TAU = cfg["Agent"]["Tau"] LR_ACTOR = cfg["Agent"]["Lr_actor"] LR_CRITIC = cfg["Agent"]["Lr_critic"] WEIGHT_DECAY = cfg["Agent"]["Weight_decay"] NOISE_DECAY = cfg["Agent"]["Noise_decay"] UPDATE_EVERY = cfg["Agent"]["Update_every"] LEARNING_STEPS = cfg["Agent"]["Learning_steps"] NORM_REWARDS = cfg["Agent"]["Norm_rewards"]
parser = argparse.ArgumentParser()
parser.add_argument('--cfg',
type=str,
default='',
help="path to a cfg file")
parser.add_argument(
'--model_name',
default='tmp',
help='Weights and samples will be saved under experiments/model_name')
parser.add_argument("--save_iter",
type=int,
default=1000,
help="interval between image sampling")
parser.add_argument("--epochs",
type=int,
default=15,
help="number of epochs of training")
parser.add_argument("--batch_size",
type=int,
default=16,
help="size of the batches")
args = parser.parse_args()
# get cfg
cfg = load_cfg(args.cfg) if args.cfg else get_cfg_defaults()
# add additional arguments in the argparser and in the function below
cfg = merge_args_and_cfg(args, cfg)
print(cfg)
main(cfg)
def test_load_cfg(self):
config = utils.load_cfg('test/conf.cfg')
self.assertIsNotNone(config)
parser.add_argument('-f',
help='filename e.g.tomcat1-httpClient.log',
required=True)
parser.add_argument('-c', help='config file')
args = parser.parse_args()
return args.k, args.f, args.c or 'config.json'
def count(command, servers):
job = Count(command, Dispatcher(servers))
job.do()
return job.occurrences
if __name__ == '__main__':
keyword, log_file, cfg_file = init_args()
cfg = load_cfg(cfg_file)
if not cfg:
sys.exit(0)
servers = cfg.get("servers", [])
_servers = [(server.split(' ')[0], server.split(' ')[1],
server.split(' ')[2]) for server in servers
if len(server.split(' ')) == 3]
servers = _servers if _servers else servers
if not servers:
logger.error("miss required configuration")
sys.exit(0)
command = "zgrep -c '{0}' {1}".format(keyword, log_file)
job = Count(command, Dispatcher(servers))
job.do()
def selfplay(cfg):
cfg = load_cfg()
db = get_db()
env, agent = get_agent_unity(cfg)
# agent = load_pretrained(agent)
return ddpg_selfplay(env, agent, cfg, db)
def multiagent(cfg):
cfg = load_cfg()
db = get_db()
env, agent1, agent2 = get_agents_unity(cfg)
return ddpg_multiagent(env, agent1, agent2, cfg, db)
def main(yaml_filepath):
cfg = u.load_cfg(yaml_filepath)
pp = pprint.PrettyPrinter(indent=2)
pp.pprint(cfg)
model_name = cfg['model']['name']
logger = logging.getLogger(model_name)
log_path = cfg['training']['artifacts_path'] + '/' + \
cfg['dataset']['name'] + '/' + model_name + '/' + \
str(datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')).replace(' ', '/') + '/'
print(log_path)
if not os.path.exists(log_path):
os.makedirs(log_path)
os.makedirs(log_path + 'save/')
copyfile(yaml_filepath,
log_path + os.path.basename(os.path.normpath(yaml_filepath)))
hdlr = logging.FileHandler(log_path + model_name + '.log')
formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s')
hdlr.setFormatter(formatter)
logger.addHandler(hdlr)
logger.setLevel(logging.INFO)
logger.info("Loading data...")
data_path = cfg['dataset']['data_path'] + '/'
areas_tr = cfg['training']['areas']
areas_va = cfg['validation']['areas']
label_to_idx = {
'<UNK>': 0,
'beam': 1,
'board': 2,
'bookcase': 3,
'ceiling': 4,
'chair': 5,
'clutter': 6,
'column': 7,
'door': 8,
'floor': 9,
'sofa': 10,
'table': 11,
'wall': 12,
'window': 13
}
idx_to_label = {
0: '<UNK>',
1: 'beam',
2: 'board',
3: 'bookcase',
4: 'ceiling',
5: 'chair',
6: 'clutter',
7: 'column',
8: 'door',
9: 'floor',
10: 'sofa',
11: 'table',
12: 'wall',
13: 'window'
}
batch_size_tr = int(cfg['training']['batch_size'])
batch_size_va = int(cfg['validation']['batch_size'])
workers_tr = int(cfg['training']['num_workers'])
workers_va = int(cfg['validation']['num_workers'])
rate_tr = float(cfg['training']['rate'])
rate_va = float(cfg['validation']['rate'])
flip_prob = float(cfg['data_augmentation']['flip_prob'])
crop_size = int(cfg['data_augmentation']['crop_size'])
dataset_module = importlib.import_module(cfg['dataset']['module_name'],
cfg['dataset']['script_path'])
dataset_tr = dataset_module.Dataset(data_path,
areas_tr,
rate=rate_tr,
flip_prob=flip_prob,
crop_type='Random',
crop_size=crop_size)
dataloader_tr = DataLoader(dataset_tr,
batch_size=batch_size_tr,
shuffle=True,
num_workers=workers_tr,
drop_last=False,
pin_memory=True)
dataset_va = dataset_module.Dataset(data_path,
areas_va,
rate=rate_va,
flip_prob=0.0,
crop_type='Center',
crop_size=crop_size)
dataloader_va = DataLoader(dataset_va,
batch_size=batch_size_va,
shuffle=False,
num_workers=workers_va,
drop_last=False,
pin_memory=True)
cv2.setNumThreads(
workers_tr
) # Necessary for num_workers > 0 in DataLoader, otherwise freeze
logger.info("Preparing model...")
# without beam and column: [0.,0.,1.,1.,1.,1.,1.,0.,1.,1.,1.,1.,1.,1.]
class_weights = cfg['training']['class_weights']
nclasses = len(class_weights)
num_epochs = int(cfg['training']['epochs'])
use_gnn = bool(cfg['model']['use_gnn'])
gnn_iterations = int(cfg['gnn']['iterations'])
gnn_k = int(cfg['gnn']['k'])
mlp_num_layers = int(cfg['gnn']['mlp_num_layers'])
model_module = importlib.import_module(cfg['model']['module_name'],
cfg['model']['script_path'])
use_half_precision = bool(cfg['model']['use_half_precision'])
model = model_module.Model(nclasses, mlp_num_layers)
if use_half_precision:
model.half()
for layer in model.modules():
if isinstance(layer, nn.BatchNorm2d):
layer.float()
use_bootstrap_loss = bool(cfg['loss']['use_bootstrap_loss'])
bootstrap_rate = float(cfg['loss']['bootstrap_rate'])
loss = nn.NLLLoss(reduce=not use_bootstrap_loss,
weight=torch.FloatTensor(class_weights))
softmax = nn.Softmax(dim=1)
log_softmax = nn.LogSoftmax(dim=1)
model.cuda()
loss.cuda()
softmax.cuda()
log_softmax.cuda()
opt_mode = 'half_precision_optimizer' if use_half_precision else 'single_precision_optimizer'
base_initial_lr = float((cfg[opt_mode]['base_initial_lr']))
gnn_initial_lr = float(cfg[opt_mode]['gnn_initial_lr'])
betas = cfg[opt_mode]['betas']
eps = float(cfg[opt_mode]['eps'])
weight_decay = float(cfg[opt_mode]['weight_decay'])
amsgrad = bool(cfg[opt_mode]['amsgrad'])
lr_schedule_type = cfg['schedule']['lr_schedule_type']
lr_decay = float(cfg['schedule']['lr_decay'])
lr_patience = int(cfg['schedule']['lr_patience'])
# optimizer = torch.optim.Adam(model.parameters(), lr=initial_lr, betas=betas, eps=eps,
# weight_decay=weight_decay, amsgrad=amsgrad)
optimizer = torch.optim.Adam([{
'params': model.decoder.parameters()
}, {
'params': model.gnn.parameters(),
'lr': gnn_initial_lr
}],
lr=base_initial_lr,
betas=betas,
eps=eps,
weight_decay=weight_decay,
amsgrad=amsgrad)
if lr_schedule_type == 'exp':
lambda1 = lambda epoch: pow((1 - ((epoch - 1) / num_epochs)), lr_decay)
scheduler = torch.optim.lr_scheduler.LambdaLR(optimizer,
lr_lambda=lambda1)
elif lr_schedule_type == 'plateau':
scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(
optimizer, factor=lr_decay, patience=lr_patience)
else:
print('bad scheduler')
exit(1)
model_parameters = filter(lambda p: p.requires_grad, model.parameters())
params = sum([np.prod(p.size()) for p in model_parameters])
logger.info("Number of trainable parameters: %d", params)
def get_current_learning_rates():
learning_rates = []
for param_group in optimizer.param_groups:
learning_rates.append(param_group['lr'])
return learning_rates
def eval_set(dataloader):
model.eval()
with torch.no_grad():
loss_sum = 0.0
confusion_matrix = torch.cuda.FloatTensor(np.zeros(14**2))
start_time = time.time()
for batch_idx, rgbd_label_xy in enumerate(dataloader):
sys.stdout.write('\rEvaluating test set... {}/{}'.format(
batch_idx + 1, len(dataloader)))
x = rgbd_label_xy[0].cuda(async=True)
xy = rgbd_label_xy[2].cuda(async=True)
if use_half_precision:
x = x.half()
xy = xy.half()
else:
x = x.float()
xy = xy.float()
input = x.permute(0, 3, 1, 2).contiguous()
xy = xy.permute(0, 3, 1, 2).contiguous()
target = rgbd_label_xy[1].cuda(async=True).long()
output = model(input,
gnn_iterations=gnn_iterations,
k=gnn_k,
xy=xy,
use_gnn=use_gnn,
use_half_precision=use_half_precision)
if use_bootstrap_loss:
loss_per_pixel = loss.forward(log_softmax(output.float()),
target)
topk, indices = torch.topk(
loss_per_pixel.view(output.size()[0], -1),
int((crop_size**2) * bootstrap_rate))
loss_ = torch.mean(topk)
else:
loss_ = loss.forward(log_softmax(output.float()), target)
loss_sum += loss_
pred = output.permute(0, 2, 3, 1).contiguous()
pred = pred.view(-1, nclasses)
pred = softmax(pred)
pred_max_val, pred_arg_max = pred.max(1)
pairs = target.view(-1) * 14 + pred_arg_max.view(-1)
for i in range(14**2):
cumu = pairs.eq(i).float().sum()
confusion_matrix[i] += cumu.item()
sys.stdout.write(" - Eval time: {:.2f}s \n".format(time.time() -
start_time))
loss_sum /= len(dataloader)
confusion_matrix = confusion_matrix.cpu().numpy().reshape((14, 14))
class_iou = np.zeros(14)
# we ignore void values
confusion_matrix[0, :] = np.zeros(14)
confusion_matrix[:, 0] = np.zeros(14)
for i in range(1, 14):
class_iou[i] = confusion_matrix[i, i] / (
np.sum(confusion_matrix[i, :]) +
np.sum(confusion_matrix[:, i]) - confusion_matrix[i, i])
return loss_sum.item(), class_iou, confusion_matrix
eval_loss = None
class_iou = None
confusion_matrix = None
model_to_load = None
logger.info("num_epochs: %d", num_epochs)
batch_loss_interval = cfg['training']['batch_loss_interval']
train_losses = []
eval_losses = []
if model_to_load:
logger.info("Loading old model...")
model.load_state_dict(torch.load(model_to_load))
else:
logger.info("Starting training from scratch...")
for epoch in range(1, num_epochs + 1):
batch_loss_avg = 0
if lr_schedule_type == 'exp':
scheduler.step(epoch)
for batch_idx, rgbd_label_xy in enumerate(
tqdm(dataloader_tr, smoothing=0.99)):
x = rgbd_label_xy[0].cuda(async=True)
xy = rgbd_label_xy[2].cuda(async=True)
if use_half_precision:
x = x.half()
xy = xy.half()
else:
x = x.float()
xy = xy.float()
input = x.permute(0, 3, 1, 2).contiguous()
xy = xy.permute(0, 3, 1, 2).contiguous()
target = rgbd_label_xy[1].cuda(async=True).long()
optimizer.zero_grad()
model.train()
output = model(input,
gnn_iterations=gnn_iterations,
k=gnn_k,
xy=xy,
use_gnn=use_gnn,
use_half_precision=use_half_precision)
if use_bootstrap_loss:
loss_per_pixel = loss.forward(log_softmax(output.float()),
target)
topk, indices = torch.topk(
loss_per_pixel.view(output.size()[0], -1),
int((crop_size**2) * bootstrap_rate))
loss_ = torch.mean(topk)
else:
loss_ = loss.forward(log_softmax(output.float()), target)
loss_.backward()
optimizer.step()
batch_loss_avg += loss_.item()
if batch_idx % batch_loss_interval == 0 and batch_idx > 0:
batch_loss_avg /= batch_loss_interval
train_losses.append(batch_loss_avg)
logger.info("e%db%d Batch loss average: %s", epoch, batch_idx,
batch_loss_avg)
batch_loss_avg = 0
batch_idx = len(dataloader_tr)
logger.info("e%db%d Saving model...", epoch, batch_idx)
torch.save(
model.state_dict(), log_path + '/save/' + model_name + '_' +
str(epoch) + '_' + str(batch_idx) + '.pth')
eval_loss, class_iou, confusion_matrix = eval_set(dataloader_va)
eval_losses.append(eval_loss)
if lr_schedule_type == 'plateau':
scheduler.step(eval_loss)
logger.info("e%db%d Def learning rate: %s", epoch, batch_idx,
get_current_learning_rates()[0])
logger.info("e%db%d GNN learning rate: %s", epoch, batch_idx,
get_current_learning_rates()[1])
logger.info("e%db%d Eval loss: %s", epoch, batch_idx, eval_loss)
logger.info("e%db%d Class IoU:", epoch, batch_idx)
for cl in range(14):
logger.info("%+10s: %-10s" % (idx_to_label[cl], class_iou[cl]))
logger.info("Mean IoU: %s", np.mean(class_iou[1:]))
logger.info("e%db%d Confusion matrix:", epoch, batch_idx)
logger.info(confusion_matrix)
