def __lt__(self, other):
my_attrs = utils.flatten_dict(self.attrs)
other_attrs = utils.flatten_dict(other.attrs)
if my_attrs != other_attrs:
return my_attrs < other_attrs
else:
return self.actions < other.actions
def __lt__(self, other):
my_attrs = utils.flatten_dict(self.attrs)
other_attrs = utils.flatten_dict(other.attrs)
if my_attrs != other_attrs:
return my_attrs < other_attrs
else:
return self.actions < other.actions
def update_service(self, host, service, data):
""" This function updates/inserts a service
It used by arbiter in hook_late_configuration
to put the configuration in the database
Return
* query_result: None
* error: bool
"""
# Prepare mongo Filter
mongo_filter = {"host": host,
"service": service}
# Flatten dict serv
data = flatten_dict(data)
# Save in mongo
try:
mongo_res = getattr(self.db_conn,
self.db_name).services.update(mongo_filter,
{"$set": data},
upsert=True)
except Exception as exp:
logger.error("[SnmpBooster] [code 1204] [%s, %s] "
"%s" % (host,
service,
str(exp)))
return (None, True)
return (None, self.handle_error(mongo_res, mongo_filter))
def csv_fieldnames(self):
if hasattr(self, "_csv_fieldnames"):
return self._csv_fieldnames
result_structure = {
"metrics": {
"metatrain": {
"roc_auc": None,
"precision": None,
"recall": None,
"f1": None
},
"metaval": {
"roc_auc": None,
"precision": None,
"recall": None,
"f1": None
},
"metatest": {
"roc_auc": None,
"precision": None,
"recall": None,
"f1": None
}
},
"deviation_after_best": None,
"training_iteration": None
}
fieldnames = list(flatten_dict(result_structure))
self._csv_fieldnames = fieldnames
return self._csv_fieldnames
def _serialize_v1(self, request_params):
parameters = flatten_dict(request_params)
key_values = []
for key in sorted(parameters.iterkeys()):
value = parameters[key]
quoted_value = get_quoted_value(value, quote=True)
key_values.append(key + '=' + quoted_value)
return '&'.join(key_values)
def unique_ids(components, flags):
for x in components['section']:
this_df = pd.DataFrame(utils.flatten_dict_detailed(
utils.flatten_dict(x)).items(),
columns=['tag', 'value'])
flags.append(check_duplicates_ids(this_df))
return 0
def __init__(self, params, normal_weight_init=True):
self.adv_bce_loss = nn.BCELoss()
self.aux_nll_loss = nn.NLLLoss()
self.discriminator = AuxDiscriminator(
params["n_classes"],
params["disc"]["n_feature"],
params["disc"]["n_channel"],
n_conv_block=params["n_conv_block"]).to(var.device)
self.generator = self.init_generator(params["generator_type"], params)
self.disc_optim = torch.optim.Adam(
self.discriminator.parameters(),
lr=params['disc']['lr'],
betas=tuple(params['disc']['betas'].values()))
"""
self.disc_optim = torch.optim.SGD(self.discriminator.parameters(), lr=params['disc']['lr'])
"""
self.gen_optim = torch.optim.Adam(self.generator.parameters(),
lr=params['gen']['lr'],
betas=tuple(
params['gen']['betas'].values()))
self.z_dim = params['z_dim']
self.n_classes = params["n_classes"]
self.label_smoothing = params["label_smoothing"]
self.use_inception = params["use_inception"]
if self.use_inception:
self.inception = InceptionV3()
self.writer = None
self.h_params_added = False
self.step = 0
self.epoch = 0
self.adv_fake_acc = list()
self.adv_real_acc = list()
self.aux_fake_acc = list()
self.aux_real_acc = list()
self.fake_aux_entropy = list()
self.real_aux_entropy = list()
if normal_weight_init:
self.discriminator.apply(ut.weights_init)
self.generator.apply(ut.weights_init)
params['GAN_type'] = self.__class__.__name__
params['gen_optim_type'] = self.gen_optim.__class__.__name__
params['disc_optim_type'] = self.disc_optim.__class__.__name__
self.params = ut.flatten_dict(params)
def process_response(self, request, response):
if not self.start:
return response
try:
path = smart_unicode(request.path)
proctime = round(time.time() - self.start, 3)
get = flatten_dict(request.GET)
post = flatten_dict(request.POST)
ip = request.META.get('REMOTE_ADDR', u'')
status_code = response.status_code
self._logger.info(
u'%u %s exec_time=%.3fs GET=%s POST=%s IP=%s RESP=%s' %
(status_code, path, proctime, get, post, ip, ''))
except Exception as e:
self._logger(e)
return response
def __init__(self, params, normal_weight_init=True):
self.aux_nll_loss = nn.NLLLoss()
self.critic = Critic(params["n_classes"],
params["critic"]["n_feature"],
params["critic"]["n_channel"],
params["n_conv_block"]).to(var.device)
self.generator = self.init_generator(params["generator_type"], params)
self.critic_optim = torch.optim.Adam(
self.critic.parameters(),
lr=params['critic']['lr'],
betas=tuple(params['critic']['betas'].values()))
self.gen_optim = torch.optim.Adam(self.generator.parameters(),
lr=params['gen']['lr'],
betas=tuple(
params['gen']['betas'].values()))
self.z_dim = params['z_dim']
self.n_classes = params["n_classes"]
self.gradient_penalty_factor = params['gradient_penalty_factor']
self.stability_noise_std = params['stability_noise_std']
self.use_inception = params["use_inception"]
if self.use_inception:
self.inception = InceptionV3()
self.writer = None
self.h_params_added = False
self.step = 0
self.epoch = 0
self.aux_fake_acc = list()
self.aux_real_acc = list()
self.fake_aux_entropy = list()
self.real_aux_entropy = list()
if normal_weight_init:
self.critic.apply(ut.weights_init)
self.generator.apply(ut.weights_init)
params['GAN_type'] = self.__class__.__name__
params['gen_optim_type'] = self.gen_optim.__class__.__name__
params['disc_optim_type'] = self.critic_optim.__class__.__name__
self.params = ut.flatten_dict(params)
def parse_available_button(list_action):
"""
Assert actions are possible and create all possible combinations of available actions
"""
all_actions = get_subkey(ALL_AVAILABLE_ACTIONS)
reversed_dict = flatten_dict(ALL_AVAILABLE_ACTIONS)
a_action = []
new_dict_act = defaultdict(list)
for action in list_action:
if action in all_actions:
a_action.append(action)
new_dict_act[reversed_dict[action]].append(action)
else:
logger.warning("{} is not an action available!".format(action))
# check if at least one action was correct
assert a_action
return a_action, new_dict_act
def update_service(self, host, service, data):
""" This function updates/inserts a service
It used by arbiter in hook_late_configuration
to put the configuration in the database
Return
* query_result: None
* error: bool
"""
# Prepare mongo Filter
mongo_filter = {"host": host, "service": service}
# Flatten dict serv
data = flatten_dict(data)
# Save in mongo
try:
mongo_res = getattr(self.db_conn, self.db_name).services.update(mongo_filter, {"$set": data}, upsert=True)
except Exception as exp:
logger.error("[SnmpBooster] [code 1204] [%s, %s] " "%s" % (host, service, str(exp)))
return (None, True)
return (None, self.handle_error(mongo_res, mongo_filter))
def get_best_lr_for_linear_bert_sentiment():
datasets = ['mr', 'subj', 'mpqa', 'sst']
lrs = [0.1, 0.01, 0.001, 0.0001, 0.00001, 0.000001, 0.0000001]
metric = "best_valid_err"
best_lr = {}
for dataset in datasets:
all_json_regex = "../../results/predictions/dimensionality_2019-07-07/{}/nbit_32/*/final_results.json".format(
dataset)
results = utils.gather_results(all_json_regex)
results = [utils.flatten_dict(result) for result in results]
best_err = 1.0
for lr in lrs:
keys = {"lr": [lr]}
subset_results = utils.extract_result_subset(results, keys)
assert len(subset_results) == 3
ave, std = utils.stats_on_subset_json(subset_results, metric)
print(dataset, " lr ", lr, " val err ave/std", ave, std)
if ave < best_err:
best_err = ave
best_lr[dataset] = lr
print(best_lr)
return best_lr
def __init__(self, result, compare_method, compare_metric):
super(Compare, self).__init__()
""" Compare
Args:
result (dict): result dict
compare_method (fn): function to compare two metrics
compare_metric (fn): function to specify metrics to be compared
"""
self.result = result
self.compare_metric = compare_metric
self.compare_method = compare_method
self.four_metrics = {}
self.compare_result = {}
for error_type in config.error_types:
self.compare_result[error_type['name']], self.four_metrics[
error_type['name']] = self.compare_error(error_type['name'])
# key order: error/clean_method/dataset/models/scenario/ [compare_keys...]
self.compare_result = utils.flatten_dict(self.compare_result)
# rearrange key order: error/dataset/clean_method/models/scenario/ [compare_keys...]
self.compare_result = utils.rearrange_dict(self.compare_result,
[0, 2, 1, 3, 4])
def label_predict(_config, _seed, _run):
"""Train a model with configurations."""
if ('complete' in _run.info) and (_run.info['complete']):
print("The task is already finished")
return None
log_dir = os.path.join(
"logs", _config['db_name'], datetime.datetime.now().strftime('%Y%m%d%H%M%S%f'))
_run.info['log_dir'] = log_dir
random.seed(_seed)
np.random.seed(_seed)
torch.manual_seed(_seed)
torch.cuda.manual_seed(_seed)
# torch.cuda.set_device(_config['gpu'])
torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
writer = SummaryWriter(log_dir)
# load datasets and build model
datasets = get_dataset(**_config['dataset'])
train_dataset_joint, valid_dataset_joint, train_dataset_marginal, valid_dataset_marginal, test_dataset = datasets
if _config['classifier']['label_size'] < 1.0:
# train_unsup_dataset_joint = train_dataset_joint
train_dataset_joint, _ = get_split_datasets(
train_dataset_joint, split_size=_config['classifier']['label_size'])
train_dataset_marginal, _ = get_split_datasets(
train_dataset_marginal, split_size=_config['classifier']['label_size'])
if _config['method']['sampler_mode'] == 'random':
print("Sample mode: Random")
train_loader_joint = data.DataLoader(
train_dataset_joint, batch_size=_config['optim']['batch_size'], shuffle=True)
train_loader_marginal = data.DataLoader(
train_dataset_marginal, batch_size=_config['optim']['batch_size'], shuffle=True)
elif _config['method']['sampler_mode'] == 'diff':
joint_sampler = get_split_samplers(train_dataset_joint, [0, 1, 2])
joint_batch_sampler = SplitBatchSampler(joint_sampler, _config['optim']['batch_size'], True)
train_loader_joint = data.DataLoader(train_dataset_joint, batch_sampler=joint_batch_sampler)
marginal_sampler = get_split_samplers(train_dataset_marginal, [1, 2, 0])
marginal_batch_sampler = SplitBatchSampler(marginal_sampler, _config['optim']['batch_size'], True)
train_loader_marginal = data.DataLoader(train_dataset_marginal, batch_sampler=marginal_batch_sampler)
elif _config['method']['sampler_mode'] == 'same':
joint_sampler = get_split_samplers(train_dataset_joint, [0, 1, 2])
joint_batch_sampler = SplitBatchSampler(joint_sampler, _config['optim']['batch_size'], True)
train_loader_joint = data.DataLoader(train_dataset_joint, batch_sampler=joint_batch_sampler)
marginal_sampler = get_split_samplers(train_dataset_marginal, [0, 1, 2])
marginal_batch_sampler = SplitBatchSampler(marginal_sampler, _config['optim']['batch_size'], True)
train_loader_marginal = data.DataLoader(train_dataset_marginal, batch_sampler=marginal_batch_sampler)
print("Dataset: {} train, {} valid, {} test".format(
len(train_dataset_joint), len(valid_dataset_joint), len(test_dataset)))
model = get_model(
input_shape=train_dataset_joint.get('input_shape'), K=_config['dataset']['K'], **_config['method'])
if _config['classifier']['pretrain']:
query = deepcopy(_config)
del query['classifier']
del query['classifier_optim']
del query['db_name']
del query['gpu']
del query['unsup_db_name']
query = flatten_dict(query)
extractor = MongoExtractor(None, _config['unsup_db_name'])
result = list(extractor.find(query, ['config', 'info'], False, 'COMPLETED'))
assert len(result) == 1, "There are too many or no results. Please check the query {}".format(query)
result = result[0]
if _config['method']['name'] == 'CPC':
path = os.path.join(result['info']['log_dir'], 'model_{}.pth'.format(_config['optim']['num_batch']))
model = model.cpu()
model.load_state_dict(torch.load(path, map_location='cpu'))
model = model.cuda()
elif _config['method']['name'] == 'VAE':
path = os.path.join(result['info']['log_dir'], 'q_{}.pth'.format(_config['optim']['num_batch']))
# TODO: Need refactor
from run_sacred import Encoder, CPCModel
from vae import Inference
g_enc_size = _config['method']['hidden']
g_enc = Encoder(input_shape=train_dataset_joint.get('input_shape'), hidden_size=None).cuda()
c_enc = model.c_enc
predictor = model.predictor
q = Inference(g_enc, network_output=g_enc.output_shape()[1], z_size=g_enc_size)
q.load_state_dict(torch.load(path, map_location='cpu'))
q = q.cuda()
g_enc = nn.Sequential(q.network, q.network_mu, nn.ReLU(True), nn.Dropout(0.5))
g_enc.output_shape = lambda: (None, g_enc_size) # dummy function, may be there exists a better way
model = CPCModel(g_enc, c_enc, predictor).cuda()
classifier = get_classifier(model, train_dataset_joint.get('num_classes'), **_config['classifier'])
print(classifier)
# TODO: Select valid poarameter from dictionary
# MEMO: It can be donw with inspect module.
optimizer = optim.Adam(classifier.parameters(), lr=_config['classifier_optim']['lr'])
criterion = nn.NLLLoss()
# Training and evaluation
L = _config['dataset']['L']
monitor_per = _config['classifier_optim']['monitor_per']
divergence_criterion = CMD(n_moments=5)
get_g_of = get_feature_of(model.g_enc, None, _config['dataset']['L'])
get_c_of = get_feature_of(model.g_enc, model.c_enc, _config['dataset']['L'])
# early stopping
metric_names = ['test-accuracy', 'valid-accuracy']
best_values = dict(zip(metric_names, [0] * len(metric_names)))
patient = 10
counter = 0
start_time = time.time()
for num_iter in range(_config['classifier_optim']['num_batch']):
optimizer.zero_grad()
X, Y = train_loader_joint.__iter__().__next__()
y = Y[:, 0, L-1].long().cuda()
pred_y = classifier(X[..., :L].float().cuda())
loss = criterion(pred_y, y)
if _config['classifier']['auxiliary'] != 0.0:
assert _config['method']['name'] == 'CPC'
aux_criterion = nn.BCELoss()
# X_m, _ = train_loader_marginal.__iter__().__next__()
num_negative = _config['method']['num_negative']
num_spy_mask = _config['method']['num_spy_mask']
X_m = [None] * num_negative
for i in range(num_negative):
_X, _ = train_loader_marginal.__iter__().__next__()
X_m[i] = _X.float().cuda()
K = X_m[0].shape[-1]
L = X.shape[-1] - K
X = X.float().cuda()
score_j_list, score_m_list = model(X, X_m, L, K)
score_j_list, score_m_list, predictions = model(X, X_m, L, K, return_predictions=True)
_loss = 0
if num_spy_mask != 0:
masks = model._get_masks(predictions[0], p=model.p, num_mask=num_spy_mask)
spy_score_j_list = model.get_score_of(X[..., L:], K, predictions, masks)
spy_score_m_list = [None] * len(X_m)
for i, X in enumerate(X_m):
score_m = model.get_score_of(X, K, predictions, masks)
spy_score_m_list[i] = score_m
for k in range(K):
score_j = score_j_list[k]
loss += aux_criterion(torch.sigmoid(score_j), torch.ones((len(score_j), 1)).cuda())
for m in range(num_negative):
score_m = score_m_list[m][k]
if num_spy_mask != 0:
spy_score_j = spy_score_j_list[k]
spy_score_m = spy_score_m_list[m][k]
spy_mask = (spy_score_j > spy_score_m).float().detach()
score_m = score_m * spy_mask
_loss += aux_criterion(torch.sigmoid(score_m), torch.zeros((len(score_j), 1)).cuda())
_loss = _loss / (2*K)
loss += _config['classifier']['auxiliary'] * _loss
loss.backward()
optimizer.step()
if ((num_iter + 1) % monitor_per) != 0:
continue
train_result = validate_label_prediction(classifier, train_dataset_joint, L=L, nb_batch=None)
valid_result = validate_label_prediction(classifier, valid_dataset_joint, L=L, nb_batch=None)
test_result = validate_label_prediction(classifier, test_dataset, L=L, nb_batch=None)
elapsed_time_before_cmd = time.time() - start_time
valid_result['cmdg'] = pairwise_divergence(
valid_dataset_joint.datasets, get_g_of,
divergence_criterion, num_batch=None, batch_size=128
)
valid_result['cmdc'] = pairwise_divergence(
valid_dataset_joint.datasets, get_c_of,
divergence_criterion, num_batch=None, batch_size=128
)
test_result['cmdg'] = pairwise_divergence(
[valid_dataset_joint, test_dataset], get_g_of,
divergence_criterion, num_batch=None, batch_size=128
)
test_result['cmdc'] = pairwise_divergence(
[valid_dataset_joint, test_dataset], get_c_of,
divergence_criterion, num_batch=None, batch_size=128
)
# model_path = '{}/model_{}.pth'.format(log_dir, num_iter+1)
# torch.save(model.state_dict(), model_path)
writer.add_scalars('train', train_result, num_iter+1)
writer.add_scalars('valid', valid_result, num_iter+1)
writer.add_scalars('test', test_result, num_iter+1)
elapsed_time = time.time() - start_time
print("-------- #iter: {}, elapad time: {} ({}) --------".format(
num_iter+1, elapsed_time, elapsed_time_before_cmd))
print('train', ', '.join(['{}:{:.3f}'.format(k, v) for k, v in iteritems(train_result)]))
print('valid', ', '.join(['{}:{:.3f}'.format(k, v) for k, v in iteritems(valid_result)]))
print('test', ', '.join(['{}:{:.3f}'.format(k, v) for k, v in iteritems(test_result)]))
start_time = time.time()
# early stopping
is_better = False
for metric_name in metric_names:
db_name, metric = metric_name.split('-')
if db_name == 'valid':
current_value = valid_result[metric]
elif db_name == 'test':
current_value = test_result[metric]
if current_value >= best_values[metric_name]:
counter = 0
best_values[metric_name] = current_value
is_better = True
if is_better:
print("Update the best results", best_values)
else:
counter += 1
print("Not update the best results (counter={})".format(counter), best_values)
if counter == patient:
break
# Post process
result = writer.scalar_dict
for k, v in iteritems(result):
ks = k.split('/')
_run.info['{}-{}'.format(ks[-2], ks[-1])] = v
def entry_train(cfg, record_file=""):
loader_train, _, _ = create_dataloader(split='train', **cfg.DATALOADER)
loader_valid, _, _ = create_dataloader('valid', **cfg.DATALOADER)
loader_test, _, _ = create_dataloader('test', **cfg.DATALOADER)
model = get_model(cfg)
model.to(DEVICE)
print(model)
optimizer = torch.optim.Adam(model.parameters(),
lr=cfg.TRAIN.learning_rate,
weight_decay=cfg.TRAIN.l2)
scheduler = ReduceLROnPlateau(optimizer,
mode='max',
factor=0.5,
patience=10,
verbose=True)
best_acc_rel_avg_valid = -1
best_epoch_rel_avg_valid = 0
best_acc_rel_avg_test = -1
log_dir = f"./runs/{cfg.EXP.EXP_ID}"
if not os.path.exists(log_dir):
os.makedirs(log_dir)
tb = TensorboardManager(log_dir)
for epoch in range(cfg.TRAIN.num_epochs):
print('\nEpoch #%d' % epoch)
print('Training..')
(acc_train, pre_train, rec_train, f1_train, acc_rel_train,
acc_rel_avg_train) = train(loader_train, model, optimizer,
DEVICE, cfg.TRAIN.weighted_loss)
print(f'Train, acc avg: {acc_rel_avg_train} acc: {acc_train},'
f' pre: {pre_train}, rec: {rec_train}, f1: {f1_train}')
print({x: round(y, 3) for x, y in acc_rel_train.items()})
tb.update('train', epoch, {'acc': acc_train})
print('\nValidating..')
(acc_valid, pre_valid, rec_valid, f1_valid, acc_rel_valid,
acc_rel_avg_valid) = validate(loader_valid, model, DEVICE)
print(f'Valid, acc avg: {acc_rel_avg_valid} acc: {acc_valid},'
f' pre: {pre_valid}, rec: {rec_valid}, f1: {f1_valid}')
print({x: round(y, 3) for x, y in acc_rel_valid.items()})
tb.update('val', epoch, {'acc': acc_valid})
print('\nTesting..')
(acc_test, pre_test, rec_test, f1_test, acc_rel_test,
acc_rel_avg_test) = validate(loader_test, model, DEVICE)
print(f'Test, acc avg: {acc_rel_avg_test} acc: {acc_test},'
f' pre: {pre_test}, rec: {rec_test}, f1: {f1_test}')
print({x: round(y, 3) for x, y in acc_rel_test.items()})
if acc_rel_avg_valid > best_acc_rel_avg_valid:
print('Accuracy has improved')
best_acc_rel_avg_valid = acc_rel_avg_valid
best_epoch_rel_avg_valid = epoch
save_checkpoint(epoch, model, optimizer, acc_rel_avg_valid, cfg)
if acc_rel_avg_test > best_acc_rel_avg_test:
best_acc_rel_avg_test = acc_rel_avg_test
if (epoch - best_epoch_rel_avg_valid) > cfg.TRAIN.early_stop:
print(f"Early stopping at {epoch} as val acc did not improve"
f" for {cfg.TRAIN.early_stop} epochs.")
break
scheduler.step(acc_train)
print('\nTesting..')
load_best_checkpoint(model, cfg)
(acc_test, pre_test, rec_test, f1_test, acc_rel_test,
acc_rel_avg_test) = validate(loader_test, model, DEVICE)
print(f'Best valid, acc: {best_acc_rel_avg_valid}')
print(f'Best test, acc: {best_acc_rel_avg_test}')
print(f'Test at best valid, acc avg: {acc_rel_avg_test}, acc: {acc_test},'
f' pre: {pre_test}, rec: {rec_test}, f1: {f1_test}')
print({x: round(y, 3) for x, y in acc_rel_test.items()})
if record_file != "":
exp = RecordExp(record_file)
exp.record_param(flatten_dict(dict(cfg)))
exp.record_result({
"final_train": acc_rel_avg_train,
"best_val": best_acc_rel_avg_valid,
"best_test": best_acc_rel_avg_test,
"final_test": acc_rel_avg_test
})
def train(train_data_dir: str, hparams: HyperParameters,
train_config: TrainConfig) -> TrainingResults:
print("Hyperparameters:", hparams)
print("Train_config:", train_config)
start_time = time.time()
# save the hyperparameter config to a file.
with open(os.path.join(train_config.log_dir, "hyperparameters.json"),
"w") as f:
json.dump(asdict(hparams), f, indent=4)
with open(os.path.join(train_config.log_dir, "train_config.json"),
"w") as f:
json.dump(asdict(train_config), f, indent=4)
model = get_model(hparams)
model.summary()
train_dataset, valid_dataset = train_input_pipeline(
train_data_dir, hparams, train_config)
if DEBUG:
train_dataset = train_dataset.repeat(100)
if valid_dataset:
valid_dataset = valid_dataset.repeat(100)
using_validation_set = valid_dataset is not None
hparams_dict = asdict(hparams)
import pprint
pprint.pprint(hparams_dict, indent=4)
flattened_dict = utils.flatten_dict(hparams_dict)
training_callbacks = [
tf.keras.callbacks.TensorBoard(log_dir=train_config.log_dir,
profile_batch=0),
hp.KerasCallback(train_config.log_dir, flattened_dict),
tf.keras.callbacks.TerminateOnNaN(),
utils.EarlyStoppingWhenValueExplodes(monitor="loss",
check_every_batch=True),
tf.keras.callbacks.ModelCheckpoint(
filepath=os.path.join(train_config.log_dir, "model.h5"),
monitor="val_loss" if using_validation_set else "loss",
verbose=1,
save_best_only=True,
mode='auto'),
tf.keras.callbacks.EarlyStopping(
patience=train_config.early_stopping_patience,
monitor='val_loss' if using_validation_set else "loss"),
]
history = None
try:
history = model.fit(
train_dataset,
validation_data=valid_dataset if using_validation_set else None,
epochs=train_config.epochs,
callbacks=training_callbacks,
# steps_per_epoch=int(train_samples / hparams.batch_size),
)
end_time = time.time()
training_time_secs = end_time - start_time
loss_metric = "val_loss" if using_validation_set else "loss"
best_loss_value = min(history.history[loss_metric])
num_epochs = len(history.history[loss_metric])
print(
f"BEST {'VALIDATION' if using_validation_set else 'TRAIN'} LOSS:",
best_loss_value)
results = TrainingResults()
results.total_epochs = num_epochs
results.model_param_count = model.count_params()
results.training_time_secs = training_time_secs
if using_validation_set:
metrics = model.evaluate(valid_dataset)
results.metrics_dict = OrderedDict(
zip(model.metrics_names, metrics))
else:
results.metrics_dict = {
metric_name: values[-1]
for metric_name, values in history.history.items()
}
return results
except Exception as e:
print(f"\n\n {e} \n\n")
return TrainingResults()
def forward(self, x_0, x, a, prior_sample=True, return_x=False):
keys = set(locals().keys())
loss = 0. # for backprop
s_losss = [0.] * self.num_states
x_losss_p = [0.] * self.num_states
x_losss_q = [0.] * self.num_states
s01_losss = [0.] * self.num_states
if self.args.posterior_s_0:
s_0_s_loss = None
s_0_x_loss = None
if self.debug:
s_abss_p = [0.] * self.num_states
s_stds_p = [0.] * self.num_states
s_abss_q = [0.] * self.num_states
s_stds_q = [0.] * self.num_states
keys = set(locals().keys()) - keys - {"keys"}
x = x.transpose(0, 1).to(self.device) # T,B,3,28,28
a = a.transpose(0, 1).to(self.device) # T,B,4
x_0 = x_0.to(self.device)
_T, _B = x.size(0), x.size(1)
_x_p = []
_x_q = []
if self.args.posterior_s_0:
s_prevs, s_0_s_loss, s_0_x_loss = self.sample_s_0(x_0)
else:
s_prevs = self.sample_s_0(x_0)
for t in range(_T):
x_t, a_t = x[t], a[t]
s_ts = [] # store all hierarchy's state
for i in range(self.num_states):
h_t = self.encoders[i](x_t)
s_prev = s_prevs[i]
a_list = [a_t] + s_ts[-1:] # use upper state
q = Normal(*self.posteriors[i](s_prev, h_t, a_list))
p = Normal(*self.priors[i](s_prev, a_list))
s_losss[i] += torch.sum(kl_divergence(q, p), dim=[1]).mean()
s_t_p = p.mean if prior_sample else p.rsample()
s_t_q = q.rsample()
s_t = s_t_p if prior_sample else s_t_q
s_ts.append(s_t)
s01_losss[i] += torch.sum(kl_divergence(q, Normal(0., 1.)),
dim=[1]).mean()
if self.debug:
s_abss_p[i] += p.mean.abs().mean()
s_stds_p[i] += p.stddev.mean()
s_abss_q[i] += q.mean.abs().mean()
s_stds_q[i] += q.stddev.mean()
decoder_dist_p = Normal(*self.decoders[i](s_t_p))
x_losss_p[i] += -torch.sum(decoder_dist_p.log_prob(x_t),
dim=[1, 2, 3]).mean()
decoder_dist_q = Normal(*self.decoders[i](s_t_q))
x_losss_q[i] += -torch.sum(decoder_dist_q.log_prob(x_t),
dim=[1, 2, 3]).mean()
if return_x:
_x_p.append(decoder_dist_p.mean)
_x_q.append(decoder_dist_q.mean)
s_prevs = s_ts
if return_x:
_x_p = torch.stack(_x_p).transpose(0, 1)
_x_p = torch.clamp(_x_p, 0, 1)
_x_q = torch.stack(_x_q).transpose(0, 1)
_x_q = torch.clamp(_x_q, 0, 1)
return _x_p, _x_q
else:
for i in range(self.num_states):
if i not in self.args.static_hierarchy:
loss += s_losss[i] + x_losss_q[i]
if self.args.beta_x_p:
loss += self.args.beta_x_p * x_losss_p[i]
if self.args.beta_s01:
loss += self.args.beta_s01 * s01_losss[i]
if self.args.posterior_s_0:
loss += s_0_s_loss + s_0_x_loss
_locals = locals()
info = flatten_dict({key: _locals[key] for key in keys})
info = dict(sorted(info.items()))
return loss, info
def parse_loaded_data(data_dict):
effective_score_labels = [
"arrival_order",
"frame_no",
"horse_no",
# "horse_name",
"horse_sex",
"horse_age",
"horse_weight",
"horse_weight_diff",
# "horse_b",
# "arrival_diff",
"time",
"last3f_time",
# "passing_order_1st",
# "passing_order_2nd",
# "passing_order_3rd",
# "passing_order_4th",
# "jockey_name",
"jockey_weight",
"odds",
"popularity",
# "trainer_name",
]
effective_racehead_labels = [
"race_no",
# "date",
"week",
"kai",
# "lacation",
"nichi",
# "start_time",
"weather",
"condition",
]
# ネスト化された辞書を平坦化
score_dicts = []
racehead_dict = {}
for k, v in data_dict.items():
if k == 'scores':
for s in v:
flattened_dict = flatten_dict(s)
score_dict = {
k_: v_
for k_, v_ in flattened_dict.items()
if k_ in effective_score_labels
}
score_dicts.append(score_dict)
elif k == 'racehead':
flattened_dict = flatten_dict(v)
racehead_dict = {
k_: v_
for k_, v_ in flattened_dict.items()
if k_ in effective_racehead_labels
}
# データを整形
score_and_racehead = []
# racehead
racehead = []
for k, v in racehead_dict.items():
if k == 'week':
racehead.append(parse_week(v))
# elif k == 'start_time':
# racehead.append(parse_start_time(v))
elif k == 'weather':
racehead.append(parse_weather(v))
elif k == 'condition':
racehead.append(parse_condition(v))
else:
racehead.append(v)
arrival_orders = []
for score_dict in score_dicts:
# score
score = []
for k, v in score_dict.items():
if k == 'arrival_order':
arrival_orders.append(v)
elif k == 'horse_sex':
score.append(parse_horse_sex(v))
elif k == 'horse_weight_diff':
score.append(parse_horse_weight_diff(v))
elif k == 'jockey_weight':
score.append(parse_jockey_weight(v))
else:
score.append(v)
score_and_racehead.append(score + racehead)
return score_and_racehead, arrival_orders
def format_record(recid, record=None):
if not record:
record = ALL[str(recid)]
if len(record):
user_id = get_id(record)
# uid = record.get('UID', '')
# guid = record.get('GUID', '')
# pa = record.get('PA', '')
# user_id = uid
# if len(user_id) != 6:
# user_id = guid
# if len(user_id) != 6:
# user_id = pa
# if len(user_id) != 6:
# user_id = uid
# if len(user_id) != 7:
# user_id = guid
# if len(user_id) != 7:
# user_id = pa
# if len(user_id) != 7:
# user_id = uid
# print 'recno: %s\tuid: %s\tguid: %s\tpa: %s' % (recid, uid, guid, pa)
template = {'tag': '999', 'ind1': ' ', 'ind2': ' ', 'subs': {'a' : user_id}}
tags = []
tags.append(Field(tag=template['tag'], indicators=[template['ind1'], template['ind2']], subfields=flatten_dict(template['subs'])))
for (field, value) in record.items():
if field in mapping:
subfields = mapping[field](value)
for template in subfields:
tags.append(Field(tag=template['tag'], indicators=[template['ind1'], template['ind2']], subfields=flatten_dict(template['subs'])))
tags.sort()
res = Record()
for tag in tags:
res.add_ordered_field(tag)
return merge_subfields(res, ['101', '104', '110', '130', '400'])
def format_record(recid, record=None):
if not record:
record = ALL[str(recid)]
if len(record):
tags = []
for (field, value) in record.items():
if field in mapping:
subfields = mapping[field](value)
for template in subfields:
tags.append(Field(tag=template['tag'], indicators=[template['ind1'], template['ind2']], subfields=flatten_dict(template['subs'])))
tags.sort()
res = Record()
for tag in tags:
res.add_ordered_field(tag)
return merge_subfields(res, [])
def norm(params, p=2.):
sqnorm = jnp.sum(flatten_dict(lambda x: (x**p).sum(), params))
sqnorm = (sqnorm**(1. / p))
return sqnorm
def main():
X_train = pd.read_pickle('input/train.pkl').drop(
'building_id', axis=1).sample(frac=0.1, random_state=42)
y_train = np.log1p(X_train.pop('meter_reading'))
X_test = pd.read_pickle('input/test.pkl').drop(['row_id', 'building_id'],
axis=1)
process_date(X_train)
process_date(X_test)
label_encoding(X_train, X_test)
models = []
metrics = {'train': [], 'valid': []}
feature_importances_split = np.zeros(X_train.shape[1])
feature_importances_gain = np.zeros(X_train.shape[1])
fold = KFold(**PARAMS['fold'])
mlflow.set_experiment('ashrae-energy-prediction')
with mlflow.start_run() as run:
mlflow.log_params(flatten_dict(PARAMS))
for fold_idx, (idx_tr, idx_val) in enumerate(fold.split(X_train)):
print_divider(f'Fold {fold_idx}')
X_tr, X_val = X_train.iloc[idx_tr], X_train.iloc[idx_val]
y_tr, y_val = y_train.iloc[idx_tr], y_train.iloc[idx_val]
model = lgbm.LGBMRegressor(**PARAMS['model'])
model.fit(X_tr,
y_tr,
eval_set=[(X_tr, y_tr), (X_val, y_val)],
eval_names=list(metrics.keys()),
**PARAMS['fit'])
feature_importances_split += devide_by_sum(
model.booster_.feature_importance(
importance_type='split')) / fold.n_splits
feature_importances_gain += devide_by_sum(
model.booster_.feature_importance(
importance_type='gain')) / fold.n_splits
for key in metrics.keys():
metrics[key].append({
'name': 'rmse',
'values': model.evals_result_[key]['rmse'],
'best_iteration': model.best_iteration_
})
models.append(model)
del idx_tr, idx_val, X_tr, X_val, y_tr, y_val
for key in metrics.keys():
log_plot(metrics[key], pf.metric_history,
f'metric_history_{key}.png')
features = np.array(model.booster_.feature_name())
log_plot((features, feature_importances_split, 'split'),
pf.feature_importance, 'feature_importance_split.png')
log_plot((features, feature_importances_gain, 'gain'),
pf.feature_importance, 'feature_importance_gain.png')
# prediction (split the data into chunks because it's too large)
print_divider('prediction')
i = 0
pred = []
chunk_size = 500000
for j in tqdm(range(int(np.ceil(X_test.shape[0] / chunk_size)))):
pred.append(
np.expm1(
sum([
model.predict(X_test.iloc[i:i + chunk_size])
for model in models
]) / fold.n_splits))
i += chunk_size
# make submission
submission = pd.read_csv('input/sample_submission.csv')
submission['meter_reading'] = pred
submission.loc[submission['meter_reading'] < 0, 'meter_reading'] = 0
submission_path = f'submission/{timestamp()}.csv'
submission.to_csv(submission_path, index=False)
mlflow.log_param('submission_path', submission_path)
# run the MLflow UI to check the training result
print_divider('MLflow UI')
url_base = 'http://127.0.0.1:5000/#/experiments/{0}/runs/{1}'
print('Run URL:', url_base.format(run.info.experiment_id,
run.info.run_uuid))
mlflow_ui()
def flatten(self, filename):
return utils.flatten(utils.flatten_dict(utils.load_file(filename)))
def train_and_predict(X_train, y_train, X_test, params, experiment_path):
# explicitly setting up mlflow experiment
try:
mlflow.create_experiment(experiment_path)
except (mlflow.exceptions.RestException,
mlflow.exceptions.MlflowException):
print(f'The specified experiment ({experiment_path}) already exists.')
skf = StratifiedKFold(**params['fold'])
proba_oof = np.zeros(X_train.shape[0])
pred_oof = np.zeros(X_train.shape[0])
proba_test = np.zeros(X_test.shape[0])
metric_history = []
avg_scores = defaultdict(int)
feature_importances_split = np.zeros(X_train.shape[1])
feature_importances_gain = np.zeros(X_train.shape[1])
with mlflow.start_run() as run:
for fold, (trn_idx, val_idx) in enumerate(skf.split(X_train, y_train)):
print_divider(f'Fold: {fold}')
X_trn, y_trn = X_train.iloc[trn_idx, :], y_train.iloc[trn_idx]
X_val, y_val = X_train.iloc[val_idx, :], y_train.iloc[val_idx]
model = lgbm.LGBMClassifier(**params['model'])
model.fit(X_trn,
y_trn,
eval_set=[(X_val, y_val)],
eval_names=['valid'],
**params['fit'])
metric_history.append({
'name':
model.metric,
'values':
model.evals_result_['valid'][model.metric],
'best_iteration':
model.best_iteration_
})
feature_importances_split += devide_by_sum(
model.booster_.feature_importance(
importance_type='split')) / skf.n_splits
feature_importances_gain += devide_by_sum(
model.booster_.feature_importance(
importance_type='gain')) / skf.n_splits
proba_val = model.predict_proba(
X_val, num_iteration=model.best_iteration_)[:, 1]
pred_val = (proba_val > 0.5).astype(np.int8)
proba_oof[val_idx] = proba_val
pred_oof[val_idx] = pred_val
scores_val = get_scores(y_val, pred_val)
mlflow.log_metrics(
{
**scores_val,
'best_iteration': model.best_iteration_,
},
step=fold)
for k, v in scores_val.items():
avg_scores[k] += v / skf.n_splits
proba_test += model.predict_proba(X_test)[:, 1] / skf.n_splits
del trn_idx, X_trn, y_trn, val_idx, X_val, y_val, proba_val, pred_val
gc.collect()
mlflow.log_params({
**flatten_dict(params), 'fold.strategy':
skf.__class__.__name__,
'model.type':
model.__class__.__name__
})
print_divider('Saving plots')
# scores
log_plot(avg_scores, pf.scores, 'scores.png')
# feature importance (only top 30)
features = np.array(model.booster_.feature_name())
log_plot((features, feature_importances_split, 'split', 30),
pf.feature_importance, 'feature_importance_split.png')
log_plot((features, feature_importances_gain, 'gain', 30),
pf.feature_importance, 'feature_importance_gain.png')
# metric history
log_plot(metric_history, pf.metric_history, 'metric_history.png')
# confusion matrix
cm = confusion_matrix(y_train, pred_oof)
log_plot(cm, pf.confusion_matrix, 'confusion_matrix.png')
# roc curve
fpr, tpr, _ = roc_curve(y_train, pred_oof)
roc_auc = roc_auc_score(y_train, pred_oof)
log_plot((fpr, tpr, roc_auc), pf.roc_curve, 'roc_curve.png')
# precision-recall curve
pre, rec, _ = precision_recall_curve(y_train, pred_oof)
pr_auc = average_precision_score(y_train, pred_oof)
log_plot((pre, rec, pr_auc), pf.pr_curve, 'pr_curve.png')
return proba_test, run.info.experiment_id, run.info.run_uuid
def save_four_metrics(self, save_dir):
for error_type in config.error_types:
save_path = os.path.join(
save_dir, "{}_four_metrics.xlsx".format(error_type['name']))
utils.dfs_to_xls(self.four_metrics[error_type['name']], save_path)
flat_metrics = utils.flatten_dict(self.four_metrics)
def flatten_dict(self, dct): return flatten_dict(dct)
def format_record(recid, record=None):
if not record:
record = ALL[str(recid)]
if len(record):
tags = []
try:
tags.append(Field(tag='999', indicators=[' ', ' '], subfields=['a', match_serial(record)]))
except:
with codecs.open(log_dir + 'unmatched_serials.txt', 'a', encoding='utf8') as f:
f.write(recid + '\r\n')
f.close()
for (field, value) in record.items():
if field in mapping:
try:
subfields = mapping[field](value)
for template in subfields:
tags.append(Field(tag=template['tag'], indicators=[template['ind1'], template['ind2']], subfields=flatten_dict(template['subs'])))
except:
with codecs.open(log_dir + 'log_serials.txt', 'a', encoding='utf8') as f:
f.write(("==================\n%s\n%s:\t%s\n\n" % (recid, field, record[field])).replace('\n', '\r\n'))
f.close()
tags.sort()
res = Record()
for tag in tags:
res.add_ordered_field(tag)
return merge_subfields(res, [])
