def run_HAC(FLAGS, env, agent):
# Print task summary
print_summary(FLAGS, env)
# Determine training mode. If not testing and not solely training, interleave training and testing to track progress
mix_train_test = False
if not FLAGS.test and not FLAGS.train_only:
mix_train_test = True
# Track total training episodes completed
total_episodes = 0
for batch in range(NUM_BATCH):
num_episodes = agent.other_params["num_exploration_episodes"]
# Evaluate policy every TEST_FREQ batches if interleaving training and testing
if mix_train_test and batch % TEST_FREQ == 0:
print("\n--- TESTING ---")
agent.FLAGS.test = True
num_episodes = num_test_episodes
# Reset successful episode counter
successful_episodes = 0
for episode in range(num_episodes):
print("\nBatch %d, Episode %d" % (batch, episode))
# Train for an episode
success = agent.train(env, episode, total_episodes)
if FLAGS.train_only or (mix_train_test and batch % TEST_FREQ != 0):
total_episodes += 1
if success:
print("Batch %d, Episode %d End Goal Achieved\n" %
(batch, episode))
# Increment successful episode counter if applicable
if mix_train_test and batch % TEST_FREQ == 0:
successful_episodes += 1
# Save agent
agent.save_model(episode)
# Finish evaluating policy if tested prior batch
if mix_train_test and batch % TEST_FREQ == 0:
# Log performance
success_rate = successful_episodes / num_test_episodes * 100
print("\nTesting Success Rate %.2f%%" % success_rate)
agent.log_performance(success_rate)
agent.FLAGS.test = False
print("\n--- END TESTING ---\n")
def infinite_fcn(train_embedding, test_embedding, data_set, binary=True):
_, _, kernel_fn = stax.serial(
stax.Dense(64, 2., 0.05),
stax.Relu(),
stax.Dense(32, 2., 0.05),
stax.Relu(),
stax.Dense(4, 2., 0.05),
stax.Relu(),
)
# 0 for no batching, whole batch
kernel_fn = nt.batch(kernel_fn, device_count=0, batch_size=0)
start = time.time()
# Bayesian and infinite-time gradient descent inference with infinite network.
#for i in range(10):
predict_fn = \
nt.predict.gradient_descent_mse_ensemble(kernel_fn, train_embedding, data_set['Y_train'],
diag_reg_absolute_scale=True, learning_rate=1, diag_reg=1e-3) #1e0 1e-3
nngp_mean, nngp_covariance = predict_fn(x_test=test_embedding,
get='nngp',
compute_cov=True)
#fx_test_nngp.block_until_ready()
#fx_test_ntk.block_until_ready()
duration = time.time() - start
print('Kernel construction and inference done in %s seconds.' % duration)
# Print out accuracy and loss for infinite network predictions.
loss = lambda fx, y_hat: 0.5 * np.mean((fx - y_hat)**2)
utils.print_summary('NNGP test',
data_set['Y_test'],
nngp_mean,
None,
loss,
nngp_covariance,
binary=binary)
print(f"Graph {graph} at ec1 vs {epoch} with feature {feature_group}")
X, y, group = filter_dataframe(graph, epoch, feature_group)
# Create a pipeline
pipe = Pipeline([('imputer',
SimpleImputer(missing_values=np.nan, strategy='mean')),
('scaler', StandardScaler()),
('clf', DummyEstimator())]) # Placeholder Estimator
# We will use as many processor as possible for the gridsearch
gs = GridSearchCV(pipe, search_space, cv=LeaveOneGroupOut(), n_jobs=-1)
accuracies, f1s, cms, best_params = classify_loso_model_selection(
X, y, group, gs)
# Saving the performance metrics
clf_data = {
'accuracies': accuracies,
'f1s': f1s,
'cms': cms,
'best_params': best_params,
}
best_clf_file = open(best_clf_filename, 'ab')
pickle.dump(clf_data, best_clf_file)
best_clf_file.close()
# Print out the summary in the console
print_summary(accuracies, group)
def show_reviews_per_person_dist(reviews_counts, idcol):
count_table = utils.create_value_counts_table(reviews_counts, 'num_of_reviews', 'num_of_guest_reviews')
print(count_table.head(10))
utils.print_summary(reviews_counts['num_of_reviews'], False)
def main():
try:
# init spark
spark = get_spark(app_name="pyspark-xgb")
# get logger
logger = get_logger(spark, "app")
# load data
train = spark.read.csv(DATASET_PATH + "/iris_train.csv",
get_mtrain_schema(),
header=True)
test = spark.read.csv(DATASET_PATH + "/iris_test.csv",
get_mtrain_schema(),
header=True)
# preprocess
# get label encode result from csv, since StringIndexer will get different result
STR_LABEL = 'class'
LABEL = 'label'
FEATURES = 'features'
N_CLASS = 3
features = [c for c in train.columns if c not in [STR_LABEL, LABEL]]
assembler = VectorAssembler(inputCols=features, outputCol=FEATURES)
pipeline = Pipeline(stages=[assembler])
preprocess = pipeline.fit(train)
train = preprocess.transform(train).select(FEATURES, LABEL)
test = preprocess.transform(test).select(FEATURES, LABEL)
# set param map
xgb_params = {
"eta": 0.1, "eval_metric": "mlogloss",
"gamma": 0, "max_depth": 5, "min_child_weight": 1.0,
"objective": "multi:softprob", "seed": 0,
"num_class": N_CLASS,
# xgboost4j only
"num_round": 100, "num_early_stopping_rounds": 10,
"maximize_evaluation_metrics": False,
"num_workers": 1, "use_external_memory": False,
"missing": np.nan,
}
scala_map = spark._jvm.PythonUtils.toScalaMap(xgb_params)
# set evaluation set
eval_set = {'eval': test._jdf}
scala_eval_set = spark._jvm.PythonUtils.toScalaMap(eval_set)
logger.info('training')
j = JavaWrapper._new_java_obj(
"ml.dmlc.xgboost4j.scala.spark.XGBoostClassifier", scala_map) \
.setFeaturesCol(FEATURES).setLabelCol(LABEL) \
.setEvalSets(scala_eval_set)
jmodel = j.fit(train._jdf)
print_summary(jmodel)
# get validation metric
preds = jmodel.transform(test._jdf)
pred = DataFrame(preds, spark)
slogloss = pred.withColumn('log_loss', udf_logloss(LABEL, 'probability', N_CLASS)) \
.agg({"log_loss": "mean"}).collect()[0]['avg(log_loss)']
logger.info('[xgboost4j] valid logloss: {}'.format(slogloss))
# save model - using native booster for single node library to read
model_path = MODEL_PATH + '/model.bin'
logger.info('save model to {}'.format(model_path))
jbooster = jmodel.nativeBooster()
jbooster.saveModel(model_path)
# get feature score
imp_type = "gain"
feature_map_path = MODEL_PATH + '/feature.map'
create_feature_map(feature_map_path, features)
jfeatureMap = jbooster.getScore(feature_map_path, imp_type)
f_imp = dict()
for feature in features:
if not jfeatureMap.get(feature).isEmpty():
f_imp[feature] = jfeatureMap.get(feature).get()
feature_imp_path = MODEL_PATH + '/feature.imp'
create_feature_imp(feature_imp_path, f_imp)
# [Optional] load model training by xgboost, predict and get validation metric
local_model_path = LOCAL_MODEL_PATH + '/model.bin'
if os.path.exists(local_model_path):
logger.info('load model from {}'.format(local_model_path))
scala_xgb = spark.sparkContext._jvm.ml.dmlc.xgboost4j.scala.XGBoost
jbooster = scala_xgb.loadModel(local_model_path)
# uid, num_class, booster
xgb_cls_model = JavaWrapper._new_java_obj(
"ml.dmlc.xgboost4j.scala.spark.XGBoostClassificationModel",
"xgbc", N_CLASS, jbooster)
jpred = xgb_cls_model.transform(test._jdf)
pred = DataFrame(jpred, spark)
slogloss = pred.withColumn('log_loss', udf_logloss(LABEL, 'probability', N_CLASS)) \
.agg({"log_loss": "mean"}).collect()[0]['avg(log_loss)']
logger.info('[xgboost] valid logloss: {}'.format(slogloss))
else:
logger.info(
"local model is not exist, call python_xgb/train_multi.py to get the model "
"and compare logloss between xgboost and xgboost4j"
)
except Exception:
logger.error(traceback.print_exc())
finally:
# stop spark
spark.stop()
def main(argv=None):
tstart = time.time()
# Arguments
parser = argparse.ArgumentParser(
description=
'FACIL - Framework for Analysis of Class Incremental Learning')
# miscellaneous args
parser.add_argument('--gpu',
type=int,
default=0,
help='GPU (default=%(default)s)')
parser.add_argument('--results-path',
type=str,
default='../results',
help='Results path (default=%(default)s)')
parser.add_argument('--exp-name',
default=None,
type=str,
help='Experiment name (default=%(default)s)')
parser.add_argument('--seed',
type=int,
default=0,
help='Random seed (default=%(default)s)')
parser.add_argument(
'--log',
default=['disk'],
type=str,
choices=['disk', 'tensorboard'],
help='Loggers used (disk, tensorboard) (default=%(default)s)',
nargs='*',
metavar="LOGGER")
parser.add_argument('--save-models',
action='store_true',
help='Save trained models (default=%(default)s)')
parser.add_argument('--last-layer-analysis',
action='store_true',
help='Plot last layer analysis (default=%(default)s)')
parser.add_argument(
'--no-cudnn-deterministic',
action='store_true',
help='Disable CUDNN deterministic (default=%(default)s)')
# dataset args
parser.add_argument('--datasets',
default=['cifar100'],
type=str,
choices=list(dataset_config.keys()),
help='Dataset or datasets used (default=%(default)s)',
nargs='+',
metavar="DATASET")
parser.add_argument(
'--num-workers',
default=4,
type=int,
required=False,
help=
'Number of subprocesses to use for dataloader (default=%(default)s)')
parser.add_argument(
'--pin-memory',
default=False,
type=bool,
required=False,
help=
'Copy Tensors into CUDA pinned memory before returning them (default=%(default)s)'
)
parser.add_argument(
'--batch-size',
default=64,
type=int,
required=False,
help='Number of samples per batch to load (default=%(default)s)')
parser.add_argument(
'--num-tasks',
default=4,
type=int,
required=False,
help='Number of tasks per dataset (default=%(default)s)')
parser.add_argument(
'--nc-first-task',
default=None,
type=int,
required=False,
help='Number of classes of the first task (default=%(default)s)')
parser.add_argument(
'--use-valid-only',
action='store_true',
help='Use validation split instead of test (default=%(default)s)')
parser.add_argument(
'--stop-at-task',
default=0,
type=int,
required=False,
help='Stop training after specified task (default=%(default)s)')
# model args
parser.add_argument('--network',
default='resnet32',
type=str,
choices=allmodels,
help='Network architecture used (default=%(default)s)',
metavar="NETWORK")
parser.add_argument(
'--keep-existing-head',
action='store_true',
help='Disable removing classifier last layer (default=%(default)s)')
parser.add_argument('--pretrained',
action='store_true',
help='Use pretrained backbone (default=%(default)s)')
# training args
parser.add_argument('--approach',
default='finetuning',
type=str,
choices=approach.__all__,
help='Learning approach used (default=%(default)s)',
metavar="APPROACH")
parser.add_argument(
'--nepochs',
default=200,
type=int,
required=False,
help='Number of epochs per training session (default=%(default)s)')
parser.add_argument('--lr',
default=0.1,
type=float,
required=False,
help='Starting learning rate (default=%(default)s)')
parser.add_argument('--lr-min',
default=1e-4,
type=float,
required=False,
help='Minimum learning rate (default=%(default)s)')
parser.add_argument(
'--lr-factor',
default=3,
type=float,
required=False,
help='Learning rate decreasing factor (default=%(default)s)')
parser.add_argument(
'--lr-patience',
default=5,
type=int,
required=False,
help=
'Maximum patience to wait before decreasing learning rate (default=%(default)s)'
)
parser.add_argument('--clipping',
default=10000,
type=float,
required=False,
help='Clip gradient norm (default=%(default)s)')
parser.add_argument('--momentum',
default=0.0,
type=float,
required=False,
help='Momentum factor (default=%(default)s)')
parser.add_argument('--weight-decay',
default=0.0,
type=float,
required=False,
help='Weight decay (L2 penalty) (default=%(default)s)')
parser.add_argument('--warmup-nepochs',
default=0,
type=int,
required=False,
help='Number of warm-up epochs (default=%(default)s)')
parser.add_argument(
'--warmup-lr-factor',
default=1.0,
type=float,
required=False,
help='Warm-up learning rate factor (default=%(default)s)')
parser.add_argument(
'--multi-softmax',
action='store_true',
help='Apply separate softmax for each task (default=%(default)s)')
parser.add_argument(
'--fix-bn',
action='store_true',
help='Fix batch normalization after first task (default=%(default)s)')
parser.add_argument(
'--eval-on-train',
action='store_true',
help='Show train loss and accuracy (default=%(default)s)')
# gridsearch args
parser.add_argument(
'--gridsearch-tasks',
default=-1,
type=int,
help=
'Number of tasks to apply GridSearch (-1: all tasks) (default=%(default)s)'
)
# Args -- Incremental Learning Framework
args, extra_args = parser.parse_known_args(argv)
args.results_path = os.path.expanduser(args.results_path)
base_kwargs = dict(nepochs=args.nepochs,
lr=args.lr,
lr_min=args.lr_min,
lr_factor=args.lr_factor,
lr_patience=args.lr_patience,
clipgrad=args.clipping,
momentum=args.momentum,
wd=args.weight_decay,
multi_softmax=args.multi_softmax,
wu_nepochs=args.warmup_nepochs,
wu_lr_factor=args.warmup_lr_factor,
fix_bn=args.fix_bn,
eval_on_train=args.eval_on_train)
if args.no_cudnn_deterministic:
print('WARNING: CUDNN Deterministic will be disabled.')
utils.cudnn_deterministic = False
utils.seed_everything(seed=args.seed)
print('=' * 108)
print('Arguments =')
for arg in np.sort(list(vars(args).keys())):
print('\t' + arg + ':', getattr(args, arg))
print('=' * 108)
# Args -- CUDA
if torch.cuda.is_available():
torch.cuda.set_device(args.gpu)
device = 'cuda'
else:
print('WARNING: [CUDA unavailable] Using CPU instead!')
device = 'cpu'
# Multiple gpus
# if torch.cuda.device_count() > 1:
# self.C = torch.nn.DataParallel(C)
# self.C.to(self.device)
####################################################################################################################
# Args -- Network
from networks.network import LLL_Net
if args.network in tvmodels: # torchvision models
tvnet = getattr(importlib.import_module(name='torchvision.models'),
args.network)
if args.network == 'googlenet':
init_model = tvnet(pretrained=args.pretrained, aux_logits=False)
else:
init_model = tvnet(pretrained=args.pretrained)
set_tvmodel_head_var(init_model)
else: # other models declared in networks package's init
net = getattr(importlib.import_module(name='networks'), args.network)
# WARNING: fixed to pretrained False for other model (non-torchvision)
init_model = net(pretrained=False)
# Args -- Continual Learning Approach
from approach.incremental_learning import Inc_Learning_Appr
Appr = getattr(importlib.import_module(name='approach.' + args.approach),
'Appr')
assert issubclass(Appr, Inc_Learning_Appr)
appr_args, extra_args = Appr.extra_parser(extra_args)
print('Approach arguments =')
for arg in np.sort(list(vars(appr_args).keys())):
print('\t' + arg + ':', getattr(appr_args, arg))
print('=' * 108)
# Args -- Exemplars Management
from datasets.exemplars_dataset import ExemplarsDataset
Appr_ExemplarsDataset = Appr.exemplars_dataset_class()
if Appr_ExemplarsDataset:
assert issubclass(Appr_ExemplarsDataset, ExemplarsDataset)
appr_exemplars_dataset_args, extra_args = Appr_ExemplarsDataset.extra_parser(
extra_args)
print('Exemplars dataset arguments =')
for arg in np.sort(list(vars(appr_exemplars_dataset_args).keys())):
print('\t' + arg + ':', getattr(appr_exemplars_dataset_args, arg))
print('=' * 108)
else:
appr_exemplars_dataset_args = argparse.Namespace()
# Args -- GridSearch
if args.gridsearch_tasks > 0:
from gridsearch import GridSearch
gs_args, extra_args = GridSearch.extra_parser(extra_args)
Appr_finetuning = getattr(
importlib.import_module(name='approach.finetuning'), 'Appr')
assert issubclass(Appr_finetuning, Inc_Learning_Appr)
GridSearch_ExemplarsDataset = Appr.exemplars_dataset_class()
print('GridSearch arguments =')
for arg in np.sort(list(vars(gs_args).keys())):
print('\t' + arg + ':', getattr(gs_args, arg))
print('=' * 108)
assert len(extra_args) == 0, "Unused args: {}".format(' '.join(extra_args))
####################################################################################################################
# Log all arguments
full_exp_name = reduce(
(lambda x, y: x[0] + y[0]),
args.datasets) if len(args.datasets) > 0 else args.datasets[0]
full_exp_name += '_' + args.approach
if args.exp_name is not None:
full_exp_name += '_' + args.exp_name
logger = MultiLogger(args.results_path,
full_exp_name,
loggers=args.log,
save_models=args.save_models)
logger.log_args(
argparse.Namespace(**args.__dict__, **appr_args.__dict__,
**appr_exemplars_dataset_args.__dict__))
# Loaders
utils.seed_everything(seed=args.seed)
trn_loader, val_loader, tst_loader, taskcla = get_loaders(
args.datasets,
args.num_tasks,
args.nc_first_task,
args.batch_size,
num_workers=args.num_workers,
pin_memory=args.pin_memory)
# Apply arguments for loaders
if args.use_valid_only:
tst_loader = val_loader
max_task = len(taskcla) if args.stop_at_task == 0 else args.stop_at_task
# Network and Approach instances
utils.seed_everything(seed=args.seed)
net = LLL_Net(init_model, remove_existing_head=not args.keep_existing_head)
utils.seed_everything(seed=args.seed)
# taking transformations and class indices from first train dataset
first_train_ds = trn_loader[0].dataset
transform, class_indices = first_train_ds.transform, first_train_ds.class_indices
appr_kwargs = {**base_kwargs, **dict(logger=logger, **appr_args.__dict__)}
if Appr_ExemplarsDataset:
appr_kwargs['exemplars_dataset'] = Appr_ExemplarsDataset(
transform, class_indices, **appr_exemplars_dataset_args.__dict__)
utils.seed_everything(seed=args.seed)
appr = Appr(net, device, **appr_kwargs)
# GridSearch
if args.gridsearch_tasks > 0:
ft_kwargs = {
**base_kwargs,
**dict(logger=logger,
exemplars_dataset=GridSearch_ExemplarsDataset(
transform, class_indices))
}
appr_ft = Appr_finetuning(net, device, **ft_kwargs)
gridsearch = GridSearch(appr_ft, args.seed, gs_args.gridsearch_config,
gs_args.gridsearch_acc_drop_thr,
gs_args.gridsearch_hparam_decay,
gs_args.gridsearch_max_num_searches)
# Loop tasks
print(taskcla)
acc_taw = np.zeros((max_task, max_task))
acc_tag = np.zeros((max_task, max_task))
forg_taw = np.zeros((max_task, max_task))
forg_tag = np.zeros((max_task, max_task))
for t, (_, ncla) in enumerate(taskcla):
# Early stop tasks if flag
if t >= max_task:
continue
print('*' * 108)
print('Task {:2d}'.format(t))
print('*' * 108)
# Add head for current task
net.add_head(taskcla[t][1])
net.to(device)
# GridSearch
if t < args.gridsearch_tasks:
# Search for best finetuning learning rate -- Maximal Plasticity Search
print('LR GridSearch')
best_ft_acc, best_ft_lr = gridsearch.search_lr(
appr.model, t, trn_loader[t], val_loader[t])
# Apply to approach
appr.lr = best_ft_lr
gen_params = gridsearch.gs_config.get_params('general')
for k, v in gen_params.items():
if not isinstance(v, list):
setattr(appr, k, v)
# Search for best forgetting/intransigence tradeoff -- Stability Decay
print('Trade-off GridSearch')
best_tradeoff, tradeoff_name = gridsearch.search_tradeoff(
args.approach, appr, t, trn_loader[t], val_loader[t],
best_ft_acc)
# Apply to approach
if tradeoff_name is not None:
setattr(appr, tradeoff_name, best_tradeoff)
print('-' * 108)
# Train
appr.train(t, trn_loader[t], val_loader[t])
print('-' * 108)
# Test
for u in range(t + 1):
test_loss, acc_taw[t, u], acc_tag[t,
u] = appr.eval(u, tst_loader[u])
if u < t:
forg_taw[t, u] = acc_taw[:t, u].max(0) - acc_taw[t, u]
forg_tag[t, u] = acc_tag[:t, u].max(0) - acc_tag[t, u]
print(
'>>> Test on task {:2d} : loss={:.3f} | TAw acc={:5.1f}%, forg={:5.1f}%'
'| TAg acc={:5.1f}%, forg={:5.1f}% <<<'.format(
u, test_loss, 100 * acc_taw[t, u], 100 * forg_taw[t, u],
100 * acc_tag[t, u], 100 * forg_tag[t, u]))
logger.log_scalar(task=t,
iter=u,
name='loss',
group='test',
value=test_loss)
logger.log_scalar(task=t,
iter=u,
name='acc_taw',
group='test',
value=100 * acc_taw[t, u])
logger.log_scalar(task=t,
iter=u,
name='acc_tag',
group='test',
value=100 * acc_tag[t, u])
logger.log_scalar(task=t,
iter=u,
name='forg_taw',
group='test',
value=100 * forg_taw[t, u])
logger.log_scalar(task=t,
iter=u,
name='forg_tag',
group='test',
value=100 * forg_tag[t, u])
# Save
print('Save at ' + os.path.join(args.results_path, full_exp_name))
logger.log_result(acc_taw, name="acc_taw", step=t)
logger.log_result(acc_tag, name="acc_tag", step=t)
logger.log_result(forg_taw, name="forg_taw", step=t)
logger.log_result(forg_tag, name="forg_tag", step=t)
logger.save_model(net.state_dict(), task=t)
logger.log_result(acc_taw.sum(1) /
np.tril(np.ones(acc_taw.shape[0])).sum(1),
name="avg_accs_taw",
step=t)
logger.log_result(acc_tag.sum(1) /
np.tril(np.ones(acc_tag.shape[0])).sum(1),
name="avg_accs_tag",
step=t)
aux = np.tril(
np.repeat([[tdata[1] for tdata in taskcla[:max_task]]],
max_task,
axis=0))
logger.log_result((acc_taw * aux).sum(1) / aux.sum(1),
name="wavg_accs_taw",
step=t)
logger.log_result((acc_tag * aux).sum(1) / aux.sum(1),
name="wavg_accs_tag",
step=t)
# Last layer analysis
if args.last_layer_analysis:
weights, biases = last_layer_analysis(net.heads,
t,
taskcla,
y_lim=True)
logger.log_figure(name='weights', iter=t, figure=weights)
logger.log_figure(name='bias', iter=t, figure=biases)
# Output sorted weights and biases
weights, biases = last_layer_analysis(net.heads,
t,
taskcla,
y_lim=True,
sort_weights=True)
logger.log_figure(name='weights', iter=t, figure=weights)
logger.log_figure(name='bias', iter=t, figure=biases)
# Print Summary
utils.print_summary(acc_taw, acc_tag, forg_taw, forg_tag)
print('[Elapsed time = {:.1f} h]'.format(
(time.time() - tstart) / (60 * 60)))
print('Done!')
return acc_taw, acc_tag, forg_taw, forg_tag, logger.exp_path
parser.model.set_weights(w)
del w
else:
print('Initiate model', time.strftime("%Y-%m-%d %H:%M:%S"))
parser = Parser(params)
parser.fit(train_data)
parser.model.summary()
print('Save model', time.strftime("%Y-%m-%d %H:%M:%S"))
joblib.dump(parser, params.model_file)
if params.evaluate:
print('Predict on train', time.strftime("%Y-%m-%d %H:%M:%S"))
pred_train = parser.predict(train_data)
print_summary(pred_train, train_data)
if params.valid is not None:
print('Load valid data', time.strftime("%Y-%m-%d %H:%M:%S"))
valid_data = loader.load(params.valid)
print('Predict on valid', time.strftime("%Y-%m-%d %H:%M:%S"))
pred_valid = parser.predict(valid_data)
print_summary(pred_valid, valid_data)
elif params.mode == 'multitrain':
if params.evaluate and params.valid is not None:
print('Load valid data', time.strftime("%Y-%m-%d %H:%M:%S"))
valid_data = loader.load(params.valid)
if params.continue_training:
plt.xlabel("CLASS", fontsize=16)
plt.ylabel("COUNT", fontsize=16)
x = np.arange(len(count["CLASS"]))
y = np.array(list(count["COUNT"]))
for a,b in zip(x,y):
plt.text(a, b+0.05, '%.0f' % b, ha='center', va= 'bottom',fontsize=16)
plt.show()
# Baseline
baseline_weights, _, baseline_test_logits = train_nn(X_train, y_train, X_test=X_test, y_test=y_test, epoch=2000, batch_size=1000)
baseline_accuracy = (baseline_test_logits.argmax(axis=1)==y_test.argmax(axis=1)).mean()
_, baseline_names_logits, _ = train_nn(test_sequences_embed, y_train=None, weights=baseline_weights, epoch=0)
test_df['baseline_logits'] = baseline_names_logits[:,1] - baseline_names_logits[:,0]
print_summary(test_df, 'baseline', baseline_accuracy)
# SenSR_0 expert
expert_sens_directions = np.copy(test_sequences_embed)
sensr0_expert_weights, _, sensr0_expert_test_logits = train_fair_nn(X_train, y_train, expert_sens_directions, X_test = X_test, y_test=y_test)
sensr0_expert_accuracy = (sensr0_expert_test_logits.argmax(axis=1)==y_test.argmax(axis=1)).mean()
_, sensr0_expert_names_logits, _ = train_nn(test_sequences_embed, y_train=None, weights=sensr0_expert_weights, epoch=0)
test_df['sensr0_expert_logits'] = sensr0_expert_names_logits[:,1] - sensr0_expert_names_logits[:,0]
print_summary(test_df, 'sensr0_expert', sensr0_expert_accuracy)
# Calculate quantile
class_1 = test_df.loc[test_df['result'] == 'Class 1']
class_2 = test_df.loc[test_df['result'] == 'Class 2']
Q1 = class_1['baseline_logits'].quantile(0.25)
Q2 = class_1['baseline_logits'].quantile(0.5)
Q3 = class_1['baseline_logits'].quantile(0.75)
def train_primary(model_name, df_name, max_len, train_ratio, text_feature,
embeddings_version, embeddings_path, config_primary,
models_path, models_df):
device = utils.find_device()
loss_function, labels_num, dropout, epochs_num, threshold, \
hid_dim_lstm, lin_output_dim, lr, batch_size, momentum = \
utils.read_config_primary(config_primary, "train_primary")
train_df, val_df = utils.split_train_val(df_name, train_ratio)
train_dataloader, val_dataloader = utils.create_dataloaders_train(
train_df, val_df, text_feature, embeddings_version, max_len,
batch_size)
model = get_model(model_name, embeddings_path, hid_dim_lstm, loss_function,
labels_num, dropout, lin_output_dim)
model.to(device)
optimizer = AdamW(model.parameters(), lr=2e-5, eps=1e-8)
train_len = len(train_dataloader)
val_len = len(val_dataloader)
train_accuracy_list, train_loss_list, accuracy_list_val, loss_list_val = [], [], [], []
best_val_acc = 0.0
total_steps = len(train_dataloader) * epochs_num
scheduler = get_linear_schedule_with_warmup(optimizer,
num_warmup_steps=0,
num_training_steps=total_steps)
for epoch in range(epochs_num):
utils.print_epochs_progress(epoch, epochs_num)
start_train = time.clock()
acc_num_train = 0.0
loss_scalar = 0.0
model.train()
optimizer.zero_grad()
t0 = time.time()
predictions_dict_train = dict()
for batch_idx, batch in enumerate(train_dataloader):
if batch_idx % 10 == 0 and not batch_idx == 0:
utils.print_batches_progress(t0, batch_idx, train_dataloader)
input_ids = batch[0].to(device, dtype=torch.long)
masks = batch[1].to(device, dtype=torch.long)
labels = batch[2].to(device, dtype=torch.long)
hadm_id = batch[3]
model.zero_grad()
loss, predictions, probabilities = model(input_ids, masks, labels)
loss_scalar += loss.item()
loss.backward()
torch.nn.utils.clip_grad_norm_(model.parameters(), max_norm=1.0)
acc_num_train += utils.add_accuracy(predictions, labels)
optimizer.step()
scheduler.step()
predictions_dict_train = utils.update_predictions_dict(
predictions_dict_train, hadm_id, predictions, probabilities,
labels)
loss_scalar /= train_len
end_train = time.clock()
total_time = end_train - start_train
utils.print_train_epoch(epoch, acc_num_train, train_len, loss_scalar,
total_time)
train_loss_list.append(round(float(loss_scalar), 3))
train_accuracy_list.append(round(float(acc_num_train / train_len), 3))
utils.print_train_epoch_end(t0)
val_acc, val_loss, predictions_dict_val = evaluate_val(
model, val_dataloader, device, val_len)
accuracy_list_val.append(val_acc)
loss_list_val.append(val_loss)
if val_acc > best_val_acc:
torch.save(model.state_dict(), models_path + 'primary.pkl')
best_val_acc = val_acc
utils.save_predictions_to_df(predictions_dict_train, models_path,
'train', 'primary')
utils.save_predictions_to_df(predictions_dict_val, models_path,
'validation', 'primary')
utils.save_model(models_df, 'primary', hid_dim_lstm, labels_num,
loss_function, dropout, lin_output_dim, lr, epochs_num,
batch_size, momentum, best_val_acc)
utils.print_summary(models_path, 'primary', accuracy_list_val)
test_acc, predictions_dict_test = evaluate_test(model, device,
embeddings_version,
max_len, text_feature,
batch_size)
utils.save_predictions_to_df(predictions_dict_test, models_path, 'test',
'primary')
utils.print_test_results(models_path, 'primary', test_acc)
return models_df
def run_HAC(FLAGS, env, agent):
experiment = Experiment(api_key="M03EcOc9o9kiG95hws4mq1uqI",
project_name="HAC",
workspace="antonwiehe")
# Print task summary
print_summary(FLAGS, env)
# Determine training mode. If not testing and not solely training, interleave training and testing to track progress
mix_train_test = False
if not FLAGS.test and not FLAGS.train_only:
mix_train_test = True
for batch in range(NUM_BATCH):
num_episodes = agent.other_params["num_exploration_episodes"]
# Evaluate policy every TEST_FREQ batches if interleaving training and testing
if mix_train_test and batch % TEST_FREQ == 0:
print("\n--- TESTING ---")
agent.FLAGS.test = True
num_episodes = num_test_episodes
# Reset successful episode counter
successful_episodes = 0
for episode in range(num_episodes):
print("\nBatch %d, Episode %d" % (batch, episode))
# Train for an episode
success = agent.train(env, episode)
if success:
print("Batch %d, Episode %d End Goal Achieved\n" %
(batch, episode))
# Increment successful episode counter if applicable
if mix_train_test and batch % TEST_FREQ == 0:
successful_episodes += 1
# Save agent
agent.save_model(episode)
# Finish evaluating policy if tested prior batch
if mix_train_test and batch % TEST_FREQ == 0:
# Log performance
success_rate = successful_episodes / num_test_episodes * 100
print("\nTesting Success Rate %.2f%%" % success_rate)
agent.log_performance(success_rate)
agent.FLAGS.test = False
experiment.set_step(batch)
experiment.log_metric("Success rate", success_rate)
success_list.append(success_rate)
with open("successRates.csv", 'w', newline='') as myfile:
wr = csv.writer(myfile, quoting=csv.QUOTE_ALL)
wr.writerow(success_list)
if success_rate > 95:
print("Success rate over 95\%!")
break
print("\n--- END TESTING ---\n")
