def main():
# capture the config path from the run arguments
# then process configuration file
SRC_DIR = os.getcwd()
RADAR_DIR = os.path.join(SRC_DIR, os.pardir)
config = preprocess_meta_data(SRC_DIR)
exp_name = config.exp_name
# for graph_dir and log file
now = datetime.now()
date = now.strftime("%Y_%m_%d_%H_%M_%S")
exp_name_time = '{}_{}'.format(exp_name, date)
# visualize training performance
graph_path = os.path.join(RADAR_DIR, 'graphs', exp_name_time)
if os.path.exists(graph_path) is False:
os.makedirs(graph_path)
LOG_DIR = os.path.join(RADAR_DIR, 'logs')
if os.path.exists(LOG_DIR) is False:
os.makedirs(LOG_DIR)
log_path = '{}/{}.log'.format(LOG_DIR, exp_name_time)
'''
Configure multiprocess
'''
strategy = tf.distribute.MirroredStrategy()
if strategy.num_replicas_in_sync != 1:
config.__setattr__("batch_size", config.batch_size * strategy.num_replicas_in_sync)
config = adjust_input_size(config)
# assert configurations
assert not(config.learn_background and (config.with_rect_augmentation or config.with_preprocess_rect_augmentation))
# assert not(config.background_implicit_inference)
# load the data
data = load_data(config)
with strategy.scope():
# create a model
model = build_model(config)
# create trainer
trainer = build_trainer(model, data, config)
# train the model
history = trainer.train()
# evaluate model
eval_res = trainer.evaluate()
SUB_DIR = os.path.join(RADAR_DIR,'submission_files')
BEST_RESULT_DIR = os.path.join(RADAR_DIR, 'best_preformance_history')
if os.path.exists(SUB_DIR) is False:
os.makedirs(SUB_DIR)
sub_path = "{}/submission_{}.csv".format(SUB_DIR,exp_name_time)
test_model(model['train'], sub_path, SRC_DIR, config,BEST_RESULT_DIR)
if config.learn_background is False:
result_data = analyse_model_performance(model, data, history, config, graph_path=graph_path, res_dir=exp_name_time)
result_data['Log path'] = log_path
result_data['Graph path'] = graph_path
result_data['Submission path'] = sub_path
result_data['Model name'] = config.model_name
result_data['Exp name'] = config.exp_name
result_data['Snr type'] = config.snr_type
# compare model performance
if os.path.exists(BEST_RESULT_DIR) is False:
os.makedirs(BEST_RESULT_DIR)
compare_to_best_model_performance(result_data, model, BEST_RESULT_DIR, config)
PREVIOUS_MODELS_DIR = os.path.join(RADAR_DIR, 'previous_models_files')
if config.save_model is True:
if os.path.exists(PREVIOUS_MODELS_DIR) is False:
os.makedirs(PREVIOUS_MODELS_DIR)
os.chdir(PREVIOUS_MODELS_DIR)
save_model(name='{}_{}_{}'.format(config.model_name,config.exp_name,exp_name_time), model=model['train'])
#if config.save_history_buffer is True:
print('#' * 70)
print('log file is located at {}'.format(log_path))
print('graphs are located at {}'.format(graph_path))
print('submission file is at: {}'.format(sub_path))
print('')
uniques_events = np.unique(data_frame[[
'proton_TRACK_Key', 'Kminus_TRACK_Key', 'mu1_TRACK_Key',
'tauMu_TRACK_Key', 'proton_MC_MOTHER_KEY', 'Kminus_MC_MOTHER_KEY',
'mu1_MC_MOTHER_KEY', 'tauMu_MC_MOTHER_KEY', 'proton_MC_GD_MOTHER_KEY',
'Kminus_MC_GD_MOTHER_KEY', 'mu1_MC_GD_MOTHER_KEY',
'tauMu_MC_GD_MOTHER_KEY', 'proton_MC_GD_GD_MOTHER_KEY',
'Kminus_MC_GD_GD_MOTHER_KEY', 'mu1_MC_GD_GD_MOTHER_KEY',
'tauMu_MC_GD_GD_MOTHER_KEY'
]].values.tolist(),
axis=0)
print(len(uniques_mu1), len(uniques_k), len(uniques_p), len(uniques_taumu),
len(uniques_events), len(data_frame))
if __name__ == '__main__':
a = load_data(df_name='B_MC')
df = b_cleaning(a)
diff_candidates_check(df)
df['stretched'] = get_stretched_pikmu_mass(
df, [['proton_P', 'pi'], ['Kminus_P', 'K'], ['mu1_P', 'mu']])
df['stretched2'] = get_stretched_pikmu_mass(
df, [['proton_P', 'pi'], ['Kminus_P', 'K'], ['tauMu_P', 'mu']])
df1 = df[np.sign(df['proton_TRUEID']) != np.sign(df['mu1_TRUEID'])]
df2 = df[np.sign(df['proton_TRUEID']) == np.sign(df['mu1_TRUEID'])]
n = len(df1) + len(df2)
print(n)
print(len(df1[df1['stretched'] > 2800]) / len(df1))
print(len(df1[df1['stretched'] > 3000]) / len(df1))
print(len(df2[df2['stretched2'] > 2800]) / len(df2))
print(len(df2[df2['stretched2'] > 3000]) / len(df2))
print((len(df2[df2['stretched2'] > 2800]) +
def train(solver, dataset_name):
model_name = solver['model_name']
print('Preparing to train on {} data...'.format(dataset_name))
nb_epoch = solver['nb_epoch']
batch_size = solver['batch_size']
completed_epochs = solver['completed_epochs']
skip = solver['skip']
np.random.seed(1337) # for reproducibility
dw = solver['dw']
dh = solver['dh']
data_shape = dw * dh
nc = len(dataset.ids()) # categories + background
autoenc, model_name = autoencoder(nc=nc, input_shape=(dw, dh))
if 'h5file' in solver:
h5file = solver['h5file']
print('Loading model {}'.format(h5file))
h5file, ext = os.path.splitext(h5file)
autoenc.load_weights(h5file + ext)
else:
autoenc = transfer_weights(autoenc)
if K.backend() == 'tensorflow':
print('Tensorflow backend detected; Applying memory usage constraints')
ss = K.tf.Session(config=K.tf.ConfigProto(gpu_options=K.tf.GPUOptions(
allow_growth=True)))
K.set_session(ss)
ss.run(K.tf.global_variables_initializer())
print('Done loading {} model!'.format(model_name))
experiment_dir = os.path.join('models', dataset_name, model_name)
log_dir = os.path.join(experiment_dir, 'logs')
checkpoint_dir = os.path.join(experiment_dir, 'weights')
train_gen = load_data(dataset,
data_dir=os.path.join('data', dataset_name),
batch_size=batch_size,
nc=nc,
target_hw=(dw, dh),
data_type='train2014',
shuffle=True)
nb_train_samples = train_gen.next() # first generator item is the count
val_gen = load_data(dataset,
data_dir=os.path.join('data', dataset_name),
batch_size=batch_size,
nc=nc,
target_hw=(dw, dh),
data_type='val2014',
sample_size=nb_train_samples // 10)
nb_val_samples = val_gen.next() # first generator item is the count
autoenc.fit_generator(generator=train_gen,
samples_per_epoch=nb_train_samples,
nb_epoch=nb_epoch,
verbose=1,
callbacks=callbacks(log_dir, checkpoint_dir,
model_name),
validation_data=val_gen,
nb_val_samples=nb_val_samples,
initial_epoch=completed_epochs) # start from epoch e
import numpy as np
import pandas as pd
import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
from sklearn.metrics import confusion_matrix
from torch.optim import lr_scheduler
from torchvision import models
sys.path.append('../')
from data.data_loader import load_data
from util.file_utils import mkdirs_if_not_exist
from config.cfg import cfg
dataloaders = load_data('FGVC')
dataset_sizes = {x: len(dataloaders[x]) for x in ['train', 'val', 'test']}
batch_size = cfg['config']['FGVC']['batch_size']
def train_model(model, train_dataloader, test_dataloader, criterion, optimizer, scheduler, num_epochs,
inference=False):
"""
train model
:param model:
:param train_dataloader:
:param test_dataloader:
:param criterion:
:param optimizer:
:param scheduler:
:param num_epochs:
from background_reduction.b_MC_reduction import b_cleaning
from background_reduction.data_reduction import reduce_background
from data.data_loader import load_data
from get_vertex import obtain_lb_line_of_flight, transverse_momentum, line_plane_intersection, tau_momentum_mass, \
plot_result, plot_b_result
df_name = 'Lb_data'
# df_name = 'B_MC'
a = load_data(df_name=df_name)
a.dropna(inplace=True)
if df_name == 'Lb_data':
df = reduce_background(a, True)
elif df_name == 'B_MC':
df = b_cleaning(a)
else:
df = a
df = df.reset_index(drop=True)
df = obtain_lb_line_of_flight(df)
df = transverse_momentum(df)
df = line_plane_intersection(df)
df = tau_momentum_mass(df)
if df_name != 'B_MC':
plot_result(df)
else:
plot_b_result(df)
def run():
args = load_config()
logger.add(
os.path.join(
'logs', '{}_model_{}_codelength_{}_mu_{}_nu_{}_eta1_'
'{}_eta2_{}_eta3_{}_topk_{}.log'.format(
args.dataset,
args.arch,
','.join([str(c) for c in args.code_length]),
args.mu,
args.nu,
args.eta1,
args.eta2,
args.eta3,
args.topk,
)),
rotation='500 MB',
level='INFO',
)
logger.info(args)
torch.backends.cudnn.benchmark = True
random.seed(args.seed)
torch.manual_seed(args.seed)
torch.cuda.manual_seed(args.seed)
np.random.seed(args.seed)
# Load dataset
query_dataloader, train_dataloader, retrieval_dataloader = load_data(
args.dataset,
args.root,
args.num_query,
args.num_train,
args.batch_size,
args.num_workers,
)
# Training
for code_length in args.code_length:
logger.info('[code length:{}]'.format(code_length))
checkpoint = mdh.train(
train_dataloader,
query_dataloader,
retrieval_dataloader,
args.arch,
code_length,
args.device,
args.lr,
args.max_iter,
args.mu,
args.nu,
args.eta1,
args.eta2,
args.eta3,
args.topk,
args.evaluate_interval,
)
# Save checkpoint
torch.save(
checkpoint,
os.path.join(
'checkpoints', '{}_model_{}_codelength_'
'{}_mu_{}_nu_{}_eta1_{}_eta2_{}_eta3_{}_'
'topk_{}_map_{:.4f}.pt'.format(args.dataset, args.arch,
code_length, args.mu, args.nu,
args.eta1, args.eta2, args.eta3,
args.topk, checkpoint['map'])))
logger.info('[code_length:{}][map:{:.4f}]'.format(
code_length, checkpoint['map']))
save_mesh_fig(pts_rnd[-1].data.cpu().numpy(), offset[-1],
topology_vis[-1], loss_obj.x_grids, loss_obj.y_grids,
loss_obj.z_grids, itest, args, 'val')
print('')
return loss_eval
if __name__ == '__main__':
# parse args
args = parse_args()
# load data
args, data_val = load_data(args, dtype, 'val')
# setup loss object
loss_obj = Loss(args)
# initialize the model
assert (os.path.isfile(args.model))
print "Validating with snapshotted model %s ..." % args.model
deep_marching_cubes = torch.load(args.model)
if torch.cuda.is_available():
deep_marching_cubes.cuda()
# validation
loss = run_val(deep_marching_cubes, loss_obj, data_val, args, 'val')
print('============== average loss:%f' % (loss / args.num_val))
plt.savefig(os.path.join(args.output_dir, fname + '.png'))
plt.close()
np.savez(os.path.join(args.output_dir, fname),
loss_epochs=np.asarray(loss_epochs),
loss_evals=np.asarray(loss_evals))
if __name__ == '__main__':
# parse args
args = config.parse_args()
config.setup_logging(args, handler=None)
# load data
args, data_train = load_data(args, dtype, 'train')
args, data_val = load_data(args, dtype, 'val')
# setup loss object
loss_obj = Loss(args)
# initialize the model
curr_epoch = 0
if os.path.isfile(args.model):
curr_epoch = int(
os.path.splitext(args.model)[0][args.model.find('epoch') +
len('epoch'):])
logger.info("Resuming training from epoch %d ..." % curr_epoch)
deep_marching_cubes = torch.load(args.model)
else:
deep_marching_cubes = DeepMarchingCube(args)
lr = optimizer.param_groups[0]['lr']
if 'pooling_mode' in list(checkpoint.keys()):
cfg.POOLING_MODE = checkpoint['pooling_mode']
print("loaded checkpoint %s" % (load_name))
if args.mGPUs:
fasterRCNN = nn.DataParallel(fasterRCNN, device_ids=[5])
if args.cuda:
fasterRCNN.cuda()
DATA_AUG_CFG['dataset_file'] = args.dataset_file
# data_loader = data.load_data(data_root_path, args.batch_size, DATA_AUG_CFG, KittiLoader)
# data_loader = data.load_data(data_root_path, args.batch_size, data_config,
# KITTIBEVLoader)
data_loader = data.load_data(BATCH_SIZE, data_config, train_encoder,
KITTIBEVLoader)
train_size = len(data_loader)
iters_per_epoch = int(train_size / args.batch_size)
for epoch in range(args.start_epoch, args.max_epochs + 1):
# setting to train mode
fasterRCNN.train()
loss_temp = 0
start = time.time()
if epoch % (args.lr_decay_step + 1) == 0:
adjust_learning_rate(optimizer, args.lr_decay_gamma)
lr *= args.lr_decay_gamma
for step, _data in enumerate(data_loader):
im_data, im_info, gt_boxes, num_boxes, ry_target, sample_names = _data
pkmu_threshold=pkmu_threshold)
print('Lc cleaning', len(data_frame))
data_frame = kmu_cut(data_frame)
print('Kmu cleaning', len(data_frame))
# data_frame = remove_high_pkmu_mass(data_frame)
# print('high pkmu cleaning', len(data_frame))
if not bdt:
data_frame = data_frame.reset_index(drop=True)
data_frame = ip_star_cleaning(data_frame)
print('ip star cleaning', len(data_frame))
data_frame = data_frame.reset_index(drop=True)
return data_frame
if __name__ == '__main__':
a = load_data(df_name='Lb_data')
a.dropna(inplace=True)
df = reduce_background(a)
# df = a
# df = df[df['proton_P'] < 40000]
# df = df[df['Kminus_P'] < 60000]
# plt.hist(df['proton_P'], bins=50)
# plt.xlabel('proton_P')
# plt.show()
# plt.hist(df['Kminus_P'], bins=50)
# plt.xlabel('Kminus_P')
# plt.show()
# plt.hist2d(df['proton_P'], df['Kminus_P'], bins=75)
# plt.xlabel('proton_P')
# plt.ylabel('Kminus_P')
# plt.show()
import time import numpy as np from IPython.display import clear_output from options.train_options import TrainOptions from data.data_loader import load_data, minibatchAB from data.save_data import show_generator_image from util.image_pool import ImagePool from models.networks import get_generater_function from models.networks import resnet_generator, n_layer_discriminator from models.train_function import * opt = TrainOptions().parse() # load data dpath = opt.dataroot train_A = load_data(dpath + 'trainA/*') train_B = load_data(dpath + 'trainB/*') train_batch = minibatchAB(train_A, train_B, batch_size=opt.batch_size) val_A = load_data(dpath + 'valA/*') val_B = load_data(dpath + 'valB/*') val_batch = minibatchAB(val_A, val_B, batch_size=4) # create gennerator models netG_A, real_A, fake_B = resnet_generator() netG_B, real_B, fake_A = resnet_generator() # create discriminator models netD_A = n_layer_discriminator() netD_B = n_layer_discriminator() # create generators train function netG_train_function = netG_train_function_creator(netD_A, netD_B, netG_A, netG_B, real_A, real_B, fake_A, fake_B)
def load_data(data_name):
training_data = data_loader.load_data(data_name, "train")
test_data = data_loader.load_data(data_name, "test")
validation_data = None
return training_data, test_data
import pyfpgrowth
from data.data_loader import load_data
transactions = load_data()
sup = 0.055 # float(input("Enter support %: "))
patterns = pyfpgrowth.find_frequent_patterns(transactions,
int(len(transactions) * sup))
for k, v in patterns.items():
print(k, v)
print(len(patterns))
rules = pyfpgrowth.generate_association_rules(patterns, 0.00001)
print(rules)
print(len(rules))
def run():
# Load configuration
args = load_config()
logger.add(os.path.join('logs', '{time}.log'),
rotation="500 MB",
level="INFO")
logger.info(args)
# Load dataset
query_dataloader, train_dataloder, retrieval_dataloader = load_data(
args.dataset,
args.root,
args.num_query,
args.num_train,
args.batch_size,
args.num_workers,
)
multi_labels = args.dataset in multi_labels_dataset
if args.train:
ssdh.train(
train_dataloder,
query_dataloader,
retrieval_dataloader,
multi_labels,
args.code_length,
num_features[args.arch],
args.alpha,
args.beta,
args.max_iter,
args.arch,
args.lr,
args.device,
args.verbose,
args.evaluate_interval,
args.snapshot_interval,
args.topk,
)
elif args.resume:
ssdh.train(
train_dataloder,
query_dataloader,
retrieval_dataloader,
multi_labels,
args.code_length,
num_features[args.arch],
args.alpha,
args.beta,
args.max_iter,
args.arch,
args.lr,
args.device,
args.verbose,
args.evaluate_interval,
args.snapshot_interval,
args.topk,
args.checkpoint,
)
elif args.evaluate:
model = load_model(args.arch, args.code_length)
model.load_snapshot(args.checkpoint)
model.to(args.device)
model.eval()
mAP = ssdh.evaluate(
model,
query_dataloader,
retrieval_dataloader,
args.code_length,
args.device,
args.topk,
multi_labels,
)
logger.info('[Inference map:{:.4f}]'.format(mAP))
else:
raise ValueError(
'Error configuration, please check your config, using "train", "resume" or "evaluate".'
)
def run():
# Load config
args = load_config()
logger.add('logs/{}_model_{}_code_{}.log'.format(
args.dataset,
args.arch,
args.code_length,
0.01,
#args.alpha,
),
rotation='500 MB',
level='INFO',
)
logger.info(args)
# Set seed
torch.backends.cudnn.benchmark = True
random.seed(args.seed)
torch.manual_seed(args.seed)
torch.cuda.manual_seed(args.seed)
np.random.seed(args.seed)
# Load dataset
train_dataloader, query_dataloader, retrieval_dataloader = load_data(
args.dataset,
args.root,
args.batch_size,
args.num_workers,
)
# Training
for code_length in args.code_length:
checkpoint = ADMM.train(
train_dataloader,
query_dataloader,
retrieval_dataloader,
args.arch,
code_length,
args.device,
args.lr,
args.max_iter,
args.topk,
args.evaluate_interval,
args.anchor_num,
args.proportion,
)
logger.info('[code_length:{}][map:{:.4f}]'.format(args.code_length, checkpoint['map']))
# Save checkpoint
torch.save(
checkpoint,
os.path.join('checkpoints', '{}_model_{}_code_{}_alpha_{}_map_{:.4f}.pt'.format(
args.dataset,
args.arch,
code_length,
0.01,
#args.alpha,
checkpoint['map']),
)
)
import keras.backend as K
from IPython.display import clear_output
from options.train_options import TrainOptions
from data.data_loader import load_data, minibatchAB
from data.data_display import show_generator_image
from models.train_function import get_train_function
from models.networks import resnet_generator, n_layer_discriminator, get_generater_function
from models.loss import netG_loss, netD_loss
from keras.layers import Input, BatchNormalization
opt = TrainOptions().parse()
# load data
dpath = opt.dataroot
train_A = load_data(dpath + 'trainA/*')
train_B = load_data(dpath + 'trainB/*')
print('#training images = {}'.format(len(train_A)))
train_batch = minibatchAB(train_A, train_B, batch_size=opt.batch_size)
val_A = load_data(dpath + 'valA/*')
val_B = load_data(dpath + 'valB/*')
print('#training images = {}'.format(len(train_A)))
val_batch = minibatchAB(val_A, val_B, batch_size=4)
# create gennerator models
netG_A, real_A, fake_B = resnet_generator()
netG_B, real_B, fake_A = resnet_generator()
netG_A.summary()
# create discriminator models
