def main(data_args=None,
optimizer_args=None,
model_args=None,
loss_args=None,
train_args=None):
'''Main function for continuous BGAN.
'''
print_section(
'LOADING DATA') ##############################################
train_stream, training_samples, shape, viz_options = load_stream(
**data_args)
train_args['training_samples'] = training_samples
setup_viz(**viz_options)
model_args.update(**shape)
print_section(
'MODEL') #####################################################
noise_var = T.matrix('noise')
input_var = T.tensor4('inputs')
if loss_args['loss'] == 'bgan':
log_Z = theano.shared(lasagne.utils.floatX(0.), name='log_Z')
loss_args['loss_options']['log_Z'] = log_Z
else:
log_Z = None
logger.info('Building model and compiling GAN functions...')
logger.info('Model args: {}'.format(model_args))
generator, discriminator = build(noise_var, input_var, **model_args)
real_out = lasagne.layers.get_output(discriminator)
fake_out = lasagne.layers.get_output(discriminator,
lasagne.layers.get_output(generator))
g_results, d_results = get_losses(real_out,
fake_out,
optimizer_args=optimizer_args,
**loss_args)
if log_Z is not None:
log_Z_est = est_log_Z(fake_out)
g_results.update(**{'log Z': log_Z, 'log Z (est)': log_Z_est.mean()})
print_section(
'OPTIMIZER') #################################################
train_d, train_g, gen = setup(input_var, noise_var, log_Z, generator,
discriminator, g_results, d_results,
**optimizer_args)
print_section(
'TRAIN') #####################################################
try:
train(train_d, train_g, gen, train_stream, **train_args)
except KeyboardInterrupt:
logger.info('Training interrupted')
print_section(
'DONE') ##################################################
exit(0)
def main():
controller = Controller()
kwargs = {"num_workers": 1, "pin_memory": True} if FLAGS.CUDA else {}
train_loader = DataLoader(
datasets.Omniglot(
"data",
train=True,
download=True,
transform=transforms.Compose([
transforms.ToTensor(),
# transforms.Normalize(mean, std) should add for better performance, + other transforms
])),
batch_sizer=FLAGS.BATCH_SIZE,
shuffle=True,
**kwargs)
test_loader = DataLoader(
datasets.Omniglot(
"data",
train=False,
transform=transforms.Compose([
transforms.ToTensor(),
# transforms.Normalize(mean, std) should add for better performance, + other transforms
])),
batch_sizer=FLAGS.BATCH_SIZE,
shuffle=True,
**kwargs)
if FLAGS.TRAIN:
train(controller, train_loader)
else:
test(controller, test_loader)
def main():
para = params_setup()
logging_config_setup(para)
logging.info('Creating graph')
graph, model, data_generator = create_graph(para)
with tf.Session(config=config_setup(), graph=graph) as sess:
sess.run(tf.global_variables_initializer())
logging.info('Loading weights')
load_weights(para, sess, model)
print_num_of_trainable_parameters()
try:
if para.mode == 'train':
logging.info('Started training')
train(para, sess, model, data_generator)
if para.save_final_model_path != '':
save_weights(sess, model, para.save_final_model_path)
elif para.mode == 'validation':
logging.info('Started validation')
test(para, sess, model, data_generator)
elif para.mode == 'test':
logging.info('Started testing')
test(para, sess, model, data_generator)
elif para.mode == 'predict':
logging.info('Predicting')
predict(para, sess, model, data_generator,
'./data/solar-energy3/solar_predict.txt', para.samples)
except KeyboardInterrupt:
print('KeyboardInterrupt')
finally:
print('Stop')
def main(_):
args = params_setup()
print("[args]: ", args)
if args.mode == 'train':
train(args)
elif args.mode == 'test':
predict(args)
elif args.mode == 'chat':
chat(args)
def run_energy_model_mnist(cfg):
"""
Main script.
Args:
cfg: Dictionary defining parameters of the run
"""
# Initialize seed if specified (might slow down the model)
if cfg['seed'] is not None:
torch.manual_seed(cfg['seed'])
# Create the cost function to be optimized by the model
c_energy = utils.create_cost(cfg['c_energy'], cfg['beta'])
# Create activation functions for every layer as a list
phi = utils.create_activations(cfg['nonlinearity'], len(cfg['dimensions']))
# Initialize energy based model
if cfg["energy"] == "restr_hopfield":
model = energy.RestrictedHopfield(cfg['dimensions'], c_energy,
cfg['batch_size'],
phi).to(config.device)
elif cfg["energy"] == "cond_gaussian":
model = energy.ConditionalGaussian(cfg['dimensions'], c_energy,
cfg['batch_size'],
phi).to(config.device)
else:
raise ValueError(
f'Energy based model \"{cfg["energy"]}\" not defined.')
# Define optimizer (may include l2 regularization via weight_decay)
w_optimizer = utils.create_optimizer(model,
cfg['optimizer'],
lr=cfg['learning_rate'])
# Create torch data loaders with the MNIST data set
mnist_train, mnist_test = data.create_mnist_loaders(cfg['batch_size'])
logging.info("Start training with parametrization:\n{}".format(
json.dumps(cfg, indent=4, sort_keys=True)))
for epoch in range(1, cfg['epochs'] + 1):
# Training
train.train(model, mnist_train, cfg['dynamics'], w_optimizer,
cfg["fast_ff_init"])
# Testing
test_acc, test_energy = train.test(model, mnist_test, cfg['dynamics'],
cfg["fast_ff_init"])
# Logging
logging.info("epoch: {} \t test_acc: {:.4f} \t mean_E: {:.4f}".format(
epoch, test_acc, test_energy))
def train_wrapper(config):
print_config(config)
t0 = perf_counter()
debug(f"overwriting soft link ./latest_cwd --> {config['_cwd']}")
os.unlink('./latest_cwd')
os.symlink(config['_cwd'], './latest_cwd', target_is_directory=True)
train(config)
print(f"cwd: {config['_cwd']}")
print(f"total running time: {str(timedelta(seconds=perf_counter() - t0))}")
def main(project_name):
tic = time.time()
logger = Logger('_01_training_{}'.format(project_name))
logger.info('==> initialize model')
embedding = build_model(pretrained=True)
logger.info('==> train model')
train(embedding, project_name=project_name)
toc = time.time() - tic
logger.info('Elapsed time: {:.1f} [min]'.format(toc / 60.0))
def main(_):
args = params_setup(model_num=0)
args1 = params_setup(model_num=1)
args = check_mion_ray(args)
args1 = check_mion_ray(args1)
print("[args]: ", args)
if args.mode == 'train':
train(args)
elif args.mode == 'test':
predict(args)
elif args.mode == 'chat':
chat(args)
elif args.mode == 'fight':
fight(args, args1)
def main(project_name):
tic = time.time()
logger = Logger('_01_training_{}'.format(project_name))
logger.info('==> initialize model')
embedding = build_model()
# additional training
# logger.info('==> load model')
# embedding = torch.load(os.path.join('_model', '*****.pt'))
logger.info('==> train model')
train(embedding, project_name=project_name)
toc = time.time() - tic
logger.info('Elapsed time: {:.1f} [min]'.format(toc / 60.0))
def main():
para = params_setup()
logging_config_setup(para)
print("Creating graph...")
graph, model, data_generator = create_graph(para)
print("Done creating graph.")
with tf.Session(config=config_setup(), graph=graph) as sess:
sess.run(tf.global_variables_initializer())
print("Loading weights...")
load_weights(para, sess, model)
print_num_of_trainable_parameters()
# PRINT NAMES OF TENSORS THAT ARE ALPHAS
# example name: "model/rnn/cond/rnn/multi_rnn_cell/cell_0/cell_0/temporal_pattern_attention_cell_wrapper/attention/Sigmoid:0"
# for item in [n.name for n in tf.get_default_graph().as_graph_def().node
# if (n.name.find("temporal_pattern_attention_cell_wrapper/attention")!=-1 and
# n.name.find("Sigmoid")!=-1)]:
# print(item)
# Print names of ops
# for op in tf.get_default_graph().get_operations():
# if(op.name.find("ben_multiply")!=-1):
# print(str(op.name))
# PRINT REG KERNEL AND BIAS
# reg_weights = [v for v in tf.global_variables() if v.name == "model/dense_2/kernel:0"][0]
# reg_bias = [v for v in tf.global_variables() if v.name == "model/dense_2/bias:0"][0]
# print("Reg Weights:", sess.run(reg_weights))
# print("Reg Bias:", sess.run(reg_bias) * data_generator.scale[0])
try:
if para.mode == 'train':
train(para, sess, model, data_generator)
elif para.mode == 'test':
print("Evaluating model...")
test(para, sess, model, data_generator)
except KeyboardInterrupt:
print('KeyboardInterrupt')
finally:
print('Stop')
def main():
parser = argparse.ArgumentParser()
parser.add_argument('-n', '--name', help="Name to be identified")
parser.add_argument('-t',
'--train',
help="Train new Model",
action="store_true")
parser.add_argument(
'-trin',
'--train-file',
help="Training file in csv format containing names and gender")
parser.add_argument('-tein',
'--test-file',
help="test file to check accuracy of trained model")
args = parser.parse_args()
if args.train:
train.train(args.tin, args.tein)
else:
identify(args.name)
def main():
para = params_setup()
logging_config_setup(para)
graph, model, data_generator = create_graph(para)
with tf.Session(config=config_setup(), graph=graph) as sess:
sess.run(tf.global_variables_initializer())
load_weights(para, sess, model)
print_num_of_trainable_parameters()
try:
if para.mode == 'train':
train(para, sess, model, data_generator)
elif para.mode == 'test':
test(para, sess, model, data_generator)
except KeyboardInterrupt:
print('KeyboardInterrupt')
finally:
print('Stop')
def main():
para = params_setup()
logging_config_setup(para)
graph, model, data_generator = create_graph(para)
with tf.Session(config=config_setup(), graph=graph) as sess:
sess.run(tf.global_variables_initializer())
load_weights(para, sess, model)
print_num_of_trainable_parameters()
try:
if para.mode == 'train':
train(para, sess, model, data_generator)
elif para.mode == 'test':
obs, predicted = test(para, sess, model, data_generator)
obs = obs * data_generator.scale + data_generator.min_value
predicted = predicted * data_generator.scale + data_generator.min_value
print("MSE: ", mean_squared_error(obs[:, 0], predicted[:, 0]))
idx = pd.DatetimeIndex(start='2016-10-16',
end='2018-11-04',
freq='W')
obs_df = pd.DataFrame(data=obs[:, 0],
columns=['Observed'],
index=idx)
pred_df = pd.DataFrame(data=predicted[:, 0],
columns=['Predicted'],
index=idx)
df = pd.concat([obs_df, pred_df], axis=1)
df.plot()
plt.show()
except KeyboardInterrupt:
print('KeyboardInterrupt')
finally:
print('Stop')
def strain(self, test, test2):
parameter = {
'emb_dim': 300, #词向量维度
'ner_dim': 30,
'pos_dim': 30,
'num_class': 19, #标签个数
'hidden_dim': 200, #隐藏层参数个数
'GCN_layers': 1, #GCN层数
'rnn_layers': 1, #rnn层数
'bidirec': True, #rnn双向还是单项
'rnn_dim': 100, #rnn层节点数
'input_dropout': 0.5, #输入数据dropout率
'word_dropout': 0.04, #词的dropout率
'mlp_layers': 1, #多层感知器层数
'lr': 0.5, #sgd开始梯度下降率
'lr_decay': 0.95, #梯度下降率
'decay_epoch': 1, #梯度下降频率
'num_epoch': 100, #总循环训练数
'batch_size': 50 #批处理个数
}
parameter['GCN_layers'] = int(self.gcn_num_entry.get())
parameter['rnn_layers'] = int(self.rnn_lnum_entry.get())
parameter['rnn_dim'] = int(self.rnn_num_entry.get())
parameter['mlp_layers'] = int(self.mlp_lnum_entry.get())
parameter['hidden_dim'] = int(self.mlp_num_entry.get())
parameter['lr'] = float(self.lr_entry.get())
parameter['lr_decay'] = float(self.lr_decay_entry.get())
parameter['num_epoch'] = int(self.epoch_entry.get())
parameter['batch_size'] = int(self.bitch_entry.get())
parameter['input_dropout'] = float(self.dropout_entry.get())
new_train = train(parameter)
new_train.start_train()
self.now_epoch_text.grid(row=0, column=0)
self.time_text.grid(row=1, column=0)
self.train_loss_text.grid(row=2, column=0)
self.score_text.grid(row=3, column=0)
self.start_button["text"] = "结束训练"
for i in range(1, parameter['num_epoch'] + 1):
if not self.training:
break
train_loss, score, time = new_train.epoch_train(i)
self.now_epoch_text["text"] = "当前趟数：" + str(i)
print(train_loss)
self.time_text["text"] = "耗时：" + str(time)[:10]
self.train_loss_text["text"] = "当前损失：" + str(train_loss)[:10]
self.score_text["text"] = "训练集上F1值：" + str(score)
self.forget_train.grid_forget()
self.start_button["text"] = "开始训练"
def main():
session_start = time.time()
set_parameters_start = time.time()
current_time = datetime.datetime.now().strftime("%Y%m%d-%H%M%S")
session_name = 'more_data_test' + '_' + current_time
optimizer_params = {'learning_rate': 0.0001}
training_params = {'num_epochs': 10, 'batch_size': 256, 'apnea_weight': 10}
preprocess_params = {
'featurize_func': utils.featurize_2,
'seq_len': 60,
'pulse_sample_rate': 16,
'data_stride': 15,
'split_seed': 2
}
preprocess_func = utils.preprocess_data
model_params = {'rnn_hidden_dim': 20}
test_split = .15
val_split = 0.1765
log_dir = 'logs/gradient_tape/' + session_name
model_weights_dir = 'model_weights/' + session_name
print("set session parameters in {}".format(time.time() -
set_parameters_start))
load_data_start = time.time()
dataset = utils.load_tfrecords(tf_rec_data_dir='data/processed_data')
num_records = 0
for i in dataset:
num_records += 1
print("data loaded in {}".format(time.time() - load_data_start))
process_data_start = time.time()
train_data, test_data = utils.split_dataset(
dataset, test_split, seed=preprocess_params['split_seed'])
train_data, val_data = utils.split_dataset(
train_data, val_split, seed=preprocess_params['split_seed'])
train_data = preprocess_func(
train_data,
featurize_func=preprocess_params['featurize_func'],
seq_len=preprocess_params['seq_len'],
pulse_sample_rate=preprocess_params['pulse_sample_rate'],
data_stride=preprocess_params['data_stride'])
val_data = preprocess_func(
val_data,
featurize_func=preprocess_params['featurize_func'],
seq_len=preprocess_params['seq_len'],
pulse_sample_rate=preprocess_params['pulse_sample_rate'],
data_stride=preprocess_params['seq_len'])
test_data = preprocess_func(
test_data,
featurize_func=preprocess_params['featurize_func'],
seq_len=preprocess_params['seq_len'],
pulse_sample_rate=preprocess_params['pulse_sample_rate'],
data_stride=preprocess_params['seq_len'])
train_num = train_data._tensors[0].shape[0]
train_bal = train_data._tensors[1].numpy().mean()
val_num = val_data._tensors[0].shape[0]
val_bal = val_data._tensors[1].numpy().mean()
test_num = test_data._tensors[0].shape[0]
test_bal = test_data._tensors[1].numpy().mean()
data_bal = {'train': train_bal, 'val': val_bal, 'test': test_bal}
data_size = {'train': train_num, 'val': val_num, 'test': test_num}
print("data processed in {}".format(time.time() - process_data_start))
train_start = time.time()
if tf.test.is_gpu_available():
print('Using GPU')
with tf.device("gpu:0"):
model = BaseLSTM(rnn_hidden_dim=model_params['rnn_hidden_dim'])
loss_object = tf.keras.losses.BinaryCrossentropy()
optimizer = tf.keras.optimizers.Adam(
optimizer_params['learning_rate'])
else:
model = BaseLSTM(rnn_hidden_dim=model_params['rnn_hidden_dim'])
loss_object = tf.keras.losses.BinaryCrossentropy()
optimizer = tf.keras.optimizers.Adam(optimizer_params['learning_rate'])
train(model,
train_data,
val_data,
loss_object,
optimizer,
log_dir,
model_weights_dir,
num_epochs=training_params['num_epochs'],
batch_size=training_params['batch_size'],
apnea_weight=training_params['apnea_weight'])
train_res = evaluate(model, train_data)
test_res = evaluate(model, test_data)
print(train_res)
print(test_res)
print("model trained in {}".format(time.time() - train_start))
utils.save_session(session_name,
model,
model_params,
num_records,
test_split,
val_split,
preprocess_func,
preprocess_params,
data_bal,
data_size,
training_params,
optimizer,
optimizer_params,
train_res,
test_res,
log_dir,
model_weights_dir,
res_path='results')
print("Saving session results, session ran in {}".format(time.time() -
session_start))
def train_with_config(args, cfg):
os.environ["CUDA_VISIBLE_DEVICES"] = str(args.gpu_id)
out_dir = os.path.join(args.out_dir, args.cfg_name)
torch.manual_seed(args.seed)
torch.cuda.manual_seed(args.seed)
torch.backends.cudnn.benchmark = True
print("model configs:")
print(json.dumps(cfg, indent=2))
print()
print("run args:")
for arg in vars(args):
print("%10s: %s" % (arg, str(getattr(args, arg))))
print()
print("parsing dictionaries")
word_dict = SymbolDictionary.load_from_file(cfg.word_dict)
ent_dict = SymbolDictionary.load_from_file(cfg.ent_dict)
attr_dict = SymbolDictionary.load_from_file(cfg.attr_dict)
pred_dict = SymbolDictionary.load_from_file(cfg.pred_dict)
print("building model")
word_emb = WordEmbedding.build_from_config(cfg.language_model,
word_dict).cuda()
word_emb.init_embedding(cfg.language_model.word_emb_init)
word_emb.freeze()
vision_model = VisionModel.build_from_config(cfg.vision_model).cuda()
language_model = LanguageModel.build_from_config(cfg.language_model).cuda()
ent_loss = LossModel.build_from_config(cfg.ent_loss).cuda()
rel_loss = LossModel.build_from_config(cfg.rel_loss).cuda()
n_v_params = count_parameters(vision_model)
n_l_params = count_parameters(language_model)
print("vision model: {:,} parameters".format(n_v_params))
print("language model: {:,} parameters".format(n_l_params))
print()
print("loading train data...")
train_set = GQATriplesDataset.create(cfg,
word_dict,
ent_dict,
attr_dict,
pred_dict,
cfg.train.triples_path,
mode="train",
preload=args.preload)
train_loader = DataLoader(train_set,
batch_size=cfg.train.batch_size,
shuffle=True,
num_workers=args.n_workers)
print("loading val data...")
val_set = GQATriplesDataset.create(cfg,
word_dict,
ent_dict,
attr_dict,
pred_dict,
cfg.val.triples_path,
mode="eval",
preload=args.preload)
val_loader = DataLoader(val_set,
batch_size=cfg.val.batch_size,
shuffle=True,
num_workers=args.n_workers)
print("training started...")
train(word_emb, vision_model, language_model, ent_loss, rel_loss,
train_loader, val_loader, word_dict, ent_dict, pred_dict,
args.n_epochs, args.val_freq, out_dir, cfg, args.grad_freq)
#load model from last check point
print("resuming from last checkpoint...")
assert os.path.isfile('checkpoint/checkpoint.pth.tar'), "Error: no checkpoint found"
checkpoint = torch.load("./checkpoint/checkpoint.pth.tar")
start_epoch = checkpoint["epoch"]
net = checkpoint["net"]
best_acc = checkpoint["best_acc"]
optimizer = (checkpoint["optimizer"])
print("Loaded model from {} , epoch {}".format("./checkpoint/checkpoint.pth.tar", checkpoint["epoch"]))
else:
print("Building model")
net = lenet.lenet()
#net = resnet.ResNet18()
if use_cuda:
net.cuda()
net = torch.nn.DataParallel(net, device_ids=range(torch.cuda.device_count()))
cudnn.benchmark = True
if __name__ == "__main__":
print("USE CUDA : ", use_cuda)
for epoch in range(start_epoch, start_epoch+2):
lr = utils.lr_multiplier(epoch)
train(epoch=epoch, trainloader=trainloader, net=net, use_cuda=use_cuda,learning_rate=lr)
test(epoch, testloader=testloader, net=net, use_cuda=use_cuda,learning_rate= lr)
writer.export_scalars_to_json("./all_scalars.json")
writer.close()
args = parser.parse_args()
if args.work == None:
print('must specify work')
elif args.work == 'preprocess' or args.work == 'pre':
from lib.preprocess import do_preprocess
if args.file != None:
do_preprocess(args.file[0], args.file[1])
else:
do_preprocess(test_exist=True)
os.system('python3 main.py -w train -e 5 -b 256')
elif args.work == 'test':
from lib.test import test
test(args.file[0], args.file[1])
elif args.work == 'train':
from lib.train import train
history = train(model_path=args.model,
epochs=args.epochs,
batch_size=args.batch_size,
seed=args.random_seed)
if args.plot == True:
from lib.plot import plot
plot(history)
elif args.work == 'semi':
from lib.semiParse import semiParse
semiParse()
memory = Memory(REPLAY_SIZE)
agent = Agent(env, memory)
initial_observation = env.reset()
if 'cuda' in str(device):
print('The GPU is being used')
else:
print('The CPU is being used')
if option_dict['random']:
play_random(env, UP_ACTION, DOWN_ACTION, seconds=5)
if option_dict['train']:
print("Training")
print("ReplayMemory will require {}gb of GPU RAM".format(round(REPLAY_SIZE * 32 * 84 * 84 / 1e+9, 2)))
agent.reset_environtment()
train(env, net, target_net, epsilon_data, agent, memory, GAMMA, device,
DELAY_LEARNING, TARGET_UPDATE_FREQ, BATCH_SIZE, model)
if option_dict['oldnetwork']:
file_path = './pull/pong_v4_data/DQN/DQN_10_6-1700.dat'
seconds = 120
test_old_network(env, net, file_path, seconds, device)
def main(model,
auxiliary=True,
model_label='rcnn',
rnn_type='gru',
padding='pre',
reg='s',
prefix="crawl",
embedding_file_type="word2vec",
train_fname="./data/train.csv",
test_fname="./data/test.csv",
embeds_fname="./data/GoogleNews-vectors-negative300.bin",
logger_fname="./logs/log-aws",
mode="all",
wrong_words_fname="./data/correct_words.csv",
format_embeds="binary",
config="./config.json",
output_dir="./out",
norm_prob=False,
norm_prob_koef=1,
gpus=0,
char_level=False,
random_seed=2018,
num_folds=5):
embedding_type = prefix + "_" + embedding_file_type
logger = Logger(logging.getLogger(), logger_fname)
# ====Detect GPUs====
logger.debug(device_lib.list_local_devices())
# ====Load data====
logger.info('Loading data...')
train_df = load_data(train_fname)
test_df = load_data(test_fname)
target_labels = [
'toxic', 'severe_toxic', 'obscene', 'threat', 'insult', 'identity_hate'
]
num_classes = len(target_labels)
# ====Load additional data====
logger.info('Loading additional data...')
# swear_words = load_data(swear_words_fname, func=lambda x: set(x.T[0]), header=None)
wrong_words_dict = load_data(wrong_words_fname,
func=lambda x: {val[0]: val[1]
for val in x})
tokinizer = RegexpTokenizer(r'\S+')
regexps = [
re.compile("([a-zA-Z]+)([0-9]+)"),
re.compile("([0-9]+)([a-zA-Z]+)")
]
# ====Load word vectors====
logger.info('Loading embeddings...')
if model != 'mvcnn':
embed_dim = 300
embeds = Embeds(embeds_fname,
embedding_file_type,
format=format_embeds)
if mode in ('preprocess', 'all'):
logger.info('Generating indirect features...')
# https://www.kaggle.com/jagangupta/stop-the-s-toxic-comments-eda
# Word count in each comment:
train_df['count_word'] = train_df["comment_text"].apply(
lambda x: len(str(x).split()))
test_df['count_word'] = test_df["comment_text"].apply(
lambda x: len(str(x).split()))
# Unique word count
train_df['count_unique_word'] = train_df["comment_text"].apply(
lambda x: len(set(str(x).split())))
test_df['count_unique_word'] = test_df["comment_text"].apply(
lambda x: len(set(str(x).split())))
# Letter count
train_df['count_letters'] = train_df["comment_text"].apply(
lambda x: len(str(x)))
test_df['count_letters'] = test_df["comment_text"].apply(
lambda x: len(str(x)))
# punctuation count
train_df["count_punctuations"] = train_df["comment_text"].apply(
lambda x: len([c for c in str(x) if c in string.punctuation]))
test_df["count_punctuations"] = test_df["comment_text"].apply(
lambda x: len([c for c in str(x) if c in string.punctuation]))
# upper case words count
train_df["count_words_upper"] = train_df["comment_text"].apply(
lambda x: len([w for w in str(x).split() if w.isupper()]))
test_df["count_words_upper"] = test_df["comment_text"].apply(
lambda x: len([w for w in str(x).split() if w.isupper()]))
# title case words count
train_df["count_words_title"] = train_df["comment_text"].apply(
lambda x: len([w for w in str(x).split() if w.istitle()]))
test_df["count_words_title"] = test_df["comment_text"].apply(
lambda x: len([w for w in str(x).split() if w.istitle()]))
# Word count percent in each comment:
train_df['word_unique_pct'] = train_df[
'count_unique_word'] * 100 / train_df['count_word']
test_df['word_unique_pct'] = test_df[
'count_unique_word'] * 100 / test_df['count_word']
# Punct percent in each comment:
train_df['punct_pct'] = train_df[
'count_punctuations'] * 100 / train_df['count_word']
test_df['punct_pct'] = test_df['count_punctuations'] * 100 / test_df[
'count_word']
# Average length of the words
train_df["mean_word_len"] = train_df["comment_text"].apply(
lambda x: np.mean([len(w) for w in str(x).split()]))
test_df["mean_word_len"] = test_df["comment_text"].apply(
lambda x: np.mean([len(w) for w in str(x).split()]))
# upper case words percentage
train_df["words_upper_pct"] = train_df[
"count_words_upper"] * 100 / train_df['count_word']
test_df["words_upper_pct"] = test_df[
"count_words_upper"] * 100 / test_df['count_word']
# title case words count
train_df["words_title_pct"] = train_df[
"count_words_title"] * 100 / train_df['count_word']
test_df["words_title_pct"] = test_df[
"count_words_title"] * 100 / test_df['count_word']
# remove columns
train_df = train_df.drop('count_word', 1)
train_df = train_df.drop('count_unique_word', 1)
train_df = train_df.drop('count_punctuations', 1)
train_df = train_df.drop('count_words_upper', 1)
train_df = train_df.drop('count_words_title', 1)
test_df = test_df.drop('count_word', 1)
test_df = test_df.drop('count_unique_word', 1)
test_df = test_df.drop('count_punctuations', 1)
test_df = test_df.drop('count_words_upper', 1)
test_df = test_df.drop('count_words_title', 1)
logger.info('Cleaning text...')
train_df['comment_text_clear'] = clean_text(train_df['comment_text'],
tokinizer,
wrong_words_dict,
regexps,
autocorrect=False)
test_df['comment_text_clear'] = clean_text(test_df['comment_text'],
tokinizer,
wrong_words_dict,
regexps,
autocorrect=False)
if reg == 'w':
# remove all punctuations
train_df.to_csv(os.path.join(output_dir, 'train_clear_w.csv'),
index=False)
test_df.to_csv(os.path.join(output_dir, 'test_clear_w.csv'),
index=False)
train_df = pd.read_csv(
os.path.join(output_dir, 'train_clear_w.csv'))
test_df = pd.read_csv(os.path.join(output_dir, 'test_clear_w.csv'))
elif reg == 's':
# split by S+ keep all punctuations
train_df.to_csv(os.path.join(output_dir, 'train_clear.csv'),
index=False)
test_df.to_csv(os.path.join(output_dir, 'test_clear.csv'),
index=False)
train_df = pd.read_csv(os.path.join(output_dir, 'train_clear.csv'))
test_df = pd.read_csv(os.path.join(output_dir, 'test_clear.csv'))
if mode == 'preprocess':
return
if mode == 'processed':
if reg == 'w':
train_df = pd.read_csv(
os.path.join(output_dir, 'train_clear_w.csv'))
test_df = pd.read_csv(os.path.join(output_dir, 'test_clear_w.csv'))
elif reg == 's':
train_df = pd.read_csv(os.path.join(output_dir, 'train_clear.csv'))
test_df = pd.read_csv(os.path.join(output_dir, 'test_clear.csv'))
logger.info('Calc text length...')
train_df.fillna('unknown', inplace=True)
test_df.fillna('unknown', inplace=True)
train_df['text_len'] = train_df['comment_text_clear'].apply(
lambda words: len(words.split()))
test_df['text_len'] = test_df['comment_text_clear'].apply(
lambda words: len(words.split()))
max_seq_len = np.round(train_df['text_len'].mean() +
3 * train_df['text_len'].std()).astype(int)
logger.debug('Max seq length = {}'.format(max_seq_len))
# ====Prepare data to NN====
logger.info('Converting texts to sequences...')
max_words = 100000
if char_level:
max_seq_len = 1200
train_df['comment_seq'], test_df[
'comment_seq'], word_index = convert_text2seq(
train_df['comment_text_clear'].tolist(),
test_df['comment_text_clear'].tolist(),
max_words,
max_seq_len,
embeds,
lower=True,
char_level=char_level,
uniq=True,
use_only_exists_words=True,
position=padding)
logger.debug('Dictionary size = {}'.format(len(word_index)))
logger.info('Preparing embedding matrix...')
if model != 'mvcnn':
embedding_matrix, words_not_found = get_embedding_matrix(
embed_dim, embeds, max_words, word_index)
logger.debug('Embedding matrix shape = {}'.format(
np.shape(embedding_matrix)))
logger.debug('Number of null word embeddings = {}'.format(
np.sum(np.sum(embedding_matrix, axis=1) == 0)))
# ====Train/test split data====
# train/val
x_aux = np.matrix([
train_df["word_unique_pct"].tolist(), train_df["punct_pct"].tolist(),
train_df["mean_word_len"].tolist(),
train_df["words_upper_pct"].tolist(),
train_df["words_title_pct"].tolist()
],
dtype='float32').transpose((1, 0))
x = np.array(train_df['comment_seq'].tolist())
y = np.array(train_df[target_labels].values)
x_train_nn, x_test_nn, x_aux_train_nn, x_aux_test_nn, y_train_nn, y_test_nn, train_idxs, test_idxs = \
split_data(x, np.squeeze(np.asarray(x_aux)),y,test_size=0.2,shuffle=True,random_state=2018)
# test set
test_df_seq = np.array(test_df['comment_seq'].tolist())
test_aux = np.matrix([
train_df["word_unique_pct"].tolist(), train_df["punct_pct"].tolist(),
train_df["mean_word_len"].tolist(),
train_df["words_upper_pct"].tolist(),
train_df["words_title_pct"].tolist()
],
dtype='float32').transpose((1, 0))
test_df_seq_aux = np.squeeze(np.asarray(test_aux))
y_nn = []
logger.debug('X shape = {}'.format(np.shape(x_train_nn)))
# ====Train models====
params = Params(config)
if model_label == None:
logger.warn('Should choose a model to train')
return
if model_label == 'dense':
model = dense(
embedding_matrix,
num_classes,
max_seq_len,
dense_dim=params.get('dense').get('dense_dim'),
n_layers=params.get('dense').get('n_layers'),
concat=params.get('dense').get('concat'),
dropout_val=params.get('dense').get('dropout_val'),
l2_weight_decay=params.get('dense').get('l2_weight_decay'),
pool=params.get('dense').get('pool'),
train_embeds=params.get('dense').get('train_embeds'),
add_sigmoid=True,
gpus=gpus)
if model_label == 'cnn':
model = cnn(embedding_matrix,
num_classes,
max_seq_len,
num_filters=params.get('cnn').get('num_filters'),
l2_weight_decay=params.get('cnn').get('l2_weight_decay'),
dropout_val=params.get('cnn').get('dropout_val'),
dense_dim=params.get('cnn').get('dense_dim'),
train_embeds=params.get('cnn').get('train_embeds'),
n_cnn_layers=params.get('cnn').get('n_cnn_layers'),
pool=params.get('cnn').get('pool'),
add_embeds=params.get('cnn').get('add_embeds'),
auxiliary=auxiliary,
add_sigmoid=True,
gpus=gpus)
if model_label == 'cnn2d':
model = cnn2d(
embedding_matrix,
num_classes,
max_seq_len,
num_filters=params.get('cnn2d').get('num_filters'),
l2_weight_decay=params.get('cnn2d').get('l2_weight_decay'),
dropout_val=params.get('cnn2d').get('dropout_val'),
dense_dim=params.get('cnn2d').get('dense_dim'),
train_embeds=params.get('cnn2d').get('train_embeds'),
add_embeds=params.get('cnn2d').get('add_embeds'),
auxiliary=auxiliary,
add_sigmoid=True,
gpus=gpus)
if model_label == 'lstm':
model = rnn(
embedding_matrix,
num_classes,
max_seq_len,
l2_weight_decay=params.get('lstm').get('l2_weight_decay'),
rnn_dim=params.get('lstm').get('rnn_dim'),
dropout_val=params.get('lstm').get('dropout_val'),
dense_dim=params.get('lstm').get('dense_dim'),
n_branches=params.get('lstm').get('n_branches'),
n_rnn_layers=params.get('lstm').get('n_rnn_layers'),
n_dense_layers=params.get('lstm').get('n_dense_layers'),
train_embeds=params.get('lstm').get('train_embeds'),
mask_zero=params.get('lstm').get('mask_zero'),
kernel_regularizer=params.get('lstm').get('kernel_regularizer'),
recurrent_regularizer=params.get('lstm').get(
'recurrent_regularizer'),
activity_regularizer=params.get('lstm').get(
'activity_regularizer'),
dropout=params.get('lstm').get('dropout'),
recurrent_dropout=params.get('lstm').get('recurrent_dropout'),
auxiliary=auxiliary,
add_sigmoid=True,
gpus=gpus,
rnn_type='lstm')
if model_label == 'gru':
model = rnn(
embedding_matrix,
num_classes,
max_seq_len,
l2_weight_decay=params.get('gru').get('l2_weight_decay'),
rnn_dim=params.get('gru').get('rnn_dim'),
dropout_val=params.get('gru').get('dropout_val'),
dense_dim=params.get('gru').get('dense_dim'),
n_branches=params.get('gru').get('n_branches'),
n_rnn_layers=params.get('gru').get('n_rnn_layers'),
n_dense_layers=params.get('gru').get('n_dense_layers'),
train_embeds=params.get('gru').get('train_embeds'),
mask_zero=params.get('gru').get('mask_zero'),
kernel_regularizer=params.get('gru').get('kernel_regularizer'),
recurrent_regularizer=params.get('gru').get(
'recurrent_regularizer'),
activity_regularizer=params.get('gru').get('activity_regularizer'),
dropout=params.get('gru').get('dropout'),
recurrent_dropout=params.get('gru').get('recurrent_dropout'),
auxiliary=auxiliary,
add_sigmoid=True,
gpus=gpus,
rnn_type='gru')
if model_label == 'charrnn':
model = charrnn(
len(word_index),
num_classes,
max_seq_len,
rnn_dim=params.get('charrnn').get('rnn_dim'),
dropout_val=params.get('charrnn').get('dropout_val'),
auxiliary=auxiliary,
dropout=params.get('charrnn').get('dropout'),
recurrent_dropout=params.get('charrnn').get('recurrent_dropout'),
add_sigmoid=True,
gpus=gpus,
rnn_type=rnn_type)
if model_label == 'cnn2rnn':
model = cnn2rnn(embedding_matrix,
num_classes,
max_seq_len,
rnn_type=rnn_type)
if model_label == 'dpcnn':
model = dpcnn(embedding_matrix,
num_classes,
max_seq_len,
num_filters=params.get('dpcnn').get('num_filters'),
dense_dim=params.get('dpcnn').get('dense_dim'),
add_sigmoid=True,
gpus=gpus)
if model_label == 'rcnn':
model = rcnn(
embedding_matrix,
num_classes,
max_seq_len,
rnn_dim=params.get('rcnn').get('rnn_dim'),
dropout_val=params.get('rcnn').get('dropout_val'),
dense_dim=params.get('rcnn').get('dense_dim'),
train_embeds=params.get('rcnn').get('train_embeds'),
auxiliary=auxiliary,
dropout=params.get('rcnn').get('dropout'),
recurrent_dropout=params.get('rcnn').get('recurrent_dropout'),
add_sigmoid=True,
gpus=gpus,
rnn_type=rnn_type)
if model_label == 'capsule':
model = capsule(
embedding_matrix,
num_classes,
max_seq_len,
auxiliary=auxiliary,
Num_capsule=params.get('capsule').get('Num_capsule'),
Routings=params.get('capsule').get('Routing'),
add_sigmoid=params.get('capsule').get('add_sigmoid'),
mask_zero=params.get('capsule').get('mask_zero'),
gpus=gpus,
rnn_type='gru') # lstm may diverge but gru works better
if model == 'mvcnn':
embeds_fname1 = "./data/crawl-300d-2M.vec" # "./data/crawl-300d-2M.vec word2vec-raw.txt
embeds_fname2 = "./data/glove.840B.300d.txt"
embeds_fname3 = "./data/GoogleNews-vectors-negative300.bin"
embed_dim = 300
embeds1 = Embeds(embeds_fname1, "glove", format='file')
embeds2 = Embeds(embeds_fname2, "fasttext", format='file')
embeds3 = Embeds(embeds_fname3, "word2vec", format='binary')
embedding_matrix1, words_not_found1 = get_embedding_matrix(
embed_dim, embeds1, max_words, word_index)
embedding_matrix2, words_not_found2 = get_embedding_matrix(
embed_dim, embeds2, max_words, word_index)
#embedding_matrix3, words_not_found3 = get_embedding_matrix(embed_dim, embeds3, max_words, word_index)
model = mvcnn(embedding_matrix1,
embedding_matrix2,
num_classes,
max_seq_len,
auxiliary=auxiliary,
gpus=gpus)
# ====k-fold cross validations split data====
logger.info('Run k-fold cross validation...')
params = Params(config)
kf = KFold(n_splits=num_folds, shuffle=True, random_state=random_seed)
oof_train = np.zeros((x.shape[0], num_classes))
oof_test_skf = []
for i, (train_index, test_index) in enumerate(kf.split(x, y)):
print("TRAIN:", train_index, "TEST:", test_index)
x_train, x_aux_train, x_test, x_aux_test = x[train_index], x_aux[
train_index], x[test_index], x_aux[test_index]
y_train, y_test = y[train_index], y[test_index]
logger.info('Start training {}-th fold'.format(i))
if auxiliary:
inputs = [x_train, x_aux_train]
inputs_val = [x_test, x_aux_test]
output = [test_df_seq, test_df_seq_aux]
else:
inputs = x_train
inputs_val = x_test
output = test_df_seq
hist = train(
x_train=
inputs, # [x_train, x_aux_train] when auxiliary input is allowed.
y_train=y_train,
x_val=inputs_val, # [x_test, x_aux_test],
y_val=y_test,
model=model,
batch_size=params.get(model_label).get('batch_size'),
num_epochs=params.get(model_label).get('num_epochs'),
learning_rate=params.get(model_label).get('learning_rate'),
early_stopping_delta=params.get(model_label).get(
'early_stopping_delta'),
early_stopping_epochs=params.get(model_label).get(
'early_stopping_epochs'),
use_lr_strategy=params.get(model_label).get('use_lr_strategy'),
lr_drop_koef=params.get(model_label).get('lr_drop_koef'),
epochs_to_drop=params.get(model_label).get('epochs_to_drop'),
model_checkpoint_dir=os.path.join('.', 'model_checkpoint', reg,
model_label, embedding_type,
padding, str(i)),
logger=logger)
model.load_weights(
os.path.join('.', 'model_checkpoint', reg, model_label,
embedding_type, padding, str(i), 'weights.h5'))
oof_train[test_index, :] = model.predict(
inputs_val) # model.predict([x_test, x_aux_test])
proba = model.predict(
output) # model.predict([test_df_seq, test_df_seq_aux])
oof_test_skf.append(proba)
result = pd.read_csv("./data/sample_submission.csv")
result[target_labels] = proba
ithfold_path = "./cv/{}/{}/{}/{}/{}".format(reg, model_label,
embedding_type, padding, i)
if not os.path.exists(ithfold_path):
os.makedirs(ithfold_path)
result.to_csv(os.path.join(ithfold_path, 'sub.csv'), index=False)
# model.save(os.path.join(ithfold_path,'weights.h5'))
# dump oof_test and oof_train for later slacking
# oof_train:
oof_train_path = "./cv/{}/{}/{}/{}/oof_train".format(
reg, model_label, embedding_type, padding)
if not os.path.exists(oof_train_path):
os.makedirs(oof_train_path)
np.savetxt(os.path.join(oof_train_path, "oof_train.csv"),
oof_train,
fmt='%.24f',
delimiter=' ')
# oof_test: stacking version
oof_test = np.array(oof_test_skf).mean(axis=0)
oof_test_path = "./cv/{}/{}/{}/{}/oof_test".format(reg, model_label,
embedding_type, padding)
if not os.path.exists(oof_test_path):
os.makedirs(oof_test_path)
np.savetxt(os.path.join(oof_test_path, "oof_test.csv"),
oof_test,
fmt='%.24f',
delimiter=' ')
# oof_test: submission version
result[target_labels] = oof_test
oof_test_bag_path = "./cv/{}/{}/{}/{}/bagged".format(
reg, model_label, embedding_type, padding)
if not os.path.exists(oof_test_bag_path):
os.makedirs(oof_test_bag_path)
result.to_csv(os.path.join(oof_test_bag_path, "sub.csv"), index=False)
def main(opt):
if opt.disable_cudnn:
torch.backends.cudnn.enabled = False
print('Cudnn is disabled.')
logger = Logger(opt)
opt.device = torch.device('cuda:{}'.format(opt.gpus[0]))
Dataset = dataset_factory[opt.dataset]
train, val = task_factory[opt.task]
model, optimizer, start_epoch = create_model(opt)
if len(opt.gpus) > 1:
model = torch.nn.DataParallel(model,
device_ids=opt.gpus).cuda(opt.device)
else:
model = model.cuda(opt.device)
val_loader = torch.utils.data.DataLoader(Dataset(opt, 'val'),
batch_size=1,
shuffle=False,
num_workers=1,
pin_memory=True)
if opt.test:
log_dict_train, preds = val(0, opt, val_loader, model)
sio.savemat(os.path.join(opt.save_dir, 'preds.mat'),
mdict={'preds': preds})
return
train_loader = torch.utils.data.DataLoader(
Dataset(opt, 'train'),
batch_size=opt.batch_size * len(opt.gpus),
shuffle=True, # if opt.debug == 0 else False,
num_workers=opt.num_workers,
pin_memory=True)
best = -1
for epoch in range(start_epoch, opt.num_epochs + 1):
mark = epoch if opt.save_all_models else 'last'
log_dict_train, _ = train(epoch, opt, train_loader, model, optimizer)
for k, v in log_dict_train.items():
logger.scalar_summary('train_{}'.format(k), v, epoch)
logger.write('{} {:8f} | '.format(k, v))
if opt.val_intervals > 0 and epoch % opt.val_intervals == 0:
save_model(os.path.join(opt.save_dir, 'model_{}.pth'.format(mark)),
epoch, model, optimizer)
log_dict_val, preds = val(epoch, opt, val_loader, model)
for k, v in log_dict_val.items():
logger.scalar_summary('val_{}'.format(k), v, epoch)
logger.write('{} {:8f} | '.format(k, v))
if log_dict_val[opt.metric] > best:
best = log_dict_val[opt.metric]
save_model(os.path.join(opt.save_dir, 'model_best.pth'), epoch,
model)
else:
save_model(os.path.join(opt.save_dir, 'model_last.pth'), epoch,
model, optimizer)
logger.write('\n')
if epoch in opt.lr_step:
lr = opt.lr * (0.1**(opt.lr_step.index(epoch) + 1))
print('Drop LR to', lr)
for param_group in optimizer.param_groups:
param_group['lr'] = lr
logger.close()
def main(_):
train()
help='which model (default: xception)')
parser.add_argument('--data_root',
default='./',
help='data_root (default: ./)')
parser.add_argument('--nepoch',
type=int,
default=50,
help='epochs (default: 200)')
parser.add_argument('--seed',
type=int,
default='10',
help='seed (default: 1)')
parser.add_argument('--pretrain',
type=int,
default='0',
help='pretrain (default: 1)')
parser.add_argument('--data_name',
default='train',
help='data_name (default: train)')
parser.add_argument('--params_name',
default='segcaps.pkl',
help='params_name (default: segcaps.pkl)')
parser.add_argument('--load_params_name',
default='segcaps.pkl',
help='params_name (default: segcaps.pkl)')
args = parser.parse_args()
train_loader, model, decreasing_lr = init(args)
train(args, model, train_loader, decreasing_lr, wd=0.0001, momentum=0.9)
print('hhh')
print('Done!')
#%%
create_dir(para.model_dir)
create_dir(para.output_dir)
json_path = para.model_dir + '/parameters.json'
json.dump(vars(para), open(json_path, 'w'), indent=4)
# %%
graph = tf.Graph()
# %%
graph, model, data_generator = create_graph(para)
# %%
with tf.Session(config=config_setup(), graph=graph) as sess:
sess.run(tf.global_variables_initializer())
load_weights(para, sess, model)
print_num_of_trainable_parameters()
train(para, sess, model, data_generator)
# %%
para.mode = 'test'
graph, model, data_generator = create_graph(para)
with tf.Session(config=config_setup(), graph=graph) as sess:
sess.run(tf.global_variables_initializer())
load_weights(para, sess, model)
print_num_of_trainable_parameters()
test(para, sess, model, data_generator)
# %%
pred_df = pd.read_parquet(
os.path.join(para.output_dir, para.data_set + '_predict_output.parquet'))
pred_df.head(10)
def main_worker(gpu, ngpus_per_node, config):
config.gpu = gpu
#if config.is_cuda and not torch.cuda.is_available():
# raise Exception("No GPU found")
if config.gpu is not None:
print("Use GPU: {} for training".format(config.gpu))
device = get_torch_device(config.is_cuda)
if config.distributed:
if config.dist_url == "env://" and config.rank == -1:
config.rank = int(os.environ["RANK"])
if config.multiprocessing_distributed:
# For multiprocessing distributed training, rank needs to be the
# global rank among all the processes
config.rank = config.rank * ngpus_per_node + gpu
dist.init_process_group(backend=config.dist_backend,
init_method=config.dist_url,
world_size=config.world_size,
rank=config.rank)
logging.info('===> Configurations')
dconfig = vars(config)
for k in dconfig:
logging.info(' {}: {}'.format(k, dconfig[k]))
DatasetClass = load_dataset(config.dataset)
if config.test_original_pointcloud:
if not DatasetClass.IS_FULL_POINTCLOUD_EVAL:
raise ValueError(
'This dataset does not support full pointcloud evaluation.')
if config.evaluate_original_pointcloud:
if not config.return_transformation:
raise ValueError(
'Pointcloud evaluation requires config.return_transformation=true.'
)
if (config.return_transformation ^ config.evaluate_original_pointcloud):
raise ValueError(
'Rotation evaluation requires config.evaluate_original_pointcloud=true and '
'config.return_transformation=true.')
logging.info('===> Initializing dataloader')
if config.is_train:
train_data_loader, train_sampler = initialize_data_loader(
DatasetClass,
config,
phase=config.train_phase,
num_workers=config.num_workers,
augment_data=True,
shuffle=True,
repeat=True,
batch_size=config.batch_size,
limit_numpoints=config.train_limit_numpoints)
val_data_loader, val_sampler = initialize_data_loader(
DatasetClass,
config,
num_workers=config.num_val_workers,
phase=config.val_phase,
augment_data=False,
shuffle=True,
repeat=False,
batch_size=config.val_batch_size,
limit_numpoints=False)
if train_data_loader.dataset.NUM_IN_CHANNEL is not None:
num_in_channel = train_data_loader.dataset.NUM_IN_CHANNEL
else:
num_in_channel = 3 # RGB color
num_labels = train_data_loader.dataset.NUM_LABELS
else:
test_data_loader, val_sampler = initialize_data_loader(
DatasetClass,
config,
num_workers=config.num_workers,
phase=config.test_phase,
augment_data=False,
shuffle=False,
repeat=False,
batch_size=config.test_batch_size,
limit_numpoints=False)
if test_data_loader.dataset.NUM_IN_CHANNEL is not None:
num_in_channel = test_data_loader.dataset.NUM_IN_CHANNEL
else:
num_in_channel = 3 # RGB color
num_labels = test_data_loader.dataset.NUM_LABELS
logging.info('===> Building model')
NetClass = load_model(config.model)
if config.wrapper_type == 'None':
model = NetClass(num_in_channel, num_labels, config)
logging.info('===> Number of trainable parameters: {}: {}'.format(
NetClass.__name__, count_parameters(model)))
else:
wrapper = load_wrapper(config.wrapper_type)
model = wrapper(NetClass, num_in_channel, num_labels, config)
logging.info('===> Number of trainable parameters: {}: {}'.format(
wrapper.__name__ + NetClass.__name__, count_parameters(model)))
logging.info(model)
if config.weights == 'modelzoo': # Load modelzoo weights if possible.
logging.info('===> Loading modelzoo weights')
model.preload_modelzoo()
# Load weights if specified by the parameter.
elif config.weights.lower() != 'none':
logging.info('===> Loading weights: ' + config.weights)
state = torch.load(config.weights)
if config.weights_for_inner_model:
model.model.load_state_dict(state['state_dict'])
else:
if config.lenient_weight_loading:
matched_weights = load_state_with_same_shape(
model, state['state_dict'])
model_dict = model.state_dict()
model_dict.update(matched_weights)
model.load_state_dict(model_dict)
else:
init_model_from_weights(model, state, freeze_bb=False)
if config.distributed:
# For multiprocessing distributed, DistributedDataParallel constructor
# should always set the single device scope, otherwise,
# DistributedDataParallel will use all available devices.
if config.gpu is not None:
torch.cuda.set_device(config.gpu)
model.cuda(config.gpu)
# When using a single GPU per process and per
# DistributedDataParallel, we need to divide the batch size
# ourselves based on the total number of GPUs we have
config.batch_size = int(config.batch_size / ngpus_per_node)
config.num_workers = int(
(config.num_workers + ngpus_per_node - 1) / ngpus_per_node)
model = torch.nn.parallel.DistributedDataParallel(
model, device_ids=[config.gpu])
else:
model.cuda()
# DistributedDataParallel will divide and allocate batch_size to all
# available GPUs if device_ids are not set
model = torch.nn.parallel.DistributedDataParallel(model)
if config.is_train:
train(model,
train_data_loader,
val_data_loader,
config,
train_sampler=train_sampler,
ngpus_per_node=ngpus_per_node)
else:
test(model, test_data_loader, config)
def main(config, init_distributed=False):
if not torch.cuda.is_available():
raise Exception('No GPUs FOUND.')
# setup initial seed
torch.cuda.set_device(config.device_id)
torch.manual_seed(config.seed)
torch.cuda.manual_seed(config.seed)
device = config.device_id
distributed = config.distributed_world_size > 1
if init_distributed:
config.distributed_rank = distributed_utils.distributed_init(config)
setup_logging(config)
logging.info('===> Configurations')
dconfig = vars(config)
for k in dconfig:
logging.info(' {}: {}'.format(k, dconfig[k]))
DatasetClass = load_dataset(config.dataset)
if config.test_original_pointcloud:
if not DatasetClass.IS_FULL_POINTCLOUD_EVAL:
raise ValueError(
'This dataset does not support full pointcloud evaluation.')
if config.evaluate_original_pointcloud:
if not config.return_transformation:
raise ValueError(
'Pointcloud evaluation requires config.return_transformation=true.'
)
if (config.return_transformation ^ config.evaluate_original_pointcloud):
raise ValueError(
'Rotation evaluation requires config.evaluate_original_pointcloud=true and '
'config.return_transformation=true.')
logging.info('===> Initializing dataloader')
if config.is_train:
train_data_loader = initialize_data_loader(
DatasetClass,
config,
phase=config.train_phase,
num_workers=config.num_workers,
augment_data=True,
shuffle=True,
repeat=True,
batch_size=config.batch_size,
limit_numpoints=config.train_limit_numpoints)
val_data_loader = initialize_data_loader(
DatasetClass,
config,
num_workers=config.num_val_workers,
phase=config.val_phase,
augment_data=False,
shuffle=True,
repeat=False,
batch_size=config.val_batch_size,
limit_numpoints=False)
if train_data_loader.dataset.NUM_IN_CHANNEL is not None:
num_in_channel = train_data_loader.dataset.NUM_IN_CHANNEL
else:
num_in_channel = 3 # RGB color
num_labels = train_data_loader.dataset.NUM_LABELS
else:
test_data_loader = initialize_data_loader(
DatasetClass,
config,
num_workers=config.num_workers,
phase=config.test_phase,
augment_data=False,
shuffle=False,
repeat=False,
batch_size=config.test_batch_size,
limit_numpoints=False)
if test_data_loader.dataset.NUM_IN_CHANNEL is not None:
num_in_channel = test_data_loader.dataset.NUM_IN_CHANNEL
else:
num_in_channel = 3 # RGB color
num_labels = test_data_loader.dataset.NUM_LABELS
logging.info('===> Building model')
NetClass = load_model(config.model)
if config.wrapper_type == 'None':
model = NetClass(num_in_channel, num_labels, config)
logging.info('===> Number of trainable parameters: {}: {}'.format(
NetClass.__name__, count_parameters(model)))
else:
wrapper = load_wrapper(config.wrapper_type)
model = wrapper(NetClass, num_in_channel, num_labels, config)
logging.info('===> Number of trainable parameters: {}: {}'.format(
wrapper.__name__ + NetClass.__name__, count_parameters(model)))
logging.info(model)
if config.weights == 'modelzoo': # Load modelzoo weights if possible.
logging.info('===> Loading modelzoo weights')
model.preload_modelzoo()
# Load weights if specified by the parameter.
elif config.weights.lower() != 'none':
logging.info('===> Loading weights: ' + config.weights)
# state = torch.load(config.weights)
state = torch.load(
config.weights,
map_location=lambda s, l: default_restore_location(s, 'cpu'))
if config.weights_for_inner_model:
model.model.load_state_dict(state['state_dict'])
else:
if config.lenient_weight_loading:
matched_weights = load_state_with_same_shape(
model, state['state_dict'])
model_dict = model.state_dict()
model_dict.update(matched_weights)
model.load_state_dict(model_dict)
else:
model.load_state_dict(state['state_dict'])
model = model.cuda()
if distributed:
model = torch.nn.parallel.DistributedDataParallel(
module=model,
device_ids=[device],
output_device=device,
broadcast_buffers=False,
bucket_cap_mb=config.bucket_cap_mb)
if config.is_train:
train(model, train_data_loader, val_data_loader, config)
else:
test(model, test_data_loader, config)
def main():
config = get_config()
if config.is_cuda and not torch.cuda.is_available():
raise Exception("No GPU found")
# torch.set_num_threads(config.threads)
torch.manual_seed(config.seed)
if config.is_cuda:
torch.cuda.manual_seed(config.seed)
logging.info('===> Configurations')
dconfig = vars(config)
for k in dconfig:
logging.info(' {}: {}'.format(k, dconfig[k]))
DatasetClass = load_dataset(config.dataset)
logging.info('===> Initializing dataloader')
if config.is_train:
train_data_loader = initialize_data_loader(
DatasetClass,
config,
phase=config.train_phase,
threads=config.threads,
augment_data=True,
shuffle=True,
repeat=True,
batch_size=config.batch_size,
limit_numpoints=config.train_limit_numpoints)
val_data_loader = initialize_data_loader(
DatasetClass,
config,
threads=config.val_threads,
phase=config.val_phase,
augment_data=False,
shuffle=False,
repeat=False,
batch_size=config.val_batch_size,
limit_numpoints=False)
dataset = train_data_loader.dataset
else:
test_data_loader = initialize_data_loader(
DatasetClass,
config,
threads=config.threads,
phase=config.test_phase,
augment_data=False,
shuffle=False,
repeat=False,
batch_size=config.test_batch_size,
limit_numpoints=False)
dataset = test_data_loader.dataset
logging.info('===> Building model')
pipeline_model = load_pipeline(config, dataset)
logging.info(
f'===> Number of trainable parameters: {count_parameters(pipeline_model)}'
)
# Load weights if specified by the parameter.
if config.weights.lower() != 'none':
logging.info('===> Loading weights: ' + config.weights)
state = torch.load(config.weights)
pipeline_model.load_state_dict(
state['state_dict'], strict=(not config.lenient_weight_loading))
if config.pretrained_weights.lower() != 'none':
logging.info('===> Loading pretrained weights: ' +
config.pretrained_weights)
state = torch.load(config.pretrained_weights)
pipeline_model.load_pretrained_weights(state['state_dict'])
if config.is_train:
train(pipeline_model, train_data_loader, val_data_loader, config)
else:
test(pipeline_model, test_data_loader, config)
# Check if train or predict is called
if not args.train and not args.predict:
sys.exit(display_errors_dict('no_order'))
file_extension = '.csv'
if args.train:
file_name = args.train
else:
file_name = args.predict
file = os.path.join(os.getcwd(), file_name)
if os.path.exists(file):
if os.path.isfile(file):
if file.endswith(file_extension):
df_x, df_y = check_data_file(file)
if args.train:
train(args, df_x, df_y)
else:
if os.path.exists(os.path.join(os.getcwd(),
'network.save')):
evaluate_and_predict(0, df_x, df_y)
else:
sys.exit(display_errors_dict('no_network'))
else:
sys.exit(display_errors_dict('endswith'))
else:
sys.exit(display_errors_dict('isfile'))
else:
sys.exit(display_errors_dict('exists'))
def main():
config = get_config()
if config.resume:
json_config = json.load(open(config.resume + '/config.json', 'r'))
json_config['resume'] = config.resume
config = edict(json_config)
if config.is_cuda and not torch.cuda.is_available():
raise Exception("No GPU found")
device = get_torch_device(config.is_cuda)
logging.info('===> Configurations')
dconfig = vars(config)
for k in dconfig:
logging.info(' {}: {}'.format(k, dconfig[k]))
DatasetClass = load_dataset(config.dataset)
if config.test_original_pointcloud:
if not DatasetClass.IS_FULL_POINTCLOUD_EVAL:
raise ValueError(
'This dataset does not support full pointcloud evaluation.')
if config.evaluate_original_pointcloud:
if not config.return_transformation:
raise ValueError(
'Pointcloud evaluation requires config.return_transformation=true.'
)
if (config.return_transformation ^ config.evaluate_original_pointcloud):
raise ValueError(
'Rotation evaluation requires config.evaluate_original_pointcloud=true and '
'config.return_transformation=true.')
logging.info('===> Initializing dataloader')
if config.is_train:
train_data_loader = initialize_data_loader(
DatasetClass,
config,
phase=config.train_phase,
threads=config.threads,
augment_data=True,
shuffle=True,
repeat=True,
batch_size=config.batch_size,
limit_numpoints=config.train_limit_numpoints)
val_data_loader = initialize_data_loader(
DatasetClass,
config,
threads=config.val_threads,
phase=config.val_phase,
augment_data=False,
shuffle=True,
repeat=False,
batch_size=config.val_batch_size,
limit_numpoints=False)
if train_data_loader.dataset.NUM_IN_CHANNEL is not None:
num_in_channel = train_data_loader.dataset.NUM_IN_CHANNEL
else:
num_in_channel = 3 # RGB color
num_labels = train_data_loader.dataset.NUM_LABELS
else:
test_data_loader = initialize_data_loader(
DatasetClass,
config,
threads=config.threads,
phase=config.test_phase,
augment_data=False,
shuffle=False,
repeat=False,
batch_size=config.test_batch_size,
limit_numpoints=False)
if test_data_loader.dataset.NUM_IN_CHANNEL is not None:
num_in_channel = test_data_loader.dataset.NUM_IN_CHANNEL
else:
num_in_channel = 3 # RGB color
num_labels = test_data_loader.dataset.NUM_LABELS
logging.info('===> Building model')
NetClass = load_model(config.model)
if config.wrapper_type == 'None':
model = NetClass(num_in_channel, num_labels, config)
logging.info('===> Number of trainable parameters: {}: {}'.format(
NetClass.__name__, count_parameters(model)))
else:
wrapper = load_wrapper(config.wrapper_type)
model = wrapper(NetClass, num_in_channel, num_labels, config)
logging.info('===> Number of trainable parameters: {}: {}'.format(
wrapper.__name__ + NetClass.__name__, count_parameters(model)))
logging.info(model)
model = model.to(device)
if config.weights == 'modelzoo': # Load modelzoo weights if possible.
logging.info('===> Loading modelzoo weights')
model.preload_modelzoo()
# Load weights if specified by the parameter.
elif config.weights.lower() != 'none':
logging.info('===> Loading weights: ' + config.weights)
state = torch.load(config.weights)
if config.weights_for_inner_model:
model.model.load_state_dict(state['state_dict'])
else:
if config.lenient_weight_loading:
matched_weights = load_state_with_same_shape(
model, state['state_dict'])
model_dict = model.state_dict()
model_dict.update(matched_weights)
model.load_state_dict(model_dict)
else:
model.load_state_dict(state['state_dict'])
if config.is_train:
train(model, train_data_loader, val_data_loader, config)
else:
test(model, test_data_loader, config)
def main(config):
# load the configurations
setup_logging()
if os.path.exists('config.yaml'):
logging.info('===> Loading exsiting config file')
config = OmegaConf.load('config.yaml')
logging.info('===> Loaded exsiting config file')
logging.info(config.pretty())
# Create Dataset and Dataloader
if config.data.dataset == 'sunrgbd':
from lib.datasets.sunrgbd.sunrgbd_detection_dataset import SunrgbdDetectionVotesDataset, MAX_NUM_OBJ
from lib.datasets.sunrgbd.model_util_sunrgbd import SunrgbdDatasetConfig
dataset_config = SunrgbdDatasetConfig()
train_dataset = SunrgbdDetectionVotesDataset(
'train',
num_points=config.data.num_points,
augment=True,
use_color=config.data.use_color,
use_height=(not config.data.no_height),
use_v1=(not config.data.use_sunrgbd_v2),
data_ratio=config.data.data_ratio)
test_dataset = SunrgbdDetectionVotesDataset(
'val',
num_points=config.data.num_points,
augment=False,
use_color=config.data.use_color,
use_height=(not config.data.no_height),
use_v1=(not config.data.use_sunrgbd_v2))
elif config.data.dataset == 'scannet':
from lib.datasets.scannet.scannet_detection_dataset import ScannetDetectionDataset, MAX_NUM_OBJ
from lib.datasets.scannet.model_util_scannet import ScannetDatasetConfig
dataset_config = ScannetDatasetConfig()
train_dataset = ScannetDetectionDataset(
'train',
num_points=config.data.num_points,
augment=True,
use_color=config.data.use_color,
use_height=(not config.data.no_height),
data_ratio=config.data.data_ratio)
test_dataset = ScannetDetectionDataset(
'val',
num_points=config.data.num_points,
augment=False,
use_color=config.data.use_color,
use_height=(not config.data.no_height))
else:
logging.info('Unknown dataset %s. Exiting...' % (config.data.dataset))
exit(-1)
COLLATE_FN = None
if config.data.voxelization:
from models.backbone.sparseconv.voxelized_dataset import VoxelizationDataset, collate_fn
train_dataset = VoxelizationDataset(train_dataset,
config.data.voxel_size)
test_dataset = VoxelizationDataset(test_dataset,
config.data.voxel_size)
COLLATE_FN = collate_fn
logging.info('training: {}, testing: {}'.format(len(train_dataset),
len(test_dataset)))
train_dataloader = DataLoader(train_dataset,
batch_size=config.data.batch_size,
shuffle=True,
num_workers=config.data.num_workers,
worker_init_fn=my_worker_init_fn,
collate_fn=COLLATE_FN)
test_dataloader = DataLoader(test_dataset,
batch_size=config.data.num_workers,
shuffle=True,
num_workers=config.data.num_workers,
worker_init_fn=my_worker_init_fn,
collate_fn=COLLATE_FN)
logging.info('train dataloader: {}, test dataloader: {}'.format(
len(train_dataloader), len(test_dataloader)))
# Init the model and optimzier
MODEL = importlib.import_module('models.' +
config.net.model) # import network module
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
num_input_channel = int(
config.data.use_color) * 3 + int(not config.data.no_height) * 1
if config.net.model == 'boxnet':
Detector = MODEL.BoxNet
else:
Detector = MODEL.VoteNet
net = Detector(num_class=dataset_config.num_class,
num_heading_bin=dataset_config.num_heading_bin,
num_size_cluster=dataset_config.num_size_cluster,
mean_size_arr=dataset_config.mean_size_arr,
num_proposal=config.net.num_target,
input_feature_dim=num_input_channel,
vote_factor=config.net.vote_factor,
sampling=config.net.cluster_sampling,
backbone=config.net.backbone)
if config.net.weights is not None:
assert config.net.backbone == "sparseconv", "only support sparseconv"
print('===> Loading weights: ' + config.net.weights)
state = torch.load(
config.net.weights,
map_location=lambda s, l: default_restore_location(s, 'cpu'))
model = net
if config.net.is_train:
model = net.backbone_net.net
matched_weights = load_state_with_same_shape(model,
state['state_dict'])
model_dict = model.state_dict()
model_dict.update(matched_weights)
model.load_state_dict(model_dict)
# from pdb import set_trace; set_trace()
net.to(device)
if config.net.is_train:
train(net, train_dataloader, test_dataloader, dataset_config, config)
else:
test(net, test_dataloader, dataset_config, config)
