def get_data_model_fns(args):
dtype = get_dtype(args)
train_set, test_set, task, data_info = data_utils.make_ds_pmap_fullbatch(
args.dataset_name, dtype, truncate_to=args.subset_train_to)
net_apply, net_init = models.get_model(args.model_name, data_info)
net_apply = precision_utils.rewrite_high_precision(net_apply)
(likelihood_factory, predict_fn, ensemble_upd_fn, metrics_fns,
tabulate_metrics) = train_utils.get_task_specific_fns(task, data_info)
log_likelihood_fn = likelihood_factory(args.temperature)
log_prior_fn, log_prior_diff_fn = losses.make_gaussian_log_prior(
args.weight_decay, args.temperature)
key, net_init_key = jax.random.split(jax.random.PRNGKey(args.seed), 2)
init_data = jax.tree_map(lambda elem: elem[0][:1], train_set)
params, net_state = net_init(net_init_key, init_data, True)
param_types = tree_utils.tree_get_types(params)
assert all([
p_type == dtype for p_type in param_types
]), ("Params data types {} do not match specified data type {}".format(
param_types, dtype))
return (train_set, test_set, net_apply, params, net_state, key,
log_likelihood_fn, log_prior_fn, log_prior_diff_fn, predict_fn,
ensemble_upd_fn, metrics_fns, tabulate_metrics)
def main(args):
x, fx = get_data(args)
device = torch.device("cuda" if args.cuda else "cpu")
train_data, val_data = split_data(args, x, fx)
if args.save_splits:
save_splits(train_data, val_data)
train_loader, val_loader = get_loaders(train_data, val_data)
model = get_model(args)
trainer = get_trainer(model, train_loader, val_loader, device, args)
trainer.train()
def __init__(self,
data_name,
model_type,
model_name_or_path,
do_lower_case,
lang_id,
n_best_size,
max_answer_length,
null_score_diff_threshold,
learning_rate: float = 5e-5):
super().__init__()
self.save_hyperparameters()
# prepare model
model = get_model(self.hparams.model_type,
self.hparams.model_name_or_path)
self.model = model
# for SQuAD, KorQuAD
self.version_2_with_negative = is_squad_version_2(
self.hparams.data_name)
'fs_layer': args.fs_layer,
'save_iso': args.save_iso
}
layer_params = args.nodes.split()
node_str = "_".join(layer_params)
layer_params = [int(x) for x in layer_params if int(x) > 0]
print('Here is the number of each layer:')
for i, num in enumerate(layer_params):
print("Layer {} has {} nodes".format(i, num))
if args.model_type == 'mlp':
if not args.save_iso:
model = get_model(input_dim,
layer_params=layer_params,
hyper_params=hyper_params)
else:
model, model_iso = get_model(input_dim,
layer_params=layer_params,
hyper_params=hyper_params)
elif args.model_type == 'conv':
model = get_conv_model(input_dim)
# Compile model
if not args.trans_supervision:
#adam = keras.optimizers.Nadam(lr=args.lr, beta_1=0.9, beta_2=0.999)
adam = Adam(lr=args.lr,
beta_1=0.9,
beta_2=0.999,
epsilon=None,
def test(args):
# output folder
outdir = '%s/bfp_vis' % (args.dataset)
if not os.path.exists(outdir):
os.makedirs(outdir)
# data transforms
input_transform = transform.Compose([
transform.ToTensor(),
transform.Normalize([.485, .456, .406], [.229, .224, .225])
])
# dataset
# args.eval = True
if args.eval:
print("================ eval True ================\n")
testset = get_segmentation_dataset(args.dataset,
split='val',
mode='testval',
transform=input_transform)
else: #set split='test' for test set
print("================ eval False ================\n")
testset = get_segmentation_dataset(args.dataset,
split='test',
mode='vis',
transform=input_transform)
# dataloader
loader_kwargs = {'num_workers': args.workers, 'pin_memory': True} \
if args.cuda else {}
test_data = data.DataLoader(testset,
batch_size=args.test_batch_size,
drop_last=False,
shuffle=False,
collate_fn=test_batchify_fn,
**loader_kwargs)
if args.model_zoo is not None:
model = get_model(args.model_zoo, pretrained=True)
else:
model = get_segmentation_model(args.model,
dataset=args.dataset,
backbone=args.backbone,
aux=args.aux,
se_loss=args.se_loss,
norm_layer=BatchNorm2d,
base_size=args.base_size,
crop_size=args.crop_size,
multi_grid=args.multi_grid,
multi_dilation=args.multi_dilation)
# resuming checkpoint
if args.resume is None or not os.path.isfile(args.resume):
raise RuntimeError("=> no checkpoint found at '{}'".format(
args.resume))
checkpoint = torch.load(args.resume)
# strict=False, so that it is compatible with old pytorch saved models
model.load_state_dict(checkpoint['state_dict'], strict=False)
#print(model)
num_class = testset.num_class
evaluator = MultiEvalModule(model,
testset.num_class,
multi_scales=args.multi_scales).cuda()
evaluator.eval()
tbar = tqdm(test_data)
def eval_batch(image, dst, evaluator, eval_mode):
if eval_mode:
# evaluation mode on validation set
targets = dst
outputs = evaluator.parallel_forward(image)
batch_inter, batch_union, batch_correct, batch_label = 0, 0, 0, 0
for output, target in zip(outputs, targets):
correct, labeled = utils.batch_pix_accuracy(
output.data.cpu(), target)
inter, union = utils.batch_intersection_union(
output.data.cpu(), target, testset.num_class)
batch_correct += correct
batch_label += labeled
batch_inter += inter
batch_union += union
return batch_correct, batch_label, batch_inter, batch_union
else:
# Visualize and dump the results
im_paths = dst
outputs = evaluator.parallel_forward(image)
predicts = [
torch.max(output, 1)[1].cpu().numpy() + testset.pred_offset
for output in outputs
]
for predict, impath in zip(predicts, im_paths):
mask = utils.get_mask_pallete(predict, args.dataset)
outname = os.path.splitext(impath)[0] + '.png'
outname = outname.replace('leftImg8bit', 'gtFine_labelIds')
mask.save(os.path.join(outdir, outname))
# dummy outputs for compatible with eval mode
return 0, 0, 0, 0
total_inter, total_union, total_correct, total_label = \
np.int64(0), np.int64(0), np.int64(0), np.int64(0)
for i, (image, dst) in enumerate(tbar):
if torch_ver == "0.3":
image = Variable(image, volatile=True)
correct, labeled, inter, union = eval_batch(
image, dst, evaluator, args.eval)
else:
with torch.no_grad():
correct, labeled, inter, union = eval_batch(
image, dst, evaluator, args.eval)
pixAcc, mIoU, IoU = 0, 0, 0
if args.eval:
total_correct += correct.astype('int64')
total_label += labeled.astype('int64')
total_inter += inter.astype('int64')
total_union += union.astype('int64')
pixAcc = np.float64(1.0) * total_correct / (
np.spacing(1, dtype=np.float64) + total_label)
IoU = np.float64(1.0) * total_inter / (
np.spacing(1, dtype=np.float64) + total_union)
mIoU = IoU.mean()
tbar.set_description('pixAcc: %.4f, mIoU: %.4f' % (pixAcc, mIoU))
return pixAcc, mIoU, IoU, num_class
# Process data
x_test = x_test.reshape((-1, input_shape[0], input_shape[1]))
x_train = x_train.reshape((-1, input_shape[0], input_shape[1]))
y_test = to_categorical(ds.class_offset(y_test, args.dataset), nb_class)
y_train = to_categorical(ds.class_offset(y_train, args.dataset), nb_class)
# Augment data
x_train, y_train, augmentation_tags = run_augmentation(x_train, y_train, args)
model_prefix = "%s_%s%s"%(args.model, args.dataset, augmentation_tags)
nb_iterations = args.iterations
batch_size = args.batch_size
nb_epochs = np.ceil(nb_iterations * (batch_size / x_train.shape[0])).astype(int)
model = mod.get_model(args.model, input_shape, nb_class)
reduce_lr = ReduceLROnPlateau(monitor='acc', factor=0.1, patience=np.ceil(nb_epochs/20.).astype(int), verbose=args.verbose, mode='auto', min_lr=1e-5, cooldown=np.ceil(nb_epochs/40.).astype(int))
if args.save:
if not os.path.exists(args.weight_dir):
os.mkdir(args.weight_dir)
if not os.path.exists(os.path.join(args.weight_dir, model_prefix)):
os.mkdir(os.path.join(args.weight_dir, model_prefix))
if not os.path.exists(args.log_dir):
os.mkdir(args.log_dir)
# model_checkpoint = ModelCheckpoint(os.path.join(args.weight_dir, model_prefix, "%s_best_train_acc_weights.h5" % (model_prefix)), verbose=1, monitor='acc', save_best_only=True)
if not os.path.exists(os.path.join(args.log_dir, model_prefix)):
os.mkdir(os.path.join(args.log_dir, model_prefix))
csv_logger = CSVLogger(os.path.join(args.log_dir, '%s.csv' % (model_prefix)))
# Other parameters
parser.add_argument("--input_dir",
type=str,
default="./input",
help="path to input directory")
parser.add_argument("--output_dir",
type=str,
default="./output",
help="path to output directory")
args = parser.parse_args()
# ------------------------------------------------------------------------------------------------------------------
model_type = args.model_type.lower()
tokenizer = get_tokenizer(model_type)
model = get_model(model_type)
mrc_pipeline = pipeline("question-answering",
tokenizer=tokenizer,
model=model)
input_dir = args.input_dir
output_dir = args.output_dir
prepare_dir(output_dir)
fns = {
"input": os.path.join(args.input_dir, "qg.json"),
"output": os.path.join(args.output_dir, "output.json")
}
run_predict(mrc_pipeline, fns)
def pred(config, inpath, outputpath):
test_df_c = create_charlist(inpath, output_file=None)
test_df_w = create_wordlist(inpath, output_file=None, tongyi_file=config.tongyici)
# WORD LEVEL TOKEN
df_w = pd.read_csv(config.input_file_word, encoding="utf-8") # process_word.csv， 原始数据分词
question1_w = df_w['question1'].values
question2_w = df_w['question2'].values
test_question1_w = test_df_w['question1'].values
test_question2_w = test_df_w['question2'].values
tokenizer = Tokenizer(num_words=config.word_vocab_size)
tokenizer.fit_on_texts(list(question1_w) + list(question2_w))
list_tokenized_question1_w = tokenizer.texts_to_sequences(test_question1_w)
list_tokenized_question2_w = tokenizer.texts_to_sequences(test_question2_w)
X_test_w_q1 = pad_sequences(list_tokenized_question1_w, maxlen=config.word_seq_length)
X_test_w_q2 = pad_sequences(list_tokenized_question2_w, maxlen=config.word_seq_length)
# CHAR LEVEL TOKEN
df_c = pd.read_csv(config.input_file_char, encoding="utf-8")
question1_c = df_c['question1'].values
question2_c = df_c['question2'].values
test_question1_c = test_df_c['question1'].values
test_question2_c = test_df_c['question2'].values
tokenizer = Tokenizer(num_words=config.char_vocab_size)
tokenizer.fit_on_texts(list(question1_c) + list(question2_c))
list_tokenized_question1_c = tokenizer.texts_to_sequences(test_question1_c)
list_tokenized_question2_c = tokenizer.texts_to_sequences(test_question2_c)
X_test_c_q1 = pad_sequences(list_tokenized_question1_c, maxlen=config.char_seq_length)
X_test_c_q2 = pad_sequences(list_tokenized_question2_c, maxlen=config.char_seq_length)
pred_oob = np.zeros(shape=(len(X_test_w_q1), 1))
model = get_model(config.kernel_name2)(config, None,None)
count = 0
for index in range(config.cv_folds):
if index in [0, 3, 7, 8]:
continue
bst_model_path = config.model_dir2 + config.kernel_name2 + '_weight_%d.h5' % index
model.load_weights(bst_model_path)
y_predict = model.predict([X_test_w_q1, X_test_w_q2, X_test_c_q1, X_test_c_q2], batch_size=128, verbose=1)
pred_oob += y_predict
print("Epoch: %d is over." %(index))
count += 1
model = get_model(config.kernel_name1)(config, None,None)
for index in range(config.cv_folds):
if index in [3, 7, 8]:
continue
bst_model_path = config.model_dir1 + config.kernel_name1 + '_weight_%d.h5' % index
model.load_weights(bst_model_path)
y_predict = model.predict([X_test_w_q1, X_test_w_q2, X_test_c_q1, X_test_c_q2], batch_size=128, verbose=1)
pred_oob += y_predict
print("Epoch: %d is over." %(index))
count += 1
# pred_oob /= config.cv_folds
pred_oob /= count
pred_oob1 = (pred_oob > 0.5).astype(int)
idx = np.reshape(np.arange(1, len(pred_oob1)+1), [-1, 1])
preds = np.reshape(pred_oob1, [-1, 1])
data = np.concatenate((idx, preds),axis=1)
dataframe = pd.DataFrame(data, columns=['idx','pred'])
dataframe.to_csv(outputpath, index=None, header=None, sep='\t')
