def train(weights_path, epochs, batch_size, initial_epoch, kl_start_epoch,
kl_alpha_increase_per_epoch):
"""Trains a model."""
print('loading data...')
# Loads or creates training data.
input_shape, train, valid, train_targets, valid_targets = get_train_data()
print('getting model...')
# Loads or creates model.
model, checkpoint_path, kl_alpha = get_model(input_shape,
scale_factor=len(train) /
batch_size,
weights_path=weights_path)
# Sets callbacks.
checkpointer = ModelCheckpoint(checkpoint_path,
verbose=1,
save_weights_only=True,
save_best_only=True)
scheduler = LearningRateScheduler(schedule)
annealer = Callback() if kl_alpha is None else AnnealingCallback(
kl_alpha, kl_start_epoch, kl_alpha_increase_per_epoch)
print('fitting model...')
# Trains model.
model.fit(train,
train_targets,
batch_size,
epochs,
initial_epoch=initial_epoch,
callbacks=[checkpointer, scheduler, annealer],
validation_data=(valid, valid_targets))
def train():
os.makedirs(hparams.result_dir + "weights/", exist_ok=True)
summary_writer = tf.summary.create_file_writer(hparams.result_dir)
wavenet = WaveNet(hparams.num_mels, hparams.upsample_scales)
loss_fn = CrossEntropyLoss(num_classes=256)
lr_schedule = tf.keras.optimizers.schedules.ExponentialDecay(
hparams.learning_rate,
decay_steps=hparams.exponential_decay_steps,
decay_rate=hparams.exponential_decay_rate)
optimizer = tf.keras.optimizers.Adam(learning_rate=lr_schedule,
beta_1=hparams.beta_1)
if hparams.load_path is not None:
wavenet.load_weights(hparams.load_path)
step = np.load(hparams.result_dir + "weights/step.npy")
step = step
print(f"weights load: {hparams.load_path}")
else:
step = 0
for epoch in range(hparams.epoch):
train_data = get_train_data()
for x, mel_sp, y in train_data:
loss = train_step(wavenet, x, mel_sp, y, loss_fn, optimizer)
with summary_writer.as_default():
tf.summary.scalar('train/loss', loss, step=step)
step += 1
if epoch % hparams.save_interval == 0:
print(f'Step {step}, Loss: {loss}')
np.save(hparams.result_dir + f"weights/step.npy", np.array(step))
wavenet.save_weights(hparams.result_dir +
f"weights/wavenet_{epoch:04}")
np.save(hparams.result_dir + f"weights/step.npy", np.array(step))
wavenet.save_weights(hparams.result_dir + f"weights/wavenet_{epoch:04}")
def train(**kwargs):
# ---------------------- 更新参数 ----------------------
opt = DefaultConfig()
opt.update(**kwargs)
opt.printf()
# ---------------------- 数据处理 ----------------------
# 获取数据
train1, train2 = get_train_data(opt)
# 获取样本
# train_sample = get_sample(train1, train2, load=True)
# 获取特征
# train_feat = get_feat(train1, train_sample)
# 获取标签
# train_all = get_label(train_feat, opt)
# gc.collect()
# train_all.to_hdf('/home/xuwenchao/dyj-storage/all-feat/feat_23_24_label.hdf', 'w', complib='blosc', complevel=5)
train_all = pd.read_hdf(
'/home/xuwenchao/dyj-storage/all-feat/feat_23_24_label.hdf')
print(train_all.shape)
# 取出需要用的特征
# opt['model_name'] = 'lgb_1_2017-09-15#19:50:48_0.58820.pkl'
# gbm, use_feat = load_model(opt)
# predictors_100 = pd.DataFrame(data={'feature_name': gbm.feature_name(), 'feature_importance': gbm.feature_importance()})
# predictors_100 = predictors_100.sort_values(by=['feature_importance'], ascending=False)['feature_name'].values[:100]
# use_feat = list(predictors_100) + ['orderid', 'geohashed_end_loc', 'label'] + ['sloc_eloc_common_eloc_count', 'sloc_eloc_common_sloc_count', 'sloc_eloc_common_conn1_count', 'sloc_eloc_common_conn2_count', 'sloc_eloc_common_eloc_rate', 'sloc_eloc_common_sloc_rate', 'sloc_eloc_common_conn1_rate', 'sloc_eloc_common_conn2_rate', 'user_sloc_eloc_common_eloc_count', 'user_sloc_eloc_common_sloc_count', 'user_sloc_eloc_common_conn1_count', 'user_sloc_eloc_common_conn2_count', 'user_sloc_eloc_common_eloc_rate', 'user_sloc_eloc_common_sloc_rate', 'user_sloc_eloc_common_conn1_rate', 'user_sloc_eloc_common_conn2_rate']
# train_all = train_all[use_feat]
# gc.collect()
# -------------------- 训练第一层 ------------------------
# ********* 准备数据 **********
# 划分验证集
train, val = train_test_split(train_all, test_size=0.1)
# 定义使用哪些特征
# opt['model_name'] = 'lgb_1_2017-09-15#19:50:48_0.58820.pkl'
# gbm, use_feat = load_model(opt)
filters = set([
'orderid', 'userid', 'biketype', 'geohashed_start_loc', 'bikeid',
'starttime', 'geohashed_end_loc', 'label'
])
predictors = list(
filter(lambda x: x not in filters, train_all.columns.tolist()))
# predictors = pd.DataFrame(data={'feature_name': gbm.feature_name(), 'feature_importance': gbm.feature_importance()})
# predictors = predictors.sort_values(by=['feature_importance'], ascending=False)['feature_name'].values[:100]
# use_feat = list(predictors) + ['orderid', 'geohashed_end_loc'] + ['sloc_eloc_common_eloc_count', 'sloc_eloc_common_sloc_count', 'sloc_eloc_common_conn1_count', 'sloc_eloc_common_conn2_count', 'sloc_eloc_common_eloc_rate', 'sloc_eloc_common_sloc_rate', 'sloc_eloc_common_conn1_rate', 'sloc_eloc_common_conn2_rate', 'user_sloc_eloc_common_eloc_count', 'user_sloc_eloc_common_sloc_count', 'user_sloc_eloc_common_conn1_count', 'user_sloc_eloc_common_conn2_count', 'user_sloc_eloc_common_eloc_rate', 'user_sloc_eloc_common_sloc_rate', 'user_sloc_eloc_common_conn1_rate', 'user_sloc_eloc_common_conn2_rate']
# predictors = list(predictors_100) + ['sloc_eloc_common_eloc_count', 'sloc_eloc_common_sloc_count', 'sloc_eloc_common_conn1_count', 'sloc_eloc_common_conn2_count', 'sloc_eloc_common_eloc_rate', 'sloc_eloc_common_sloc_rate', 'sloc_eloc_common_conn1_rate', 'sloc_eloc_common_conn2_rate', 'user_sloc_eloc_common_eloc_count', 'user_sloc_eloc_common_sloc_count', 'user_sloc_eloc_common_conn1_count', 'user_sloc_eloc_common_conn2_count', 'user_sloc_eloc_common_eloc_rate', 'user_sloc_eloc_common_sloc_rate', 'user_sloc_eloc_common_conn1_rate', 'user_sloc_eloc_common_conn2_rate']
print('使用的特征：{}维\n'.format(len(predictors)), predictors)
# 定义数据集
X_train = train[predictors]
y_train = train['label']
X_val = val[predictors]
y_val = val['label']
del train, val
gc.collect()
# ********* LightGBM *********
# 数据集
lgb_train = lgb.Dataset(X_train, y_train)
lgb_val = lgb.Dataset(X_val, y_val, reference=lgb_train)
# 配置
params = {
'objective': 'binary',
'metric': {'auc', 'binary_logloss'},
'is_unbalance': True,
'num_leaves': opt['lgb_leaves'],
'learning_rate': opt['lgb_lr'],
'feature_fraction': 0.886,
'bagging_fraction': 0.886,
'bagging_freq': 5
}
gc.collect()
# ********** 开始训练 *********
gbm1 = lgb.train(params,
lgb_train,
num_boost_round=1200,
valid_sets=[lgb_train, lgb_val],
early_stopping_rounds=5)
gc.collect()
# # ********* 保存模型 *********
cur_time = datetime.datetime.now().strftime('%Y-%m-%d#%H:%M:%S')
# save_path = '{}/{}_{}_{:.5f}.pkl'.format(opt['model_dir'], 'lgb', cur_time, score[0])
save_path = '{}/{}_{}.pkl'.format(opt['model_dir'], 'lgb', cur_time)
with open(save_path, 'wb') as fout:
pickle.dump(gbm1, fout)
print('保存模型：', save_path)
gc.collect()
# # ********* 评估 *********
# # 在训练集上看效果
del X_train, y_train, X_val, y_val
gc.collect()
score = get_score(train_all, predictors, gbm1, opt)
print('训练集分数：{}'.format(score))
import sys
sys.exit(0)
# save_path = '{}/{}.pkl'.format(opt['model_dir'], 'lgb_1_300_top25')
# with open(save_path, 'wb') as fout:
# pickle.dump(gbm1, fout)
# print('保存模型(第一层)：', save_path)
# ********* save predict *****
# train_all[['orderid', 'geohashed_end_loc', 'pred']].to_hdf('/home/xuwenchao/dyj-storage/train2324_80_pred_res.hdf', 'w', complib='blosc', complevel=5)
# print('Save train_pred_res.hdf successful!!!')
# import sys
# sys.exit(0)
# -------------------- 训练第二层 ------------------------
# opt['model_name'] = 'lgb_1_300_top25.pkl'
# gbm1, use_feat1 = load_model(opt)
# train_all.loc[:, 'pred'] = gbm1.predict(train_all[use_feat1])
# 去掉重要性较低的特征，筛选出排名前十的候选样本，重新训练模型（后期可以载入模型finetune，
# 尤其是对于样本量较少的情况，甚至可以选前5，
# 但15可以覆盖99.5%的原始label，10可以覆盖98%的原始label，
# 这两者可能会好一些，备选方案：5(+finetune)，10(+finetune)，15(+finetune)）
predictors = pd.DataFrame(
data={
'feature_name': gbm1.feature_name(),
'feature_importance': gbm1.feature_importance()
})
predictors = predictors[
predictors['feature_importance'] > 0]['feature_name'].values
print('第二层使用的特征：{}维\n'.format(len(predictors)), predictors)
train_all = train_all.sort_values(
by=['orderid', 'pred'], ascending=False).groupby('orderid').head(15)
# train_all = rank(train_all, 'orderid', 'pred', ascending=False)
del train_all['pred']
print('第二层数据：', train_all.shape)
# ********* 准备数据 **********
# 划分验证集
train, val = train_test_split(train_all, test_size=0.1)
# 定义数据集
X_train = train[predictors]
y_train = train['label']
X_val = val[predictors]
y_val = val['label']
del train, val
gc.collect()
# 数据集
lgb_train = lgb.Dataset(X_train, y_train)
lgb_val = lgb.Dataset(X_val, y_val, reference=lgb_train)
# ********** 开始训练 *********
gbm2 = lgb.train(params,
lgb_train,
num_boost_round=1200,
valid_sets=[lgb_train, lgb_val],
early_stopping_rounds=5
# init_model=gbm1 # finetune
)
# ********* 评估 *********
# 在训练集上看效果
score = get_score(train_all, predictors, gbm2, opt)
print('训练集分数(第二层)：{}'.format(score))
# ********* 保存模型 *********
cur_time = datetime.datetime.now().strftime('%Y-%m-%d#%H:%M:%S')
save_path = '{}/{}_{}_{:.5f}.pkl'.format(opt['model_dir'], 'lgb_2',
cur_time, score[0])
with open(save_path, 'wb') as fout:
pickle.dump(gbm2, fout)
print('保存模型(第二层)：', save_path)
# save_path = '{}/{}.pkl'.format(opt['model_dir'], 'lgb_2_300_top15')
# with open(save_path, 'wb') as fout:
# pickle.dump(gbm2, fout)
# print('保存模型(第二层)：', save_path)
import sys
sys.exit(0)
# -------------------- 训练第三层 ------------------------
# 筛选出排名前五的候选样本
predictors = pd.DataFrame(
data={
'feature_name': gbm2.feature_name(),
'feature_importance': gbm2.feature_importance()
})
predictors = predictors[
predictors['feature_importance'] > 0]['feature_name'].values
print('第三层使用的特征：{}维\n'.format(len(predictors)), predictors)
train_all = train_all.sort_values(
by=['orderid', 'pred'], ascending=False).groupby('orderid').head(10)
# train_all = rank(train_all, 'orderid', 'pred', ascending=False)
del train_all['pred']
print('第三层数据：', train_all.shape)
# ********* 准备数据 **********
# 划分验证集
train, val = train_test_split(train_all, test_size=0.1)
# 定义数据集
X_train = train[predictors]
y_train = train['label']
X_val = val[predictors]
y_val = val['label']
del train, val
gc.collect()
# 数据集
lgb_train = lgb.Dataset(X_train, y_train)
lgb_val = lgb.Dataset(X_val, y_val, reference=lgb_train)
# ********** 开始训练 *********
gbm3 = lgb.train(params,
lgb_train,
num_boost_round=1200,
valid_sets=[lgb_train, lgb_val],
early_stopping_rounds=5
# init_model=gbm2 # finetune
)
# ********* 评估 *********
# 在训练集上看效果
score = get_score(train_all, predictors, gbm3, opt)
print('训练集分数(第三层)：{}'.format(score))
# ********* 保存模型 *********
cur_time = datetime.datetime.now().strftime('%Y-%m-%d#%H:%M:%S')
save_path = '{}/{}_{}_{:.5f}.pkl'.format(opt['model_dir'], 'lgb_3',
cur_time, score[0])
with open(save_path, 'wb') as fout:
pickle.dump(gbm3, fout)
print('保存模型(第三层)：', save_path)
save_path = '{}/{}.pkl'.format(opt['model_dir'], 'lgb_3_300_top10')
with open(save_path, 'wb') as fout:
pickle.dump(gbm3, fout)
print('保存模型(第三层)：', save_path)
# -------------------- 训练第四层 ------------------------
# 筛选出排名前三的候选样本
predictors = pd.DataFrame(
data={
'feature_name': gbm3.feature_name(),
'feature_importance': gbm3.feature_importance()
})
predictors = predictors[
predictors['feature_importance'] > 0]['feature_name'].values
print('第四层使用的特征：{}维\n'.format(len(predictors)), predictors)
train_all = train_all.sort_values(
by=['orderid', 'pred'], ascending=False).groupby('orderid').head(5)
# train_all = rank(train_all, 'orderid', 'pred', ascending=False)
del train_all['pred']
print('第四层数据：', train_all.shape)
# ********* 准备数据 **********
# 划分验证集
train, val = train_test_split(train_all, test_size=0.1)
# 定义数据集
X_train = train[predictors]
y_train = train['label']
X_val = val[predictors]
y_val = val['label']
del train, val
gc.collect()
# 数据集
lgb_train = lgb.Dataset(X_train, y_train)
lgb_val = lgb.Dataset(X_val, y_val, reference=lgb_train)
# ********** 开始训练 *********
gbm4 = lgb.train(params,
lgb_train,
num_boost_round=1200,
valid_sets=[lgb_train, lgb_val],
early_stopping_rounds=5
# init_model=gbm3 # finetune
)
# ********* 评估 *********
# 在训练集上看效果
score = get_score(train_all, predictors, gbm4, opt)
print('训练集分数(第四层)：{}'.format(score))
# ********* 保存模型 *********
cur_time = datetime.datetime.now().strftime('%Y-%m-%d#%H:%M:%S')
save_path = '{}/{}_{}_{:.5f}.pkl'.format(opt['model_dir'], 'lgb_4',
cur_time, score[0])
with open(save_path, 'wb') as fout:
pickle.dump(gbm4, fout)
print('保存模型(第四层)：', save_path)
save_path = '{}/{}.pkl'.format(opt['model_dir'], 'lgb_4_300_top5')
with open(save_path, 'wb') as fout:
pickle.dump(gbm4, fout)
print('保存模型(第四层)：', save_path)
def feed_dict(train):
if train:
x1s, x2s, ys, _ = dataset.get_train_data()
else:
x1s, x2s, ys, _ = dataset.get_test_data()
return {x1: x1s, x2: x2s, y_: ys}
lr_scheduler = torch.optim.lr_scheduler.MultiStepLR(optimizer=model.optimizer,
milestones=[5, 10], gamma=0.1)
model_path = os.path.join(opt.model_dir,opt.model_name+".pth")
global_steps = 0
best = 0
for epoch in tqdm(list(range(opt.epoch)), desc='epoch'):
for step, batch in enumerate(dataloader):
global_steps += 1
metrics = model.train(batch)
if global_steps % opt.log_steps == 0:
logger.debug(f"global steps={global_steps},{metrics}")
if global_steps % opt.save_steps == 0:
val_metrics, eval_result = eval(opt, model, eval_data)
logger.info(f"global steps={global_steps}, current={val_metrics}, best={best}, result={eval_result}")
if val_metrics > best:
best = val_metrics
torch.save(model.state_dict(), model_path)
logger.info(f"global steps={global_steps}, save model:{model_path}")
lr_scheduler.step()
if __name__ == "__main__":
parser = get_arguments()
opt = parser.parse_args()
train_set, dev_set, extinfo = get_train_data(opt)
with open(os.path.join(opt.model_dir, opt.model_name + ".conf"), 'w') as f:
print(extinfo)
f.write(json.dumps(extinfo,ensure_ascii=False))
train(opt, train_set, dev_set)
base_dir = '.'
name = 'Unsupervised learning via TIS_mask_5t_multi-coil_2149_train_on_random_4X_DC_CNN_AMAX'
# name = os.path.splitext(os.path.basename(__file__))[0]
# model_save_path = os.path.join(base_dir, name)
# if not os.path.isdir(model_save_path):
# os.makedirs(model_save_path)
checkpoint_dir = os.path.join(base_dir, 'checkpoints/%s' % name)
if not os.path.exists(checkpoint_dir):
os.makedirs(checkpoint_dir)
checkpoint_path = os.path.join(checkpoint_dir, '{}.ckpt'.format(name))
# data for train
data_dir = '/data0/ziwen/data/h5/multi_coil_strategy_v0'
train_label, validate_label = get_train_data(data_dir)
#train_label = np.zeros([100, 192, 192, 20])
#validate_label = np.zeros([100, 192, 192, 20])
mask_t = sio.loadmat(join('mask', 'random_gauss_mask_t_192_192_16_ACS_16_R_3.64.mat'))['mask_t']
mask_t = np.fft.fftshift(mask_t, axes=(0, 1))
print('Acceleration factor: {}'.format(mask_t.size/float(mask_t.sum())))
# mk = sio.loadmat('Random1D_256_256_R6.mat')
# mask_t = np.fft.fftshift(mk['mask'], axes=(-1, -2))
y_m = tf.compat.v1.placeholder(tf.complex64, (None, Nx, Ny, Nc), "y_m")
mask = tf.compat.v1.placeholder(tf.complex64, (None, Nx, Ny), "mask")
x_true = tf.compat.v1.placeholder(tf.float32, (None, Nx, Ny), "x_true")
x_pred = getCoilCombineImage_DCCNN(y_m, mask, n_iter=8)
import util
import dataset
import numpy as np
from PIL import Image
from sklearn import svm
win_stride = (8, 8)
padding = (8, 8)
locations = ((10, 20), )
hog = util.histogram_of_gradient()
hog_params = (win_stride, padding, locations)
train_data = dataset.get_train_data()
features, labels = dataset.get_imgs_feature(train_data, hog, hog_params)
linear_svm_model = svm.LinearSVC()
linear_svm_model.fit(features, labels)
test_data = dataset.get_test_data()
features, labels = dataset.get_imgs_feature(test_data, hog, hog_params)
results = linear_svm_model.predict(features)
util.precision_and_recall_evaluate(results, labels)
pos, neg = dataset.get_predict_data()
for i in range(5):
rects = util.detect_person(pos[i], hog, hog_params, linear_svm_model)
rects = util.non_max_suppression_fast(np.array(rects), 0.5)
print(pos[i])