def merge_by_year(year):
'''
Merge the countries and income data sets for any given year.
'''
income = load_data('income')
countries = load_data('countries')
income_yr = pd.DataFrame(income.ix[year])
col_name = ['Country', 'Region', 'Income']
newdf = pd.merge(countries, income_yr, left_on='Country', right_index=True)
newdf.columns = col_name
return newdf
def merge_by_year(year):
'''
Merge the countries and income data sets for any given year.
'''
income = load_data('income')
countries = load_data('countries')
income_yr = pd.DataFrame(income.ix[year])
col_name = ['Country','Region','Income']
newdf = pd.merge(countries,income_yr,left_on = 'Country',right_index =True)
newdf.columns = col_name
return newdf
def test(args):
with fluid.dygraph.guard(place):
model = getattr(models, config.model_name)()
model_dict, _ = fluid.load_dygraph(config.model_name + '_best')
model.load_dict(model_dict)
model.eval()
test_loader = load_data('eval')
data_loader = fluid.io.DataLoader.from_generator(capacity=5,
return_list=True)
data_loader.set_batch_generator(test_loader, places=place)
acc_set = []
avg_loss_set = []
for batch_id, data in enumerate(data_loader):
x_data, y_data = data
img = fluid.dygraph.to_variable(x_data)
label = fluid.dygraph.to_variable(y_data)
prediction, acc = model(img, label)
loss = fluid.layers.cross_entropy(input=prediction, label=label)
avg_loss = fluid.layers.mean(loss)
acc_set.append(float(acc.numpy()))
avg_loss_set.append(float(avg_loss.numpy()))
#计算多个batch的平均损失和准确率
acc_val_mean = np.array(acc_set).mean()
avg_loss_val_mean = np.array(avg_loss_set).mean()
print('loss={}, acc={}'.format(avg_loss_val_mean, acc_val_mean))
def val(model):
with fluid.dygraph.guard(place):
model.eval()
val_loader = load_data('valid')
data_loader = fluid.io.DataLoader.from_generator(capacity=5,
return_list=True)
data_loader.set_batch_generator(val_loader, places=place)
acc_set = []
avg_loss_set = []
for batch_id, data in enumerate(data_loader):
x_data, y_data = data
img = fluid.dygraph.to_variable(x_data)
label = fluid.dygraph.to_variable(y_data)
prediction, acc = model(img, label)
loss = fluid.layers.cross_entropy(input=prediction, label=label)
avg_loss = fluid.layers.mean(loss)
acc_set.append(float(acc.numpy()))
avg_loss_set.append(float(avg_loss.numpy()))
#计算多个batch的平均损失和准确率
acc_val_mean = np.array(acc_set).mean()
avg_loss_val_mean = np.array(avg_loss_set).mean()
print('loss={}, acc={}'.format(avg_loss_val_mean, acc_val_mean))
return acc_val_mean
def main(args=None):
parser = argparse.ArgumentParser(
description='combine adversarial learning for ISIC-2017 Segmentation')
parser.add_argument('--data_dir', default=str, help='Path to Dataset file')
parser.add_argument('--model_dir', default=str, help='Path to save model')
parser.add_argument('--tb_dir',
default=str,
help='Path to save TensorBoard log')
parser.add_argument('--epochs',
help='Number of epochs',
type=int,
default=10000)
parser.add_argument('--batch_size',
help='Number of BS',
type=int,
default=10)
parser = parser.parse_args(args)
x_train, y_train, x_valid, y_valid, x_test, y_test = load_data()
# print (x_train.shape)
# print (y_train.shape)
Net = Unet_combine_adver()
Net.set_up_unet(parser.batch_size, IMAGE_SIZE, OUTPUT_SIZE, CLASS_NUM)
Net.train(x_train, y_train, x_valid, y_valid, x_test, y_test,
parser.batch_size, parser.model_dir, parser.tb_dir,
parser.epochs)
def main():
config = Config()
print('Prepare data for train and dev ... ')
train_dev_split(config.original_file, config.train_file, config.dev_file,
config.vocab_file, config.split_ratio)
print('Prepare data sucessfully!')
lstm_config = Config()
rnn_model = LSTM_Dynamic(lstm_config)
gpu_options = tf.GPUOptions(allow_growth=True)
gpu_options =tf.GPUOptions(per_process_gpu_memory_fraction=0.5, allow_growth=True) ##每个gpu占用0.8 的显存
config=tf.ConfigProto(gpu_options=gpu_options,allow_soft_placement=True)
with tf.Session(config=config) as sess:
if lstm_config.if_train:
init=tf.global_variables_initializer()
sess.run(init)
(X_train, y_train) = load_data(lstm_config.train_file)
if len(X_train) < lstm_config.batch_size:
for i in range(0, lstm_config.batch_size - len(X_train)):
X_train.append([0])
y_train.append([0])
seq_len_train = list(map(lambda x: len(x), X_train))
rnn_model.train_epoch(sess, lstm_config.train_file, X_train,
y_train, seq_len_train,
lstm_config.model_path)
print('Success for preparing data')
def train_by_columns(data_file, voc_file, model_dir, glove_dir, epoch,
batch_size, train_split, eval_split, embedding_dim):
print('training: data = {}, voc = {}, '
'epochs = {}, batch size = {}, '
'glove dir = {}, embedding dim = {} ---> models = {}'.format(
data_file, voc_file, epoch, batch_size, glove_dir, embedding_dim,
model_dir))
train_x, train_y, test_x, test_y = data_process.load_data(
data_file, train_split)
vocab = data_process.load_vocab(voc_file)
max_seq = train_x.shape[1]
vocab_size = len(vocab) + 1
print('training: sequence length = {}, vocabulary size = {}'.format(
max_seq, vocab_size))
embedding_layers = data_process.load_embeddings(vocab, max_seq, glove_dir,
embedding_dim)
for i in range(0, 4):
model_name = 'model_{}'.format(i)
model = train_category(model_name, train_x, train_y.iloc[:, i], test_x,
test_y.iloc[:, i], embedding_layers, eval_split,
epoch, batch_size)
model.save(
os.path.join(
model_dir,
MODEL_FILE_NAME_TEMPLATE.format(max_seq, epoch, embedding_dim,
i)))
def main():
config = Config()
print('Prepare data for train and dev ... ')
train_dev_split(config.original_file, config.train_file, config.dev_file,
config.vocab_file, config.split_ratio)
print('Prepare data sucessfully!')
lstm_config = Config()
blstm_model = LSTM_Dynamic(lstm_config)
with tf.Session(config=tf.ConfigProto(allow_soft_placement=True)) as sess:
if lstm_config.if_test:
print('Start load dev data ...')
(X_test, y_test) = load_data(lstm_config.dev_file)
print('Loading sucess!')
if len(X_test) < lstm_config.batch_size:
for i in range(0, lstm_config.batch_size - len(X_test)):
X_test.append([0])
y_test.append([0])
seq_len_test = list(map(lambda x: len(x), X_test))
print('Target to special model to test')
test_model = os.path.join(lstm_config.model_path, "models_epoch38")
print('Start do predicting...')
blstm_model.test(sess, test_model, X_test, y_test, seq_len_test,
lstm_config.vocab_file,
lstm_config.model_path + 'result/')
def main(self):
inputs, labels, t2i, i2t, max_len, embedding_matrix = data.load_data()
vocab_size = len(t2i)
params = make_params(max_len, vocab_size, embedding_matrix)
estimator = tf.estimator.Estimator(model_fn=model.model_fn,
model_dir=DEFINES.check_point,
params=params)
estimator.train(lambda: data.train_input_fn(inputs, labels))
def plot_income(year):
fig, ax = plt.subplots(figsize=(10,10))
income = load_data('income')
income.ix[year].dropna().hist()
ax.set_title('Histogram of income for year '+str(year))
ax.set_xlabel('Income per person')
ax.set_ylabel('Count')
fig.savefig('Histogram of income for year '+str(year)+'.pdf')
plt.show()
plt.close()
def plot_income(year):
fig, ax = plt.subplots(figsize=(10, 10))
income = load_data('income')
income.ix[year].dropna().hist()
ax.set_title('Histogram of income for year ' + str(year))
ax.set_xlabel('Income per person')
ax.set_ylabel('Count')
fig.savefig('Histogram of income for year ' + str(year) + '.pdf')
plt.show()
plt.close()
def load_data(self):
"""
data
:return:
"""
x, y = data_process.load_data(filename=r'mulclass_clean.csv')
x = self.train_comm_vocab(x)
data_train, data_valid = self.split_data(x, y)
self.train_args['num_classes'] = len(y[0])
return data_train, data_valid
def main(test_model):
config = Config()
config.batch_size = 1024
config.hidden_size = 64
config.vocab_size = 26
config.embed_size = 320
config.max_epochs = 100
config.label_kinds = 2
config.if_train = True
config.if_test = True
config.is_biLSTM = True
config.max_seqlen = 20
config.original_file = '../data/most_frequent_words_label.txt'
config.train_file = '../data/most_frequent_words_label_train.txt'
config.dev_file = '../data/most_frequent_words_label_dev'
config.vocab_file = '../data/vocab.txt'
config.model_path = 'models/Transformer_softmax_lstm/'
config.split_ratio = 0.8
print('Prepare data for train and dev ... ')
train_dev_split(config.original_file, config.train_file, config.dev_file,
config.vocab_file, config.split_ratio)
print('Prepare data sucessfully!')
model = Transformer(config)
gpu_options = tf.GPUOptions(allow_growth=True)
gpu_options = tf.GPUOptions(
per_process_gpu_memory_fraction=0.5, allow_growth=True
) ##每个gpu占用0.8 的显存
tf_config = tf.ConfigProto(gpu_options=gpu_options,
allow_soft_placement=True)
with tf.Session(config=tf_config) as sess:
if config.if_test:
init = tf.global_variables_initializer()
sess.run(init)
(X_test, y_test) = load_data(config.dev_file)
if len(X_test) < config.batch_size:
for i in range(0, config.batch_size - len(X_test)):
X_test.append([0])
y_test.append([0])
seq_len_test = list(map(lambda x: len(x), X_test))
print('Target to special model to test')
print('Start do predicting...')
model.test(sess, test_model, X_test, y_test, seq_len_test,
config.vocab_file, config.model_path + 'result/')
print('Success for preparing data')
def train(args):
print('Now startingt training.......')
with fluid.dygraph.guard(place):
model = getattr(models, config.model_name)()
train_loader = load_data('train', config.batch_size)
data_loader = fluid.io.DataLoader.from_generator(capacity=5,
return_list=True)
data_loader.set_batch_generator(train_loader, places=place)
# train_loader = paddle.batch(paddle.dataset.mnist.train(), batch_size=config.batch_size)
# optimizer = fluid.optimizer.Adam(learning_rate=config.lr)
optimizer = fluid.optimizer.Adam(
learning_rate=fluid.layers.piecewise_decay(
boundaries=[15630, 31260], values=[1e-3, 1e-4, 1e-5]),
regularization=fluid.regularizer.L2Decay(
regularization_coeff=1e-4))
EPOCH_NUM = config.max_epoch
best_acc = -1
for epoch_id in range(EPOCH_NUM):
model.train()
for batch_id, data in enumerate(data_loader):
# image_data = np.array([x[0] for x in data]).astype('float32').reshape(-1, 28, 28)
# label_data = np.array([x[1] for x in data]).astype('int64').reshape(-1, 1)
# image_data = np.expand_dims(image_data, axis=1)
image_data, label_data = data
# print("data shape => ", image_data.shape)
# print("label shape => ", label_data.shape)
image = fluid.dygraph.to_variable(image_data)
label = fluid.dygraph.to_variable(label_data)
predict, avg_acc = model(image, label)
loss = fluid.layers.cross_entropy(predict, label)
# print(loss)
avg_loss = fluid.layers.mean(loss)
if batch_id != 0 and batch_id % 200 == 0:
print(
"epoch: {}, batch: {}, loss is: {}, acc is {}".format(
epoch_id, batch_id, avg_loss.numpy(),
avg_acc.numpy()))
avg_loss.backward()
optimizer.minimize(avg_loss)
model.clear_gradients()
fluid.save_dygraph(model.state_dict(),
config.model_name + '_current')
val_acc = val(model)
if val_acc > best_acc:
fluid.save_dygraph(model.state_dict(),
config.model_name + '_best')
best_acc = max(val_acc, best_acc)
def main():
# 指定运行的GPU
os.environ['CUDA_VISIBLE_DEVICES'] = '0'
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
args = parse_args()
# 输出参数
print "args: ", args
load_data_start_time = time.time()
# 载入数据集
train, valid, test = data_process.load_data()
print("Loading data done. (%0.3f s)" %
(time.time() - load_data_start_time))
print("%d train examples." % len(train[0]))
print("%d valid examples." % len(valid[0]))
# 数据集统计信息
print("%d test examples." % len(test[0]))
keep_probability = np.array(args.keep_probability)
no_dropout = np.array(args.no_dropout)
result_path = "./save/" + args.dataset
# Build model
with tf.Session(config=config) as sess:
# 建立模型
model = CSRM(sess=sess,
n_items=args.n_items,
dim_proj=args.dim_proj,
hidden_units=args.hidden_units,
patience=args.patience,
memory_size=args.memory_size,
memory_dim=args.memory_dim,
shift_range=args.shift_range,
controller_layer_numbers=args.controller_layer_numbers,
batch_size=args.batch_size,
epoch=args.epoch,
lr=args.lr,
keep_probability=keep_probability,
no_dropout=no_dropout,
display_frequency=args.display_frequency)
# 训练
model.train(train, valid, test, result_path)
def train():
# Load data
logger.info("✔︎ Loading data...")
x_train, y_train = dp.load_data(FLAGS.training_data_file)
logger.info("✔︎ Finish building BOW.")
model = SVR()
logger.info("✔︎ Training model...")
model.fit(x_train, y_train)
logger.info("✔︎ Finish training. Saving model...")
joblib.dump(model, FLAGS.model_file)
def train(args):
src_lang, tar_lang, pairs = load_data()
vectors = text2tensor(src_lang, tar_lang, pairs)
generator = gen_batch_data(vectors, args.batch_size, args.src_max_len,
args.tar_max_len)
logging.info('=' * 10 + '模型初始化')
encoder = Encoder(batch_size=args.batch_size,
seq_len=args.src_max_len,
embedding_size=args.embedding_size,
hidden_size=args.hidden_dim,
num_layers=args.num_layers,
src_vocab_size=src_lang.n_words)
decoder = Decoder(batch_size=args.batch_size,
embedding_size=args.embedding_size,
hidden_size=args.hidden_dim,
num_layers=args.num_layers,
tar_vocab_size=tar_lang.n_words)
criterion = nn.CrossEntropyLoss()
model = Seq2Seq(encoder=encoder, decoder=decoder,
criterion=criterion).to(device)
opt = torch.optim.Adam(model.parameters(), lr=args.lr)
logging.info('=' * 10 + 'Starting training Model...')
for iter in range(1, args.iters + 1):
for i, batch_data in enumerate(generator):
src_data = torch.tensor(batch_data[:, 0].tolist()).to(device)
tar_data = torch.tensor(batch_data[:, 1].tolist()).to(device)
#TODO 模型的损失一直都是0，效果不好，增加注意力机制试一试
loss, outputs = model(src_data, tar_data)
logging.info('=' * 10 + "【当前迭代%d，批次%d】, 当前损失:%.4f" %
(iter, i + 1, loss))
for param_group in opt.param_groups:
if iter <= 2:
param_group['lr'] = 0.1
else:
param_group['lr'] = args.lr
opt.zero_grad()
loss.backward()
opt.step()
torch.save(
{
'model_state_dict': model.state_dict(),
'src_lang': src_lang,
'tar_lang': tar_lang
}, args.save_path)
def main(self):
inputs, labels, t2i, i2t, max_len = data.load_data(DEFINES.data_path)
encoder_inputs = inputs
decoder_inputs, decoder_targets = prepare_dec(labels)
embedding_matrix = data.get_embedding_matrix(DEFINES.data_path,
DEFINES.embedding_path, i2t)
params = make_params(embedding_matrix, max_len)
estimator = tf.estimator.Estimator(model_fn=model.model_fn,
model_dir=DEFINES.check_point,
params=params)
estimator.train(lambda: data.train_input_fn(encoder_inputs, decoder_inputs,
decoder_targets))
def main():
# Find the file from the rigth folder
script_dir = os.path.dirname(__file__) #<-- absolute dir the script is in
rel_path = "data/winequality-white.csv"
abs_file_path = os.path.join(script_dir, rel_path)
file_name = abs_file_path
# Get the pandas dict-object
data = data_process.load_data(file_name)
# Visualize the table
print(data)
# Is there linear correlation?
# -1 and 1 high linear correlation, 0 no linear correlation
#cor = data.corr()
#sns.heatmap(cor)
# Train Lasso regression to identify unnecessary features for linear regression
#feature_selection.lasso_r(data)
#-----------------------------------------------------------------------------------------------------------------------------------------------------------
# Beneath all functions to create different Regression and Classification models
# Train linear regression with cross validation
#linear_reg.train_linear(data)
# Train polynomial regression
#polynomial_reg.train_poly(data, 2)
# Train logical Regression
#Logical_reg.train_logical(data)
#Train k-mean classifier
#k_means.class_kmeans(data)
#Train RandomForest Classifier
#r_forest.rforest(data)
#Train RandomForest Classifier
#n_bayes.nbayes(data)
#Train RandomForest Classifier
#SVM.SVM_train(data)
# Show plots
plt.show()
def train():
""" Training
"""
##
# ARGUMENTS
opt = Options().parse()
##
# LOAD DATA
dataloader = load_data(opt)
##
# LOAD MODEL
model = Ganomaly(opt, dataloader)
##
# TRAIN MODEL
model.train()
model.test2()
def test():
logger.info("✔︎ Loading data...")
x_test, y_test = dp.load_data(FLAGS.test_data_file)
logger.info("✔︎ Loading model...")
model = joblib.load(FLAGS.model_file)
logger.info("✔︎ Predicting...")
y_pred = model.predict(x_test)
logger.info("✔︎ Calculate Metrics...")
pcc, doa = dp.evaluation(y_test, y_pred)
rmse = mean_squared_error(y_test, y_pred) ** 0.5
logger.info("☛ Logistic: PCC {0:g}, DOA {1:g}, RSME {2:g}".format(pcc, doa, rmse))
logger.info("✔︎ Done.")
def test(filepath):
K = 5 # 5-folds cross-validation
cv_rep = 10
eps = 1.0
X, y = data_process.load_data(filepath, minmax=(-1, 1), bias_term=True)
rkf = RepeatedKFold(n_splits=K, n_repeats=cv_rep)
errSum = 0.
for train_index, test_index in rkf.split(X):
train_X, train_y = X[train_index, :], y[train_index]
test_X, test_y = X[test_index, :-1], y[test_index]
w, b = fm_logistic.fm_logistic(train_X, train_y, eps)
errorRate = evaluate.rightNum(test_X, test_y, w, b) / len(test_y)
errSum += errorRate
print(errSum / (K * cv_rep))
def load_data(data_source):
assert data_source in ["keras_data_set",
"local_dir"], "Unknown data source"
if data_source == "keras_data_set":
(x_train, y_train), (x_test,
y_test) = imdb.load_data(num_words=max_words,
start_char=None,
oov_char=None,
index_from=None)
x_train = sequence.pad_sequences(x_train,
maxlen=sequence_length,
padding="post",
truncating="post")
x_test = sequence.pad_sequences(x_test,
maxlen=sequence_length,
padding="post",
truncating="post")
vocabulary = imdb.get_word_index()
vocabulary_inv = dict((v, k) for k, v in vocabulary.items())
vocabulary_inv[0] = "<PAD/>"
else:
x, y, vocabulary, vocabulary_inv_list = data_process.load_data()
vocabulary_inv = {
key: value
for key, value in enumerate(vocabulary_inv_list)
}
y = y.argmax(axis=1)
# Shuffle data
shuffle_indices = np.random.permutation(np.arange(len(y)))
x = x[shuffle_indices]
y = y[shuffle_indices]
train_len = int(len(x) * 0.9)
x_train = x[:train_len]
y_train = y[:train_len]
x_test = x[train_len:]
y_test = y[train_len:]
return x_train, y_train, x_test, y_test, vocabulary_inv
def main(self):
arg_length = len(sys.argv)
if (arg_length < 2):
raise Exception("You should put one sentences to predict")
inputs = []
for i in sys.argv[1:]:
inputs.append(i)
inputs = " ".join(inputs)
_, _, t2i, i2t, max_len = data.load_data(DEFINES.data_path)
encoder_inputs, decoder_inputs = prepare_pred_input(inputs, t2i, max_len)
print(encoder_inputs, decoder_inputs)
embedding_matrix = data.get_embedding_matrix(DEFINES.data_path,
DEFINES.embedding_path, i2t)
params = make_params(embedding_matrix, max_len)
estimator = tf.estimator.Estimator(model_fn=model.model_fn,
model_dir=DEFINES.check_point,
params=params)
predict_input_fn = tf.estimator.inputs.numpy_input_fn(x={
"encoer_inputs":
encoder_inputs,
"decoder_inputs":
decoder_inputs
},
num_epochs=1,
shuffle=False)
predict = estimator.predict(input_fn=predict_input_fn)
prediction = next(predict)['prediction']
print(inputs)
print(data.token2str(prediction, i2t))
def test_voting(args):
with fluid.dygraph.guard(place):
model1 = ResNet50()
model2 = ano_model()
model_dict1, _ = fluid.load_dygraph('ResNet50' + '_best')
model_dict2, _ = fluid.load_dygraph('ano_model' + '_best')
model1.load_dict(model_dict1)
model2.load_dict(model_dict2)
model1.eval()
model2.eval()
test_loader = load_data('eval')
data_loader = fluid.io.DataLoader.from_generator(capacity=5,
return_list=True)
data_loader.set_batch_generator(test_loader, places=place)
acc_set = []
avg_loss_set = []
for batch_id, data in enumerate(data_loader):
x_data, y_data = data
img = fluid.dygraph.to_variable(x_data)
label = fluid.dygraph.to_variable(y_data)
out1 = model1(img) # [b, 10]
out2 = model2(img)
out = out1 + out2
out = fluid.layers.softmax(input=out)
acc = fluid.layers.accuracy(input=out, label=label)
loss = fluid.layers.cross_entropy(input=out, label=label)
avg_loss = fluid.layers.mean(loss)
acc_set.append(float(acc.numpy()))
avg_loss_set.append(float(avg_loss.numpy()))
#计算多个batch的平均损失和准确率
acc_val_mean = np.array(acc_set).mean()
avg_loss_val_mean = np.array(avg_loss_set).mean()
print('loss={}, acc={}'.format(avg_loss_val_mean, acc_val_mean))
train_X = data[:-1]
train_Y = data[1:]
train_X = np.array(train_X, dtype=np.int32)
train_Y = np.array(train_Y, dtype=np.int32)
predict_X = train_Y[-1]
return train_X, train_Y, predict_X
# import data_process
# query, response = data_process.loadData()
# train_query, train_query_response, train_response, q_max_len, r_max_len, index = data_process.pad_SentencesQR(query, response)
print("load dic...")
word_id, id_word, index = data_process.load_dic()
# word_id, id_word, index = data_process.load_dic("/home/robot/cy_work/xiaomi_dict")
print("load data ...")
query, response, q_max_len, r_max_len = data_process.load_data()
# query, response, q_max_len, r_max_len = data_process.load_data(file="/home/robot/cy_work/xiaomi_sent_2_sent")
train_query, train_query_response, train_response, q_max_len, r_max_len, index = data_process.pad_sentences_qr(query,
response,
q_max_len,
r_max_len,
index)
print("finished load data!!!")
# data = loadData()
# train_X, train_Y, predict_X = get_train_X_Y(data)
# train_XY = np.append(train_X, train_Y, axis=1)
print("input max length:{}, output max length:{}".format(q_max_len, r_max_len))
#
sp = SequencePattern(name=None, in_seq_len=q_max_len, out_seq_len=r_max_len, embding_size=index)
def train():
print("Loading data...")
train_data, val_data, test_data = load_data(FLAGS.training,
FLAGS.validation,
FLAGS.testing)
print("Train {}".format(np.array(train_data).shape))
print("Initializing...")
with tf.Graph().as_default():
session_conf = tf.ConfigProto(
allow_soft_placement=FLAGS.allow_soft_placement,
log_device_placement=FLAGS.log_device_placement)
sess = tf.Session(config=session_conf)
with sess.as_default():
cnn = Trash_CNN(
num_classes=FLAGS.num_classes,
input_shape=(FLAGS.input_size, FLAGS.input_size, 3),
filters=list(map(int, FLAGS.filter_sizes.split(","))),
input_channel=FLAGS.num_filters)
global_step = tf.Variable(0, name="global_step", trainable=False)
optimizer = tf.train.AdamOptimizer(1e-3)
grads_and_vars = optimizer.compute_gradients(cnn.loss)
train_op = optimizer.apply_gradients(grads_and_vars)
timestamp = str(int(time.time()))
outdir = os.path.abspath(
os.path.join(os.path.curdir, "runs", timestamp))
print("Writing to {}\n".format(outdir))
loss_summary = tf.summary.scalar("loss", cnn.loss)
acc_summary = tf.summary.scalar("acc", cnn.accuracy)
# Train Summaries
train_summary_op = tf.summary.merge([loss_summary, acc_summary])
train_summary_dir = os.path.join(outdir, "summaries", "train")
train_summary_writer = tf.summary.FileWriter(
train_summary_dir, sess.graph)
# Dev summaries
dev_summary_op = tf.summary.merge([loss_summary, acc_summary])
dev_summary_dir = os.path.join(outdir, "summaries", "dev")
dev_summary_writer = tf.summary.FileWriter(dev_summary_dir,
sess.graph)
# Checkpoint directory. Tensorflow assumes this directory already exists so we need to create it
checkpoint_dir = os.path.abspath(
os.path.join(outdir, "checkpoints"))
checkpoint_prefix = os.path.join(checkpoint_dir, "model")
if not os.path.exists(checkpoint_dir):
os.makedirs(checkpoint_dir)
saver = tf.train.Saver(tf.global_variables(),
max_to_keep=FLAGS.num_checkpoints)
sess.run(tf.global_variables_initializer())
def train_step(batch_x, batch_y):
'''
One single training step
'''
feed_dict = {cnn.input_x: batch_x, cnn.input_y: batch_y}
_, step, summaries, loss, accuracy = sess.run(
[
train_op, global_step, train_summary_op, cnn.loss,
cnn.accuracy
],
feed_dict=feed_dict)
time_str = datetime.datetime.now().isoformat()
print("{}: step {}, loss {:g}, acc: {:g}".format(
time_str, step, loss, accuracy))
train_summary_writer.add_summary(summaries, step)
def dev_step(batch_x, batch_y, writer=None):
'''
Evaluate model
'''
feed_dict = {cnn.input_x: batch_x, cnn.input_y: batch_y}
step, summaries, loss, accuracy = sess.run(
[global_step, train_summary_op, cnn.loss, cnn.accuracy],
feed_dict=feed_dict)
time_str = datetime.datetime.now().isoformat()
print("-------------- Step summary ---------------")
print("{}: step {}, loss {:g}, acc: {:g}".format(
time_str, step, loss, accuracy))
if writer:
writer.add_summary(summaries, step)
batches = batch_iter(list(zip(train_data[0], train_data[1])),
FLAGS.batch_size, FLAGS.num_epochs)
for batch in batches:
print("Batch {}".format(batch.shape))
x_batch, y_batch = zip(*batch)
train_step(x_batch, y_batch)
current_step = tf.train.global_step(sess, global_step)
if current_step % FLAGS.evaluate_every == 0:
print(
"\n======================= Evaluation: ======================="
)
dev_step(val_data[0],
val_data[1],
writer=dev_summary_writer)
print("")
if current_step % FLAGS.checkpoint_every == 0:
path = saver.save(sess,
checkpoint_prefix,
global_step=current_step)
print("Saved model checkpoint to {}\n".format(path))
import data_process
from data_process import MyDataSet
import numpy as np
from baseline import SVR_baseline, evaluate, HA_baseline
from MyLSTM import MyLSTM
from tqdm import tqdm
from torch.autograd import Variable
from torch.optim import lr_scheduler
import torch.nn as nn
import torch
import torch.utils.data as Data
adj, flow = data_process.load_data()
flow = np.mat(flow, dtype=np.float32)
# 读入邻接矩阵
adj_norm = data_process.cal_adj_norm(adj)
x_train, y_train, x_test, y_test = data_process.train_test_spilt(
flow, 4, 20, 0.3)
y_pred = HA_baseline(x_test, y_test)
evaluate(y_test, y_pred)
# dataset
BATCH_SIZE = 32
train_dataset = MyDataSet(x_train, y_train, type='train')
test_dataset = MyDataSet(x_test, y_test, type='test')
train_loader = Data.DataLoader(dataset=train_dataset,
batch_size=BATCH_SIZE,
shuffle=False)
test_loader = Data.DataLoader(dataset=test_dataset,
batch_size=BATCH_SIZE,
if __name__ == '__main__':
parser = argparse.ArgumentParser(description='Process train parameters')
parser.add_argument('data_directory', metavar='d', type=str, nargs='+', help='data directory')
parser.add_argument('--save_dir', type=str, nargs='?', help='save directory')
parser.add_argument('--learning_rate', type=float, nargs='?', help='learning rate')
parser.add_argument('--hidden_units', type=int, nargs='+', help='hidden units')
parser.add_argument('--epochs', type=int, nargs='?', help='epoch numbers')
parser.add_argument('--arch', type=str, nargs='?', help='model select')
parser.add_argument('--gpu', action='store_true', help='use gpu')
args = parser.parse_args()
## 数据路径
data_directory = os.path.abspath(args.data_directory[0])
## 加载数据
class_to_idx, dataloaders = data_process.load_data(data_directory)
save_directory = "."
if args.save_dir:
save_directory = args.save_dir
if not os.path.isabs(save_directory):
save_directory = os.path.abspath(save_directory)
if not os.path.exists(save_directory):
raise Exception("Save directory not exist")
## 模型名
model_name = 'vgg16'
if args.arch:
model_name = args.arch
## 创建分类器
input_size = 25088
default='onehot',
help='the feature type for the atoms in modulars')
parser.add_argument(
'--train_type',
type=str,
default='se',
help=
'training type of the model, each batch contains fixed edges or a side effect graph'
)
args = parser.parse_args()
# print the model parameters.
print(args)
# output contains the modular data like [node_attribute, edge_index, edge_weight, batch]
output, edges, edges_attr, se_name = load_data(args.modular_file,
args.ddi_file, 'onehot')
print(len(list(output.keys())))
args.num_edge_features = edges_attr.size(1)
args.device = 'cpu'
# split data into train val test.
num_edges = edges_attr.size(0) // 2
train_num = int(num_edges * args.train_ratio)
val_num = int(num_edges * args.val_ratio)
test_num = int(num_edges * args.test_ratio)
nums = [train_num, val_num, test_num]
# change the input to the the side effect name
train_edges, train_edges_attr, val_edges, val_edges_attr, test_edges, test_edges_attr \
= split_data(edges, se_name, nums)
# print(train_edges_attr)
''' Test module for the program Created on Dec 3, 2016 @author: Zhiqi Guo(zg475) @email: [email protected] ''' import unittest import pandas as pd from graph import merge_by_year from data_process import load_data countries = load_data('countries') income = load_data('income') class Test(unittest.TestCase): ''' Run the test in the project's root directory with the following command: $ python -m unittest discover ''' def test_merge(self): ''' Test merge function Exhaustively for every country at year 2000. And repeat the test 10 times. ''' for count in range(10): year = 2000 income_yr = pd.DataFrame(income.ix[year]).astype(float) merged = merge_by_year(year)
from data_process import load_data
from model import build_model, History
from keras.utils import to_categorical
from keras.preprocessing.image import ImageDataGenerator
v = int(28709 * 0.2)
r = int(28709 * 0.8 * 0.2) + v
if __name__ == '__main__':
data_x, data_y = load_data('train.csv', 'train')
valid_x, valid_y = data_x[:v], data_y[:v]
data1_x, data1_y = data_x[v:r], data_y[v:r]
data2_x, data2_y = data_x[r:], data_y[r:]
datagen = ImageDataGenerator(
#rescale = 1 / 255,
#zca_whitening = True,
rotation_range=3,
width_shift_range=0.1,
height_shift_range=0.1,
zoom_range=0.1,
horizontal_flip=True)
datagen.fit(data1_x)
history1 = History()
model = build_model()
model.compile(loss='categorical_crossentropy',
optimizer='adam',
metrics=['accuracy'])
model.fit_generator(
def main(args):
# 可选择输出日志到文件
# logging.basicConfig(format='%(asctime)s - %(levelname)s: %(message)s',
# filename=args.model_name + '.log',
# filemode='w',
# level=logging.INFO)
logging.basicConfig(format='%(asctime)s - %(levelname)s: %(message)s',
level=logging.INFO)
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
n_gpu = torch.cuda.device_count()
set_seed(args.seed, n_gpu)
logging.info("The active device is: {}, gpu_num: {}".format(device, n_gpu))
args.ckpt_path = os.path.join(args.ckpt_path, args.model_name)
try:
os.makedirs(args.ckpt_path)
except:
pass
extra_token_dict = {"unk_token": "<UNK>", "pad_token": "<PAD>"}
train_dev_sentences, test_sentences, train_dev_labels, test_labels, word2id, id2word, embeddings = \
load_data(args.raw_path, args.embedding_path, args.train_path, args.test_path, args.max_seq_len, extra_token_dict)
f1_list = [0] * 5
if args.do_train:
kf = StratifiedKFold(n_splits=5,
shuffle=True).split(train_dev_sentences,
train_dev_labels)
for cv_i, (train_index, dev_index) in enumerate(kf):
logging.info(
"******************Train CV_{}******************".format(cv_i))
# 准备模型
if args.model_name == "TextRNN":
config = TextRNN_Config(embeddings, args.num_label)
model = TextRNN(config)
if args.model_name == "TextCNN":
config = TextCNN_Config(embeddings, args.num_label)
model = TextCNN(config)
if args.model_name == "TextRCNN":
config = TextRCNN_Config(embeddings, args.max_seq_len,
args.num_label)
model = TextRCNN(config)
if args.model_name == "TextCNN_Highway":
config = TextCNN_Highway_Config(embeddings, args.num_label)
model = TextCNN_Highway(config)
if args.model_name == "TextRNN_Attention":
config = TextRNN_Attention_Config(embeddings, args.num_label)
model = TextRNN_Attention(config)
logging.info("Already load the model: {},".format(args.model_name))
model.to(device)
train_sentences = [train_dev_sentences[i] for i in train_index]
train_labels = [train_dev_labels[i] for i in train_index]
dev_sentences = [train_dev_sentences[i] for i in dev_index]
dev_labels = [train_dev_labels[i] for i in dev_index]
logging.info("Prepare dataloader...")
train_tensor, train_sent_len, train_labels_tensor = convert2feature(
train_sentences, train_labels, word2id, args.max_seq_len)
train_data = TensorDataset(train_tensor, train_sent_len,
train_labels_tensor)
train_sampler = RandomSampler(train_data)
train_dataloader = DataLoader(train_data,
sampler=train_sampler,
batch_size=args.batch_size)
train_dataloader = cycle(train_dataloader)
dev_tensor, dev_sent_len, dev_labels_tensor = convert2feature(
dev_sentences, dev_labels, word2id, args.max_seq_len)
dev_data = TensorDataset(dev_tensor, dev_sent_len,
dev_labels_tensor)
dev_sampler = SequentialSampler(dev_data)
dev_dataloader = DataLoader(dev_data,
sampler=dev_sampler,
batch_size=args.dev_batch_size)
logging.info("Begin to train...")
f1_list[cv_i] = train_eval(train_dataloader, dev_dataloader, model,
args.ckpt_path, args.train_steps,
args.check_step, args.eval_step,
args.lr, args.warmup_steps, cv_i)
if not args.do_cv:
break
if args.do_cv:
cv_f1 = np.mean(np.array(f1_list))
logging.info("CV F1_list: {}, Mean_F1: {:.4f}".format(
f1_list, cv_f1))
if args.do_test:
logging.info("******************Test******************")
logging.info("Begin to test {}...".format(args.model_name))
test_tensor, test_sent_len, test_labels_tensor = convert2feature(
test_sentences, test_labels, word2id, args.max_seq_len)
test_data = TensorDataset(test_tensor, test_sent_len,
test_labels_tensor)
test_sampler = SequentialSampler(test_data)
test_dataloader = DataLoader(test_data,
sampler=test_sampler,
batch_size=args.dev_batch_size)
final_results = np.zeros((len(test_labels), args.num_label))
test_labels = test_labels_tensor.to('cpu').numpy()
for cv_i in range(5):
ckpt_path = os.path.join(args.ckpt_path,
"pytorch_model_{}.pkl".format(cv_i))
if args.model_name == "TextRNN":
config = TextRNN_Config(embeddings, args.num_label)
model = TextRNN(config)
model.load_state_dict(torch.load(ckpt_path))
model.to(device)
if args.model_name == "TextCNN":
config = TextCNN_Config(embeddings, args.num_label)
model = TextCNN(config)
model.load_state_dict(torch.load(ckpt_path))
model.to(device)
if args.model_name == "TextRCNN":
config = TextRCNN_Config(embeddings, args.max_len,
args.num_label)
model = TextRCNN(config)
model.load_state_dict(torch.load(ckpt_path))
model.to(device)
if args.model_name == "TextRNN_Attention":
config = TextRNN_Attention_Config(embeddings, args.num_label)
model = TextRNN_Attention(config)
model.load_state_dict(torch.load(ckpt_path))
model.to(device)
output_labels, test_f1_score = test(test_dataloader, model, device,
args.dev_batch_size)
final_results = final_results + output_labels
logging.info(
"The cv_{} result of {} on test data: F1: {:.4f}".format(
cv_i, args.model_name, test_f1_score))
if not args.do_cv:
break
test_f1_score = round(macro_f1(final_results, test_labels), 4)
logging.info("The final result of {} on test data: F1: {:.4f}".format(
args.model_name, test_f1_score))
def main():
#X_train contains path to image
(X_train, Y_train), (X_test, Y_test) = load_data()
Y_train = to_categorical(Y_train, 3)
Y_test = to_categorical(Y_test, 3)
model, modelName = getModel('naive', IMG_SIZE)
#print out info about the models(layer structures etc)
model.summary()
#Adam is an optimization algorithm that can be used instead of the classical stochastic gradient descent procedure to update network weights iterative based in training data
#Most commonly used, can be changed out by SGD or anything similar
optimizer = Adam(learning_rate=LEARNING_RATE)
#best to use categorial loss for classification problems
loss_fuction = 'categorical_crossentropy'
loss = [loss_fuction]
#Metrics to measure during training, we are only interested in the prediction accuracy for now
metrics = {'prediction': 'accuracy'}
#set the optimizer, loss function, and the metrics to print when training
model.compile(optimizer=optimizer, loss=loss, metrics=metrics)
#List of Utilities called at certain points during model training
callbacks = [
LearningRateScheduler(schedular), #schedule the learning rate
ModelCheckpoint(
os.path.join(CHECKPOINT_DIRECTORY,
modelName + '{epoch:02d}-{val_loss:.2f}.hdf5'),
monitor='val_loss', #monitor validation loss
verbose=1, #print fancy progressbar
save_best_only=True, #self explanatory
mode='auto', #the decision to overwrite current save file
save_weights_only=True, #save only the weights, not full model
save_freq='epoch'), #save after every epoch
TensorBoard(log_dir=TENSORBOARD_DIRECTORY,
histogram_freq=0,
write_graph=True,
write_images=True)
]
p = getAugmentorPipeline()
#stack operations...
p.random_brightness(probability=0.25, min_factor=0.75, max_factor=1.25)
p.flip_left_right(probability=0.25)
p.rotate(probability=0.25, max_left_rotation=10, max_right_rotation=10)
p.shear(probability=0.25, max_shear_left=10, max_shear_right=10)
#p.zoom(probability=0.25, min_factor=1.1, max_factor=1.5)
p.random_erasing(probability=0.25, rectangle_area=0.25)
training_generator = TrainingGenerator(
augmentor_pipeline=p,
images=X_train,
labels=Y_train,
batch_size=BATCH_SIZE,
img_size=IMG_SIZE,
normalize=True,
data_aug=True if USE_DATA_AUGMENTATION else False)
validation_generator = TrainingGenerator(augmentor_pipeline=p,
images=X_test,
labels=Y_test,
batch_size=BATCH_SIZE,
img_size=IMG_SIZE,
normalize=True,
data_aug=False)
print('Training model...')
history = model.fit_generator(generator=training_generator,
steps_per_epoch=len(X_train) // BATCH_SIZE,
validation_data=validation_generator,
validation_steps=len(X_test) // BATCH_SIZE,
epochs=EPOCHS,
verbose=1,
callbacks=callbacks,
workers=6,
use_multiprocessing=False,
shuffle=True,
initial_epoch=0,
max_queue_size=6)
#Confusion Matrix and Classification Report
Y_pred = model.predict(x=validation_generator,
steps=len(X_test) // BATCH_SIZE + 1)
y_pred = np.argmax(Y_pred, axis=1)
y_test = np.argmax(Y_test, axis=1)
print('Saving confusion matrix as {}.png'.format(modelName))
#covid 0, normal 1, pneumonia 2
target_names = ['normal', 'corona', 'pnemonia']
c_matrix = confusion_matrix(y_test, y_pred)
df_cm = pd.DataFrame(c_matrix,
index=[i for i in target_names],
columns=[i for i in target_names])
plt.figure(figsize=(10, 10))
plot = sns.heatmap(df_cm, annot=True)
plot.figure.savefig(
os.path.join(CONFUSION_MATRIX_DIRECTIORY, '{}.png'.format(modelName)))
print(f'Saving classification report as {modelName}.txt')
with open(os.path.join(METRICS_DIRECTORY, '{}.txt'.format(modelName)),
'w') as f:
f.write(
classification_report(y_test, y_pred, target_names=target_names))
print('Saving history and model...')
#Save the history
with open(os.path.join(HISTORY_DIRECTORY, modelName + '.h5'), 'wb') as f:
pickle.dump(history.history, f)
#Save the whole model
model.save(os.path.join(MODEL_DIRECTORY, modelName + '.h5'))
""" Test module for the program Created on Dec 3, 2016 @author: Zhiqi Guo(zg475) @email: [email protected] """ import unittest import pandas as pd from graph import merge_by_year from data_process import load_data countries = load_data("countries") income = load_data("income") class Test(unittest.TestCase): """ Run the test in the project's root directory with the following command: $ python -m unittest discover """ def test_merge(self): """ Test merge function Exhaustively for every country at year 2000. And repeat the test 10 times. """ for count in range(10): year = 2000 income_yr = pd.DataFrame(income.ix[year]).astype(float)
