def load_data_dict():
"""
加载参数
:return:
"""
flags_dict = load_data(filename=os.path.join(DATA_DICT_PATH, "config.pkl"))
char_alphabet = load_data(
filename=os.path.join(DATA_DICT_PATH, "char_alphabet.pkl"))
word_alphabet = load_data(
filename=os.path.join(DATA_DICT_PATH, "word_alphabet.pkl"))
label_alphabet = load_data(
filename=os.path.join(DATA_DICT_PATH, "label_alphabet.pkl"))
return flags_dict, word_alphabet, label_alphabet, char_alphabet
def train(Model):
"""Train the model"""
train_set, target, categoricals = utils.load_data(args.dataset,
args.file_name)
dataset_train = Dataset(dataset=train_set,
categorical_indices=categoricals)
target = dataset_train[target]
dataset_train.drop(target, axis=1, inplace=True)
(
categorical_variables,
non_categorical_variables,
) = dataset_train.get_variables_names()
print("Creating cross products dataset")
cross_products = dataset_train.cross_categorical_dataset()
model = Model(categorical_variables, non_categorical_variables)
model = model.build_model(cross_products)
print("Training model")
model.fit(
[
[
train_set[categorical_variables],
train_set[non_categorical_variables]
],
cross_products,
],
target,
epochs=config.EPOCHS,
validation_split=config.VALIDATION_SPLIT,
)
def _load_data(self, filepath):
data = load_data(filepath, label=False)
data = bandpassfilter(data, srate=self.srate)
data = data[:, :,
round(self.beg * self.srate):round(self.end * self.srate),
np.newaxis]
return data
def main():
# preprocessing
prep = PreProcessor(batch_size=100, max_length=15, word_count_threshold=1, cnn_model_path='data/imagenet-vgg-verydeep-19.mat')
for split in ['train', 'dev']:
prep.run(split, cat_img_num=0) # define the number of additional training cat images in the second argument
# load train dataset
train_data = load_data(data_path='./data', split='train')
dev_data = load_data(data_path='./data', split='dev')
word_to_idx = train_data['word_to_idx']
model = CaptionGenerator(word_to_idx, feature_dim=[196, 512], embed_dim=512, hidden_dim=1024, len_sent=16, lamda=1.0)
solver = CaptioningSolver(model, train_data, dev_data, n_epochs=5, batch_size=128, learning_rate=0.001, print_every=1000, save_every=5, \
model_path='model/lstm/', test_model='model/lstm/model-5')
solver.train()
def prepare_data(folder):
print('... loading data')
letter_num = len(APPEARED_LETTERS)
data, label = load_data(folder)
data_train, data_test, label_train, label_test = \
train_test_split(data, label, test_size=0.1, random_state=0)
label_categories_train = to_categorical(label_train, letter_num)
label_categories_test = to_categorical(label_test, letter_num)
return (data_train, label_categories_train, data_test,
label_categories_test)
def run(network='rnn'):
checkpoint_file = checkpoint_dir.format(network)
if not os.path.exists(checkpoint_file):
os.mkdir(checkpoint_file)
train_l_x, val_l_x, train_l_len, val_l_len, train_r_x, val_r_x, train_r_len, val_r_len, train_y, val_y = load_train_data(
)
vocab = load_data(VOCAB_PKL)
vocab_size = len(vocab.word2idx)
batch_size = hp.batch_size
if network == 'rnn':
model = RnnSiameseNetwork(vocab_size, hp.embedding_size, vocab.max_len,
batch_size, True)
elif network == 'match_pyramid':
model = MatchPyramidNetwork(vocab_size, hp.embedding_size,
vocab.max_len, batch_size, True)
elif network == 'cnn':
model = CnnSiameseNetwork(vocab_size, hp.embedding_size, vocab.max_len,
batch_size, True)
elif network == "transformer":
model = TransformerSiameseNetwork(vocab_size, hp.embedding_size,
vocab.max_len, batch_size, True)
else:
return
sv = tf.train.Supervisor(graph=model.graph,
logdir=checkpoint_file,
save_model_secs=150)
with sv.managed_session() as sess:
print("start training...\n")
for epoch in range(1, hp.num_epochs + 1):
if sv.should_stop():
break
train_loss = []
for feed_dict, _ in get_feed_dict(model, train_l_x, train_r_x,
train_l_len, train_r_len,
train_y, batch_size):
loss, _ = sess.run([model.loss, model.train_op],
feed_dict=feed_dict)
train_loss.append(loss)
dev_loss = []
predicts = []
for feed_dict, start in get_feed_dict(model, val_l_x, val_r_x,
val_l_len, val_r_len, val_y,
batch_size):
loss, gs, pre_y = sess.run(
[model.loss, model.global_step, model.pre_y],
feed_dict=feed_dict)
dev_loss.append(loss)
predicts.extend(pre_y[start:])
print_info(epoch, gs, train_loss, dev_loss, val_y, predicts)
def load_test_data(filename):
vocab = load_data(VOCAB_PKL)
max_len = vocab.max_len
data = read_csv(filename)
data = [kk[:3] for kk in data]
idx, left_x, right_x = zip(*data)
left_x = [trim(kk) for kk in left_x]
right_x = [trim(kk) for kk in right_x]
left_x, left_len = pad_sequence(left_x, vocab, max_len)
right_x, right_len = pad_sequence(right_x, vocab, max_len)
return idx, left_x, left_len, right_x, right_len, vocab
def load_train_data():
data = load_data(DATA_PKL)
train_l_x, val_l_x, train_l_len, val_l_len, train_r_x, val_r_x, train_r_len, val_r_len, train_y, val_y = \
data['train_l_x'], data['val_l_x'], data['train_l_len'], data['val_l_len'], data['train_r_x'], data[
'val_r_x'], data['train_r_len'], data['val_r_len'], data['train_y'], data['val_y']
train_l_x = np.array(train_l_x)
val_l_x = np.array(val_l_x)
train_l_len = np.array(train_l_len)
val_l_len = np.array(val_l_len)
train_r_x = np.array(train_r_x)
val_r_x = np.array(val_r_x)
train_r_len = np.array(train_r_len)
val_r_len = np.array(val_r_len)
train_y = np.array(train_y)
val_y = np.array(val_y)
return train_l_x, val_l_x, train_l_len, val_l_len, train_r_x, val_r_x, train_r_len, val_r_len, train_y, val_y
def main():
# load train dataset
print '@@@@@@@@@@@@@'
data = load_data(data_path='./data', split='train')
label_to_idx = data['label_to_idx']
print '@@@@@'
# load val dataset to print out bleu scores every epoch
#val_data = load_data(data_path='./data', split='val')
val_data = []
print label_to_idx
print data['labels']
model = CaptionGenerator(label_to_idx,
dim_feature=[196, 512],
dim_embed=512,
dim_hidden=1024,
n_time_step=30,
prev2out=True,
ctx2out=True,
alpha_c=1.0,
selector=True,
dropout=True)
solver = CaptioningSolver(model,
data,
val_data,
n_epochs=1,
batch_size=1,
update_rule='adam',
learning_rate=0.001,
print_every=3,
save_every=1,
image_path='./image/',
pretrained_model=None,
model_path='model/lstm/',
test_model='model/lstm/model-10',
print_bleu=True,
log_path='log/')
print 'model + solver'
solver.train()
def _load_label(self, filepath):
return load_data(filepath, label=True)