def train_and_evaluate(args):
# Build model
model = model_fn(
first_layer=[args.inp_kernel, args.inp_window],
gcn_layers=[args.n_GC_units] * args.n_GC_layers,
conv_layer_filters=[int(k) for k in args.conv_kernels.split(',')],
conv_layer_windows=[int(k) for k in args.conv_windows.split(',')],
nBins=1250,
lr=args.lr,
nMarks=args.n_marks,
verbose=1
)
# Load chromosomes
st_ed_pos = chromosome_sizes[args.cell_line]
chromosomes = ['chr' + elm for elm in args.chrs.split(',')] if args.chrs.lower() != 'all' else st_ed_pos.keys()
# Load resolutions
resolutions = [int(elm) for elm in args.inp_resolutions.split(',')]
# Load epigenetic data
epi_names = ['ATAC_seq', 'CTCF', 'H3K4me1', 'H3K4me3',
'H3K9ac', 'H3K27ac', 'H3K27me3', 'H3K36me3']
epigenetic_data = load_epigenetic_data(args.cell_line, list(st_ed_pos.keys()), epi_names)
# epigenetic_data = load_processed_epigenetic_data_2(chromosomes, epi_names, args.cell_line)
for i in range(args.epochs):
print('Epoch', i, ':')
t1 = time.time()
for (epi, hics), micros in generate_batches(args.cell_line, chromosomes, resolutions, epi_names,
epigenetic_data, args.batch_size, args.inp_window):
t2 = time.time()
print(' - Loading data:', t2 - t1, 's')
model.train_on_batch([hics, epi], micros)
t3 = time.time()
print(' - Training:', t3 - t2, 's')
mse = model.evaluate([hics, epi], micros, batch_size=args.batch_size, verbose=0)
t1 = time.time()
print(' - Evaluating:', t1 - t3, 's')
print(' - MSE:', mse)
v0 = 0
for (epi, hics), micros in generate_batches(args.cell_line, ['chr2'], resolutions, epi_names,
epigenetic_data, args.batch_size, args.inp_window):
mse = model.evaluate([hics, epi], micros, batch_size=args.batch_size, verbose=0)
print(' - Validation:', v0, mse)
v0 += 1
if i % args.checkpoint_frequency == 0:
model.save_weights('temp_model_{0}.h5'.format(i))
def validate_keras(self, max_count=-1):
print('validating')
valid_total_loss = 0
count = 0
batch_count = 0
n_val_batch_iters = self.num_val_examples // self.batch_size
valid_datagen = utils.generate_batches(filename=self.valid_file,
seqlen=self.seq_len,
vocab=self.vocab,
batch_size=self.batch_size)
for _ in range(n_val_batch_iters):
x_val_batch, y_val_batch = next(valid_datagen)
print('=', end='', flush=True)
val_loss = self.model.test_on_batch(x_val_batch, y_val_batch)
valid_total_loss = valid_total_loss + val_loss
count += len(x_val_batch)
batch_count += 1
if max_count > 0 and count > max_count:
break
print('.')
valid_total_loss /= batch_count
return valid_total_loss
def main(_):
# Get config
config = utils.load_config(FLAGS.config)
config['training']['batch_size'] = 1
# Get checkpoint dir
ckpt_dir = os.path.join(config['training']['ckpt_dir'],
config['experiment_name'])
ckpt = os.path.join(ckpt_dir, 'model.ckpt')
# Build graph
results = []
g = tf.Graph()
with g.as_default():
tf.logging.info('Creating graph')
build_graph(config)
saver = tf.train.Saver()
with tf.Session() as sess:
saver.restore(sess, ckpt)
sess.run([tf.local_variables_initializer()])
for i, batch in enumerate(
utils.generate_batches('test', config, mate=True)):
batch, product = batch
try:
results.append((greedy_table_search(sess, batch,
config), product))
except tf.errors.InvalidArgumentError: # A bad JPEG
continue
if not (i + 1) % 10000:
# Setup output writer
with open(FLAGS.output, 'w') as f:
json.dump(results, f)
break
def run_model(train_data, train_labels, test_data, test_labels):
x = tf.placeholder(tf.float32, [None, 784])
y_ = tf.placeholder(tf.float32, [None, 10])
y_pred, keep_prob = cnn(x)
with tf.name_scope('loss'):
cross_entropy = tf.nn.softmax_cross_entropy_with_logits(labels=y_,
logits=y_pred)
cross_entropy = tf.reduce_mean(cross_entropy)
with tf.name_scope('adam_optimizer'):
train_step = tf.train.AdamOptimizer(1e-4).minimize(cross_entropy)
with tf.name_scope('accuracy'):
correct_prediction = tf.equal(tf.argmax(y_pred, 1), tf.argmax(y_, 1))
correct_prediction = tf.cast(correct_prediction, tf.float32)
accuracy = tf.reduce_mean(correct_prediction)
graph_location = tempfile.mkdtemp()
print('Saving graph to: %s' % graph_location)
train_writer = tf.summary.FileWriter(graph_location)
train_writer.add_graph(tf.get_default_graph())
accuracies = list()
iterations = list()
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
for epoch in range(0, 1):
print('epoch', epoch)
# regenerate batches in each epoch
batched = utils.generate_batches(train_data,
train_labels,
batch_size=50)
i = 0
for batch in batched:
if i % 100 == 0:
train_accuracy = accuracy.eval(feed_dict={
x: batch[0],
y_: batch[1],
keep_prob: 1.0
})
print('step %d, training accuracy %g' %
(i, train_accuracy))
accuracies.append(train_accuracy)
iterations.append(epoch * len(batched) + i)
print(iterations[-1])
i += 1
train_step.run(feed_dict={
x: batch[0],
y_: batch[1],
keep_prob: 0.5
})
print('test accuracy %g' % accuracy.eval(feed_dict={
x: test_data,
y_: test_labels,
keep_prob: 1.0
}))
print(accuracies)
print(iterations)
def train(test_data, test_labels):
x = tf.placeholder(tf.float32, [None, 784])
W = tf.Variable(tf.zeros([784, 10]))
b = tf.Variable(tf.zeros([10]))
y = tf.matmul(x, W) + b
y_ = tf.placeholder(tf.float32, [None, 10])
cross_entropy = tf.reduce_mean(
tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y))
train_step = tf.train.GradientDescentOptimizer(0.5).minimize(cross_entropy)
sess = tf.InteractiveSession()
tf.global_variables_initializer().run()
for epoch in range(0, 10):
print('epoch', epoch)
batched = utils.generate_batches(train_data,
train_labels,
batch_size=100)
i = 0
for batch in batched:
if i % 10 == 0:
print('running batch', i, '...')
sess.run(train_step, feed_dict={x: batch[0], y_: batch[1]})
i += 1
return sess, y, y_, x
def main():
# create the model
model = create_model()
# compile with set config
model.compile(optimizer='adam', loss='categorical_crossentropy')
# train the model
model.fit_generator(generate_batches(train_data, all_tags, word_to_id,
tag_to_id),
len(train_data) / batch_size,
callbacks=[EvaluateAccuracy()],
epochs=5)
# final test accuracy
acc = compute_test_accuracy(model)
print("\nFinal accuracy: %.5f" % acc)
def main(_):
# Get config
config = utils.load_config(FLAGS.config)
config['training']['batch_size'] = 1
# Setup output writer
output_file = open(FLAGS.output, 'w', newline='')
output_writer = csv.writer(output_file, delimiter=',')
# Get checkpoint dir
ckpt_dir = os.path.join(config['training']['ckpt_dir'],
config['experiment_name'])
ckpt = os.path.join(ckpt_dir, 'model.ckpt')
# Build graph
g = tf.Graph()
with g.as_default():
tf.logging.info('Creating graph')
build_graph(config)
saver = tf.train.Saver()
metric_op = tf.get_collection('rank_metrics')
update_op = tf.get_collection('rank_updates')
with tf.Session() as sess:
saver.restore(sess, ckpt)
sess.run([tf.local_variables_initializer()])
for i, batch in enumerate(utils.generate_batches('test', config)):
batch, uris = batch # TODO: Remove
try:
attr_query_id = batch['attr_query_ids:0'][0]
correct_value_id = batch['correct_value_ids:0'][0]
rank, scores = sess.run(['rank:0', 'scores:0'], feed_dict=batch)
output_writer.writerow([uris[0], attr_query_id, correct_value_id,
rank[0], *scores[0]])
except tf.errors.InvalidArgumentError: # A bad JPEG
continue
# Close writer
output_file.close()
def fetch_all(self):
self.server.select('inbox')
status, indices = self.server.search(None, 'ALL')
if status != 'OK':
print("Failed to perform initial search. Abort.")
exit()
indices = indices[0].split()
if self.verbose:
print(f"Found {len(indices)} emails.")
self.total_messages = len(indices)
batches = generate_batches(1, len(indices) - 1, 50)
for batch in batches:
print(f'Downloading emails {batch[0]} through {batch[1]}')
status, response = self.server.fetch(f'{batch[0]}:{batch[1]}',
'(RFC822)')
for item in response:
if len(item) == 2:
decoded = item[1].decode('ISO-8859-1')
self.messages.append(parse_email(decoded))
def main():
args = parser.parse_args()
device = torch.device(
"cuda:" +
str(args.device)) if torch.cuda.is_available() else torch.device("cpu")
Gs, class_labels = load_data(args.dataset, args.use_node_labels)
if args.use_node_labels:
Gs, feat_dim = process_node_labels(Gs)
else:
feat_dim = 1
enc = LabelEncoder()
class_labels = enc.fit_transform(class_labels)
unique_labels = np.unique(class_labels)
n_classes = unique_labels.size
y = [np.array(class_labels[i]) for i in range(class_labels.size)]
kf = KFold(n_splits=10, shuffle=True)
it = 0
accs = list()
for train_index, test_index in kf.split(y):
it += 1
idx = np.random.permutation(train_index)
train_index = idx[:int(idx.size * 0.9)].tolist()
val_index = idx[int(idx.size * 0.9):].tolist()
n_train = len(train_index)
n_val = len(val_index)
n_test = len(test_index)
Gs_train = [Gs[i] for i in train_index]
y_train = [y[i] for i in train_index]
Gs_val = [Gs[i] for i in val_index]
y_val = [y[i] for i in val_index]
Gs_test = [Gs[i] for i in test_index]
y_test = [y[i] for i in test_index]
adj_train, features_train, idx_train, y_train = generate_batches(
Gs_train,
args.use_node_labels,
feat_dim,
y_train,
args.batch_size,
args.radius,
device,
shuffle=True)
adj_val, features_val, idx_val, y_val = generate_batches(
Gs_val, args.use_node_labels, feat_dim, y_val, args.batch_size,
args.radius, device)
adj_test, features_test, idx_test, y_test = generate_batches(
Gs_test, args.use_node_labels, feat_dim, y_test, args.batch_size,
args.radius, device)
n_train_batches = ceil(n_train / args.batch_size)
n_val_batches = ceil(n_val / args.batch_size)
n_test_batches = ceil(n_test / args.batch_size)
model = k_hop_GraphNN(feat_dim, args.hidden_dim, n_classes,
args.dropout, args.radius, device).to(device)
optimizer = optim.Adam(model.parameters(), lr=args.lr)
def train(epoch, adj, features, idx, y):
optimizer.zero_grad()
output = model(adj, features, idx)
loss_train = F.cross_entropy(output, y)
loss_train.backward()
model.clip_grad(5)
optimizer.step()
return output, loss_train
def test(adj, features, idx, y):
output = model(adj, features, idx)
loss_test = F.cross_entropy(output, y)
return output, loss_test
best_val_acc = 0
for epoch in range(args.epochs):
start = time.time()
model.train()
train_loss = AverageMeter()
train_acc = AverageMeter()
# Train for one epoch
for i in range(n_train_batches):
output, loss = train(epoch, adj_train[i], features_train[i],
idx_train[i], y_train[i])
train_loss.update(loss.data.item(), output.size(0))
train_acc.update(accuracy(output.data, y_train[i].data),
output.size(0))
# Evaluate on validation set
model.eval()
val_loss = AverageMeter()
val_acc = AverageMeter()
for i in range(n_val_batches):
output, loss = test(adj_val[i], features_val[i], idx_val[i],
y_val[i])
val_loss.update(loss.data.item(), output.size(0))
val_acc.update(accuracy(output.data, y_val[i].data),
output.size(0))
# Print results
print("Cross-val iter:", '%02d' % it, "epoch:",
'%03d' % (epoch + 1), "train_loss=",
"{:.5f}".format(train_loss.avg), "train_acc=",
"{:.5f}".format(train_acc.avg), "val_loss=", "{:.5f}".format(
val_loss.avg), "val_acc=", "{:.5f}".format(val_acc.avg),
"time=", "{:.5f}".format(time.time() - start))
# Remember best accuracy and save checkpoint
is_best = val_acc.avg >= best_val_acc
best_val_acc = max(val_acc.avg, best_val_acc)
if is_best:
torch.save(
{
'state_dict': model.state_dict(),
'optimizer': optimizer.state_dict(),
}, 'model_best.pth.tar')
print("Optimization finished!")
# Testing
test_loss = AverageMeter()
test_acc = AverageMeter()
print("Loading checkpoint!")
checkpoint = torch.load('model_best.pth.tar')
model.load_state_dict(checkpoint['state_dict'])
optimizer.load_state_dict(checkpoint['optimizer'])
model.eval()
print("epoch:", epoch)
for i in range(n_test_batches):
output, loss = test(adj_test[i], features_test[i], idx_test[i],
y_test[i])
test_loss.update(loss.data.item(), output.size(0))
test_acc.update(accuracy(output.data, y_test[i].data),
output.size(0))
accs.append(test_acc.avg.cpu().numpy())
# Print results
print("test_loss=", "{:.5f}".format(test_loss.avg), "test_acc=",
"{:.5f}".format(test_acc.avg))
print()
print("avg_test_acc=", "{:.5f}".format(np.mean(accs)))
return np.mean(accs)
def main():
start = timer()
if (os.path.isfile("data/tweets" + str(max_example) + ".npy")
and os.path.isfile("data/emojis" + str(max_example) + ".npy")):
tweets = np.load("data/tweets" + str(max_example) + ".npy").tolist()
emojis = np.load("data/emojis" + str(max_example) + ".npy").tolist()
else:
tweets, emojis = utils.load_data(path='data/final_train',
max_example=max_example)
np.save("data/tweets" + str(max_example) + ".npy", np.array(tweets))
np.save("data/emojis" + str(max_example) + ".npy", np.array(emojis))
if (os.path.isfile("data/dev_tweets" + str(max_dev_example) + ".npy") and
os.path.isfile("data/dev_emojis" + str(max_dev_example) + ".npy")):
dev_tweets = np.load("data/dev_tweets" + str(max_dev_example) +
".npy").tolist()
dev_emojis = np.load("data/dev_emojis" + str(max_dev_example) +
".npy").tolist()
else:
dev_tweets, dev_emojis = utils.load_data(max_example=max_dev_example)
np.save("data/dev_tweets" + str(max_dev_example) + ".npy",
np.array(dev_tweets))
np.save("data/dev_emojis" + str(max_dev_example) + ".npy",
np.array(dev_emojis))
start1 = timer()
print(start1 - start)
word_dict = utils.build_dict(tweets)
# embeddings = utils.generate_embeddings(word_dict, dim=300, pretrained_path='data/glove.6B.300d.txt')
embeddings = utils.generate_embeddings(word_dict,
dim=300,
pretrained_path=None)
end0 = timer()
print(end0 - start1)
x, y = utils.vectorize(tweets, emojis, word_dict)
dev_x, dev_y = utils.vectorize(dev_tweets, dev_emojis, word_dict)
end1 = timer()
print(end1 - end0)
batch_size, input_size, hidden_size, output_size, layers = 32, 300, 200, 20, 1
all_train = utils.generate_batches(x, y, batch_size=batch_size)
all_dev = utils.generate_batches(dev_x, dev_y, batch_size=batch_size)
end2 = timer()
print(end2 - end1)
# set the parameters
# batch_size, input_size, hidden_size, output_size, layers = 64, 50, 200, 20, 1
vocabulary_size = len(embeddings)
if run_GRU:
print("running GRU...")
# initialize the model
model = GRU_Classifier(vocabulary_size, input_size, hidden_size,
output_size, layers, run_BD_GRU)
model.word_embeddings.weight.data = torch.FloatTensor(
embeddings.tolist())
if torch.cuda.is_available():
model.cuda()
(model.word_embeddings.weight.data).cuda()
loss_function = nn.CrossEntropyLoss()
if torch.cuda.is_available():
loss_function.cuda()
optimizer = optim.Adam(model.parameters(), lr=global_learning_rate)
epoch_num = 500
it = 0
best_dev_acc = 0
best_f1 = 0
# model training
for epoch in range(epoch_num):
np.random.shuffle(all_train)
for idx, (mb_x, mb_y, mb_lengths) in enumerate(all_train):
# sort the input in descending order according to sentence length
# This is required by nn.utils.rnn.pack_padded_sequence
sorted_index = len_value_argsort(mb_lengths)
mb_x = [mb_x[i] for i in sorted_index]
mb_y = [mb_y[i] for i in sorted_index]
mb_lengths = [mb_lengths[i] for i in sorted_index]
print('#Examples = %d, max_seq_len = %d' %
(len(mb_x), len(mb_x[0])))
mb_x = Variable(torch.from_numpy(np.array(mb_x,
dtype=np.int64)),
requires_grad=False)
if torch.cuda.is_available():
mb_x = mb_x.cuda()
y_pred = model(mb_x.t(), mb_lengths)
mb_y = Variable(torch.from_numpy(np.array(mb_y,
dtype=np.int64)),
requires_grad=False)
if torch.cuda.is_available():
mb_y = mb_y.cuda()
loss = loss_function(y_pred, mb_y)
# print('epoch ', epoch, 'batch ', idx, 'loss ', loss.data[0])
optimizer.zero_grad()
loss.backward(retain_graph=True)
optimizer.step()
it += 1
if it % 100 == 0: # every 100 updates, check dev accuracy
correct = 0
n_examples = 0
ground_truth = []
predicted = []
for idx, (d_x, d_y, d_lengths) in enumerate(all_dev):
ground_truth += d_y
n_examples += len(d_x)
sorted_index = len_value_argsort(d_lengths)
d_x = [d_x[i] for i in sorted_index]
d_y = [d_y[i] for i in sorted_index]
d_lengths = [d_lengths[i] for i in sorted_index]
d_x = Variable(torch.from_numpy(
np.array(d_x, dtype=np.int64)),
requires_grad=False)
if torch.cuda.is_available():
d_x = d_x.cuda()
# use pytorch way to calculate the correct count
d_y = Variable(torch.from_numpy(
np.array(d_y, dtype=np.int64)),
requires_grad=False)
if torch.cuda.is_available():
d_y = d_y.cuda()
y_pred = model(d_x.t(), d_lengths)
predicted += list(
torch.max(y_pred, 1)[1].view(d_y.size()).data)
correct += (torch.max(y_pred, 1)[1].view(
d_y.size()).data == d_y.data).sum()
dev_acc = correct / n_examples
f1 = f1_score(ground_truth, predicted, average='macro')
print("Dev Accuracy: %f, F1 Score: %f" % (dev_acc, f1))
if f1 > best_f1:
best_f1 = f1
print("Best F1 Score: %f" % best_f1)
gru_output = open('./out/gru_best', 'w')
gru_output.write(str(ground_truth) + '\n')
gru_output.write(str(predicted) + '\n')
gru_output.write(str(best_f1) + ' ' + str(dev_acc))
gru_output.close()
if dev_acc > best_dev_acc:
best_dev_acc = dev_acc
print("Best Dev Accuracy: %f" % best_dev_acc)
if run_LSTM:
print("Running LSTM...")
model = LSTM_Classifier(vocabulary_size, input_size, hidden_size,
output_size, layers, run_BD_LSTM)
model.word_embeddings.weight.data = torch.FloatTensor(
embeddings.tolist())
if torch.cuda.is_available():
model.cuda()
(model.word_embeddings.weight.data).cuda()
loss_function = nn.CrossEntropyLoss()
if torch.cuda.is_available():
loss_function.cuda()
optimizer = optim.Adam(model.parameters(), lr=global_learning_rate)
it = 0
best_dev_acc = 0
best_f1 = 0
epoch_num = 500
# train LSTM
for epoch in range(epoch_num):
np.random.shuffle(all_train)
for idx, (mb_x, mb_y, mb_lengths) in enumerate(all_train):
sorted_index = len_value_argsort(mb_lengths)
mb_x = [mb_x[i] for i in sorted_index]
mb_y = [mb_y[i] for i in sorted_index]
mb_lengths = [mb_lengths[i] for i in sorted_index]
print('#Examples = %d, max_seq_len = %d' %
(len(mb_x), len(mb_x[0])))
mb_x = Variable(torch.from_numpy(np.array(mb_x,
dtype=np.int64)),
requires_grad=False)
if torch.cuda.is_available():
mb_x = mb_x.cuda()
y_pred = model(mb_x.t(), mb_lengths)
mb_y = Variable(torch.from_numpy(np.array(mb_y,
dtype=np.int64)),
requires_grad=False)
if torch.cuda.is_available():
mb_y = mb_y.cuda()
loss = loss_function(y_pred, mb_y)
# print('epoch ', epoch, 'batch ', idx, 'loss ', loss.data[0])
optimizer.zero_grad()
loss.backward(retain_graph=True)
optimizer.step()
it += 1
if it % 100 == 0: # every 100 updates, check dev accuracy
correct = 0
n_examples = 0
ground_truth = []
predicted = []
for idx, (d_x, d_y, d_lengths) in enumerate(all_dev):
ground_truth += d_y
n_examples += len(d_x)
sorted_index = len_value_argsort(d_lengths)
d_x = [d_x[i] for i in sorted_index]
d_y = [d_y[i] for i in sorted_index]
d_lengths = [d_lengths[i] for i in sorted_index]
d_x = Variable(torch.from_numpy(
np.array(d_x, dtype=np.int64)),
requires_grad=False)
if torch.cuda.is_available():
d_x = d_x.cuda()
d_y = Variable(torch.from_numpy(
np.array(d_y, dtype=np.int64)),
requires_grad=False)
if torch.cuda.is_available():
d_y = d_y.cuda()
y_pred = model(d_x.t(), d_lengths)
predicted += list(
torch.max(y_pred, 1)[1].view(d_y.size()).data)
correct += (torch.max(y_pred, 1)[1].view(
d_y.size()).data == d_y.data).sum()
dev_acc = correct / n_examples
f1 = f1_score(ground_truth, predicted, average='macro')
print("Dev Accuracy: %f, F1 Score: %f" % (dev_acc, f1))
if f1 > best_f1:
best_f1 = f1
print("Best F1 Score: %f" % best_f1)
lstm_output = open('./out/lstm_best', 'w')
lstm_output.write(str(ground_truth) + '\n')
lstm_output.write(str(predicted) + '\n')
lstm_output.write(str(best_f1) + ' ' + str(dev_acc))
lstm_output.close()
if dev_acc > best_dev_acc:
best_dev_acc = dev_acc
print("Best Dev Accuracy: %f" % best_dev_acc)
def train(self,
X,
y,
X_valid=None,
y_valid=None,
batch_size=200,
n_epoch=40,
batch_seed=0,
verbose=True):
"""Train the neural network with data.
Args:
X: numpy.ndarray
Input data of size `n` (sample size) by `p` (data dimension).
y: numpy.ndarray (binary)
One-hot labels of size `n` (sample size) by `k` (# classes).
X_valid: numpy.ndarray
Optional validation data matrix. If provided with `y_valid`,
current validation error rate is stored in the model.
y_valid: numpy.ndarray
Optional validation outcome vector. If provided with `X_valid`,
current validation error rate is stored in the model.
batch_size: int
Size of random batches of the input data.
n_epoch: int
Number of epochs to train on the input data.
batch_seed: int
First random seed for batch selection.
verbose: bool
If true (default), report training updates per epoch to stdout.
Returns:
nn: NN
Trained `NN` model.
"""
assert self.layers[0].n_in == X.shape[1]
assert X.shape[0] == y.shape[0]
n = X.shape[0]
n_batches = int(np.ceil(n / batch_size))
if verbose:
print(
'|-------|---------------------------|---------------------------|'
)
print(
'| Epoch | Training | Validation |'
)
print(
'|-------|---------------------------|---------------------------|'
)
print(
'| # | Error | Cross-Ent | Error | Cross-Ent |'
)
print(
'|-------|---------------------------|---------------------------|'
)
for t in range(n_epoch):
for i, batch in enumerate(\
generate_batches(n, batch_size, batch_seed + t)):
# Forward propagation (last h is output prob)
h = X[batch, :]
for l in self.layers:
h = l.fprop(h, update_units=True)
# Backpropagation
grad = -(y[batch, :] - h)
for l in self.layers[::-1]:
grad = l.bprop(grad)
self.epoch += 1
for l in self.layers:
l.learning_rate.epoch = self.epoch
# Errors
training_error = self.compute_error(X, y)
training_loss = self.compute_cross_entropy(X, y)
self.training_error.append((self.epoch, training_error))
self.training_loss.append((self.epoch, training_loss))
if X_valid is not None and y_valid is not None:
validation_error = self.compute_error(X_valid, y_valid)
validation_loss = self.compute_cross_entropy(X_valid, y_valid)
self.validation_error.append((self.epoch, validation_error))
self.validation_loss.append((self.epoch, validation_loss))
if verbose:
print('| {:3d} | {:.5f} | {:.5f} | {:.5f} | {:.5f} |'.\
format(self.epoch, training_error, training_loss,
validation_error, validation_loss))
if self.early_stopping:
if (self.epoch >= 40
and self.validation_loss[-2][1] < validation_loss
and self.validation_loss[-3][1] < validation_loss
and self.validation_loss[-4][1] < validation_loss):
print('======Early stopping: validation loss increase at epoch {:3d}======'.\
format(self.epoch))
break
else:
if verbose:
print('| {:3d} | {:.5f} | {:.5f} | | |'.\
format(self.epoch, training_error, training_loss))
if verbose:
print(
'|-------|---------------------------|---------------------------|'
)
return self
def train_keras(self):
tf.set_random_seed(1)
# self.session.run(tf.global_variables_initializer())
sys.stdout.write('\n Global variables initialized \n')
n_batch_iters = self.num_train_examples // self.batch_size
batch_eval_thresh = self.print_freq // self.batch_size
for epoch in range(self.epochs):
# Hack to continue training from previous model
# epoch = epoch + 1
train_datagen = utils.generate_batches(filename=self.train_file,
seqlen=self.seq_len,
vocab=self.vocab,
batch_size=self.batch_size)
print('starting training epoch {}'.format(epoch + 1))
train_loss = 0
count = 0
batch_count = 0
for batch_idx in range(n_batch_iters):
x_batch, y_batch = next(train_datagen)
train_loss_ = self.model.train_on_batch(x_batch, y_batch)
count += len(x_batch)
batch_count += 1
train_loss += train_loss_
update_loss = train_loss / batch_count
update_prpl = math.exp(update_loss)
sys.stdout.write(
'\rloss = {0:8.3f} - perplexity = {1:8.3f} - train iters = {2} / {3} '
.format(update_loss, update_prpl, batch_count,
n_batch_iters))
if (batch_count > 0) and (batch_count % batch_eval_thresh
== 0):
print(
'\n>> Update: Average train loss : {} - Average train prpl : {}\n'
.format(update_loss, update_prpl))
valid_loss = self.validate_keras(
max_count=self.print_freq // 16)
print(
'\n>>Update: Average validation loss : {} - Average valdiation prpl : {}\n'
.format(valid_loss, math.exp(valid_loss)))
# Epoch summaries
print('current count:', count)
train_loss /= batch_count
valid_total_loss = self.validate_keras(
max_count=self.num_val_examples)
print()
print('Epoch {} Summary:'.format(epoch + 1))
print('=' * 80)
print(
"(epoch {0:5}) train loss = {1:8.3f} ({2:8.3f}); valid loss = {3:8.3f} ({4:8.3f})"
.format(epoch, train_loss, math.exp(train_loss),
valid_total_loss, math.exp(valid_total_loss)))
print('=' * 80)
print()
# Save epoch model here
self.model.save(
'keras_lstm_lm_epoch{0}_val-prpl{1:8.3f}_v2.h5'.format(
epoch + 21, math.exp(valid_total_loss)))
features_test = list()
subgraphs_test = list()
subgraphs_tmp = [subgraphs[i] for i in test_index]
c = 0
for s in subgraphs_tmp:
l = list()
for i in s:
adj_test.append(adj[i])
features_test.append(features[i])
l.append(c)
c += 1
subgraphs_test.append(l)
y_test = [y[i] for i in test_index]
adj_train, adj_s_train, features_train, shapes_train, y_train = generate_batches(
adj_train, features_train, subgraphs_train, y_train, args.batch_size,
args.use_master_node, args.graph_of_sentences)
adj_val, adj_s_val, features_val, shapes_val, y_val = generate_batches(
adj_val, features_val, subgraphs_val, y_val, args.batch_size,
args.use_master_node, args.graph_of_sentences)
adj_test, adj_s_test, features_test, shapes_test, y_test = generate_batches(
adj_test, features_test, subgraphs_test, y_test, args.batch_size,
args.use_master_node, args.graph_of_sentences)
n_train_batches = ceil(n_train / args.batch_size)
n_val_batches = ceil(n_val / args.batch_size)
n_test_batches = ceil(n_test / args.batch_size)
# Model and optimizer
model = MPAD(embeddings.shape[1], args.message_passing_layers, args.hidden,
args.penultimate, nclass, args.dropout, embeddings,
layer.trainable = False
model.compile(optimizer='rmsprop', loss='categorical_crossentropy')
print("Start training...")
import glob
files = glob.glob(train_path + '/*/*jpg')
samples = len(files)
if data_augmentation:
model.fit_generator(generate_batches_with_augmentation(
train_path, batch_size, validation_split, augmented_data),
verbose=1,
epochs=epochs,
callbacks=[checkpoint])
else:
model.fit_generator(generate_batches(train_path, batch_size),
epochs=epochs,
steps_per_epoch=samples // batch_size,
verbose=1,
callbacks=[checkpoint])
print("Saving...")
model.save(model_path + "/save_model_stage1.h5")
# print ("Visualization...")
# for i, layer in enumerate(base_model.layers):
# print(i, layer.name)
if epochs_after_unfreeze > 0:
print("Unfreezing all layers...")
for i in range(len(model.layers)):
# Load input data and train model on the input data
# Create complete training set
triples_data = pickle.load(
open((config.data_dir) + '/input_train_triples.pkl', 'rb'))
cornell_data = pickle.load(
open((config.data_dir) + '/input_train_cornell.pkl', 'rb'))
twitter_data = pickle.load(
open((config.data_dir) + '/input_train_twitter.pkl', 'rb'))
train_data = [
triples_data[0] + cornell_data[0] + twitter_data[0],
triples_data[1] + cornell_data[1] + twitter_data[1]
]
eval_data = pickle.load(
open((config.data_dir) + '/input_test_triples.pkl', 'rb'))
train_batches = generate_batches(train_data,
batch_size=config.batch_size,
num_epochs=config.n_epochs)
eval_batches = generate_batches(eval_data,
batch_size=config.batch_size,
num_epochs=1)
# model.train(train_batches, eval_batches, verbose=True)
# # Evaluate perplexity of trained model on validation data
# model_dir = 'runs/1496648182'
# model_file = '/model-12000'
# # print(model.evaluate(eval_batches, model_dir=model_dir, model_file=model_file))
#
# Infer outputs on a subset of test data
model_dir = 'runs/149664818'
model_file = '/model-12000'
checkpoint_file = model_dir + model_file
def __init__(self,
sess,
data_type='synthetic',
batch_size=10,
Xtrain=None,
Ytrain=None,
Xtest=None,
Ytest=None,
Ytest_und=None,
Sig=None,
M0=None,
Params=None):
# Initialize the object
self.sess = sess
# Input shape
if data_type == 'synthetic':
self.height_MR = 70
self.width_MR = 70
self.channels = 10 # Number of PLDs or time-points
else:
self.height_MR = 40
self.width_MR = 40
self.channels = 7
self.output_dim = 2 # number of ASL parameters to estimate (CBF, ATT)
self.data_type = data_type
# Train Data
self.Xtrain = Xtrain #
self.Ytrain = Ytrain
# Test Data
self.Xtest = Xtest
self.Ytest = Ytest
self.Ytest_und = Ytest_und
# Number of residual blocks in the generator
self.n_residual_blocks = 8
# Depth of dual paths in the generator
self.depth_dual = 4
# Weight decay
self.wd = 0.0005
self.df = 64
# Batch normalization flag
self.bn_g = False # For generator
self.bn_d = True # For discriminator
# Lambda value for loss terms
self.lam_lp = 1.0
self.lam_asl = 3.0 # 3.0 is the optimal
self.lam_adv = 0.5
print(
"[ LP loss coef: %f, ASL loss coef: %f, Adversarial loss coef: %f ]"
% (self.lam_lp, self.lam_asl, self.lam_adv))
# Learning rate
self.learning_rate = 1e-3
self.batch_size = batch_size
# Norm of reconstruction term
self.l_num = 1
# Get the generic ASL parameters
self.common_vars = ut.load_variables(Params)
# Call the batch generation for every iteration of GAN
self.data_generator = ut.generate_batches(Xtrain, Ytrain, Sig, M0,
self.batch_size)
# Build GAN model
self.build_model()
dataset.set_split('test')
test_bar = tqdm(desc='split=test',
total=dataset.get_num_batches(args.batch_size),
position=1,
leave=True)
if args.train:
try:
for epoch_index in range(args.num_epochs):
train_state['epoch_index'] = epoch_index
# Iterate over training dataset
# setup: batch generator, set loss and acc to 0, set train mode on
dataset.set_split('train')
batch_generator = generate_batches(dataset,
batch_size=args.batch_size,
device=args.device)
running_loss = 0.0
running_acc = 0.0
classifier.train()
for batch_index, batch_dict in enumerate(batch_generator):
# the training routine is these 5 steps:
# --------------------------------------
# step 1. zero the gradients
optimizer.zero_grad()
# step 2. compute the output
# print(f'Classifier is \n {classifier}')
if args.architecture_type == 'RNN':
def main(_):
# Load config
config = utils.load_config(FLAGS.config)
ckpt_dir = os.path.join(config['training']['ckpt_dir'],
config['experiment_name'])
ckpt = os.path.join(ckpt_dir, 'model.ckpt')
log_dir = os.path.join(config['training']['log_dir'],
config['experiment_name'])
if not os.path.exists(ckpt_dir):
tf.logging.info('Creating checkpoint directory: %s' % ckpt_dir)
os.mkdir(ckpt_dir)
if not os.path.exists(log_dir):
tf.logging.info('Creating log directory: %s' % log_dir)
config_path = os.path.join(ckpt_dir, 'config.yaml')
if os.path.exists(config_path):
tf.logging.info('Existing configuration file detected.')
if config != utils.load_config(config_path):
if FLAGS.force:
warnings.warn(
'Specified configuration does not match '
'existing configuration in checkpoint directory. '
'Forcing overwrite of existing configuration.')
shutil.copyfile(FLAGS.config, config_path)
else:
raise ValueError('Specified configuration does not match '
'existing configuration in checkpoint '
'directory.')
else:
tf.logging.info('No existing configuration found. Copying config file '
'to "%s".' % config_path)
shutil.copyfile(FLAGS.config, config_path)
g = tf.Graph()
with g.as_default():
tf.logging.info('Creating graph')
build_graph(config)
global_step = tf.train.get_or_create_global_step()
saver = tf.train.Saver(max_to_keep=5)
init_fn = get_init_fn(config)
total_loss = tf.losses.get_total_loss()
trainable_vars = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES)
trainable_vars = [
var for var in trainable_vars if 'vgg' not in var.name
]
learning_rate = tf.train.exponential_decay(
learning_rate=config['training']['initial_learning_rate'],
global_step=global_step,
decay_steps=config['training']['decay_steps'],
decay_rate=config['training']['decay_rate'],
staircase=True)
optimizer = tf.train.AdamOptimizer(learning_rate)
train_op = slim.learning.create_train_op(
total_loss,
optimizer,
clip_gradient_norm=config['training']['gradient_clipping'],
variables_to_train=trainable_vars,
summarize_gradients=True)
summary_op = tf.summary.merge_all()
eval_logger = tf.summary.FileWriter(log_dir)
metric_op = tf.get_collection('metrics')
update_op = tf.get_collection('updates')
streaming_vars = [
i for i in tf.local_variables() if 'streaming' in i.name
]
reset_op = [tf.variables_initializer(streaming_vars)]
with tf.Session() as sess:
sess.run([
tf.global_variables_initializer(),
tf.local_variables_initializer()
])
init_fn(sess) # Responsible for restoring variables / warm starts
# Generate data loop.
for feed_dict, uris in utils.generate_batches('train',
config,
mc_attr=True):
try:
i, _ = sess.run([global_step, train_op],
feed_dict=feed_dict)
except tf.errors.InvalidArgumentError: # Encountered a bad JPEG
continue
if not i % config['training']['log_frequency']:
try:
loss = sess.run(total_loss, feed_dict=feed_dict)
except tf.errors.InvalidArgumentError: # Encountered a bad JPEG
continue
tf.logging.info('Iteration %i - Loss: %0.4f' % (i, loss))
if not i % config['training']['save_frequency']:
tf.logging.info('Saving checkpoint for iteration %i' % i)
saver.save(sess, ckpt)
sess.run(reset_op)
# Evaluate on test data.
for feed_dict, uris in utils.generate_batches(
'val', config, mc_attr=True):
try:
sess.run(update_op, feed_dict=feed_dict)
except tf.errors.InvalidArgumentError:
continue
print(sess.run(metric_op))
# Write summaries.
summary = sess.run(summary_op, feed_dict=feed_dict)
eval_logger.add_summary(summary, i)
if i >= config['training']['max_steps']:
tf.logging.info('Training complete')
sys.exit(0)
def train_3grams_layered_model(train_datasets,
dev_datasets,
test_datasets,
model_path,
model_ID=0,
epochs=NUM_EPOCHS,
batch_size=BATCH_SIZE,
hidden_size=HIDDEN_SIZE,
lr=LEARNING_RATE,
dropout=DROPOUT,
rec_dropout=REC_DROPOUT):
"""
Builds and trains a 2 layers 3-grams model with the given hyper-parameters and training options.
:param train_datasets: Tuple (input file, label file) of the training set
:param dev_datasets: Tuple (input file, label file) of the development set
:param test_datasets: Tuple (input file, label file) of the test set
:param model_path: Path to file to be used for saving the trained model
:param model_ID: self-explanatory
:param epochs: Number of epochs the model has to train for
:param batch_size: self-explanatory
:param hidden_size: Number of hidden units of the model
:param lr: Learning rate
:param dropout: Dropout probability for each unit
:param rec_dropout: Recurrent dropout probability for each unit
:return: tuple (last recorded accuracy on the training set, last recorded accuracy on dev set, accuracy on test set)
"""
print("Creating model...")
x_1grams, x_2grams, x_3grams, y, \
keep_pr, recurrent_keep_pr, \
lengths, train, \
loss, preds = m.get_3grams_layered_model(
pretrained_emb_1grams=emb_matrix_1grams,
pretrained_emb_2grams=emb_matrix_2grams,
pretrained_emb_3grams=emb_matrix_3grams,
hidden_size=hidden_size,
layers=1,
y_size=len(labels_to_idx),
learning_rate=lr
)
model_params = "ID:%d, units:%d, lr:%.5f, dr:%.2f, rec_dr: %.2f]" % (model_ID, hidden_size, lr, dropout, rec_dropout)
saver = tf.train.Saver()
with \
tf.Session() as sess,\
tf.summary.FileWriter("logging/3grams_2layers_model %s" % model_params, sess.graph) as train_logger:
sess.run(tf.global_variables_initializer())
# keep track of last accuracies recorded to allow greedy optimization
accuracies = []
print("Starting training...")
for epoch in range(epochs):
time = "[%s]" % (str(datetime.now()))
print("Epoch %d %s" % (epoch, time))
accumulated_loss = 0
accumulated_acc = 0
iterations = 0
for batch_inputs,\
batch_labels,\
batch_lengths in u.generate_batches(dataset_input=train_datasets[0],
dataset_label=train_datasets[1],
batch_size=batch_size,
label_to_idx=labels_to_idx,
ngram_features=[1, 2, 3],
word_to_idx=[word_to_idx_1grams, word_to_idx_2grams, word_to_idx_3grams]):
_, loss_val, preds_val = sess.run(
[train, loss, preds],
feed_dict={x_1grams: batch_inputs[0],
x_2grams: batch_inputs[1],
x_3grams: batch_inputs[2],
y: batch_labels,
lengths: batch_lengths,
keep_pr: 1.0-dropout,
recurrent_keep_pr: 1.0-rec_dropout}
)
accumulated_loss += loss_val
accumulated_acc += compute_accuracy(preds_val, batch_labels)
iterations += 1
accumulated_loss /= iterations
accumulated_acc /= iterations
print("\t- Train set:\tLoss: %.5f\tAcc: %.5f" % (accumulated_loss, accumulated_acc))
# only store the last accuracy recorded
if epoch == epochs-1:
accuracies.append(accumulated_acc)
add_summary(train_logger,
"train loss",
accumulated_loss,
epoch)
add_summary(train_logger,
"train acc",
accumulated_acc,
epoch)
# dev set
accumulated_loss = 0
accumulated_acc = 0
iterations = 0
for batch_inputs,\
batch_labels,\
batch_lengths in u.generate_batches(dataset_input=dev_datasets[0],
dataset_label=dev_datasets[1],
batch_size=batch_size,
label_to_idx=labels_to_idx,
ngram_features=[1, 2, 3],
word_to_idx=[word_to_idx_1grams, word_to_idx_2grams, word_to_idx_3grams]):
loss_val, preds_val = sess.run(
[loss, preds],
feed_dict={x_1grams: batch_inputs[0],
x_2grams: batch_inputs[1],
x_3grams: batch_inputs[2],
y: batch_labels,
lengths: batch_lengths,
keep_pr: 1.0,
recurrent_keep_pr: 1.0}
)
accumulated_loss += loss_val
accumulated_acc += compute_accuracy(preds_val, batch_labels)
iterations += 1
accumulated_loss /= iterations
accumulated_acc /= iterations
print("\t- Dev set:\t\tLoss: %.5f\tAcc: %.5f" % (accumulated_loss, accumulated_acc))
# only store the last accuracy recorded
if epoch == epochs - 1:
accuracies.append(accumulated_acc)
# save model periodically
if epoch % 10 == 0:
saver.save(sess, model_path)
time = "[%s]" % (str(datetime.now()))
print("Model saved: %s" % time)
add_summary(train_logger,
"dev loss",
accumulated_loss,
epoch)
add_summary(train_logger,
"dev acc",
accumulated_acc,
epoch)
# save model after training ended
saver.save(sess, model_path)
time = "[%s]" % (str(datetime.now()))
print("Training ended %s.\nEvaluating model on test set..." % time)
accumulated_loss = 0
accumulated_acc = 0
iterations = 0
for batch_inputs, \
batch_labels, \
batch_lengths in u.generate_batches(dataset_input=test_datasets[0],
dataset_label=test_datasets[1],
batch_size=batch_size,
label_to_idx=labels_to_idx,
ngram_features=[1, 2, 3],
word_to_idx=[word_to_idx_1grams, word_to_idx_2grams, word_to_idx_3grams],
to_shuffle=False):
loss_val, preds_val = sess.run(
[loss, preds],
feed_dict={x_1grams: batch_inputs[0],
x_2grams: batch_inputs[1],
x_3grams: batch_inputs[2],
y: batch_labels,
lengths: batch_lengths,
keep_pr: 1.0,
recurrent_keep_pr: 1.0}
)
accumulated_loss += loss_val
accumulated_acc += compute_accuracy(preds_val, batch_labels)
iterations += 1
accumulated_loss /= iterations
accumulated_acc /= iterations
print("Test set:\tLoss: %.5f\tAcc: %.5f" % (accumulated_loss, accumulated_acc))
add_summary(train_logger,
"test loss",
accumulated_loss,
epoch)
add_summary(train_logger,
"test acc",
accumulated_acc,
epoch)
# always store accuracy on test set
accuracies.append(accumulated_acc)
return accuracies
n_val = len(val_index)
n_test = len(test_index)
adj_train = [adj_lst[i] for i in train_index]
features_train = [features_lst[i] for i in train_index]
y_train = [y[i] for i in train_index]
adj_val = [adj_lst[i] for i in val_index]
features_val = [features_lst[i] for i in val_index]
y_val = [y[i] for i in val_index]
adj_test = [adj_lst[i] for i in test_index]
features_test = [features_lst[i] for i in test_index]
y_test = [y[i] for i in test_index]
adj_train, features_train, graph_indicator_train, y_train = generate_batches(
adj_train, features_train, y_train, args.batch_size, device)
adj_val, features_val, graph_indicator_val, y_val = generate_batches(
adj_val, features_val, y_val, args.batch_size, device)
adj_test, features_test, graph_indicator_test, y_test = generate_batches(
adj_test, features_test, y_test, args.batch_size, device)
n_train_batches = ceil(n_train / args.batch_size)
n_val_batches = ceil(n_val / args.batch_size)
n_test_batches = ceil(n_test / args.batch_size)
model = RW_NN(features_dim, args.max_step, args.hidden_graphs,
args.size_hidden_graphs, args.hidden_dim,
args.penultimate_dim, args.normalize, n_classes,
args.dropout, device).to(device)
optimizer = optim.Adam(model.parameters(), lr=args.lr)
scheduler = optim.lr_scheduler.StepLR(optimizer, step_size=50, gamma=0.5)
def create_multiple_link_clicks_ads(
self,
accountid,
pageid,
name,
titles,
bodies,
urls,
image_paths,
targeting,
optimization_goal,
billing_event,
bid_amount,
daily_budget=None,
lifetime_budget=None,
start_time=None,
end_time=None,
):
"""
There are 7 steps in this sample:
1. Create a campaign
2. Create an ad set
3. Upload images
4. Make combinations of specified creative elements
5. Prepare an API batch
6. For each creative combination, add a call to the batch to create a
new ad
7. Execute API batch
Params:
* `accountid` is your Facebook AdAccount id
* `name` is a string of basename of your ads.
* `page` is the Facebook page used to publish the ads
* `titles` array of strings of what user will see as ad title
* `bodies` array of strings of what user will see as ad body
* `image_paths` array of image file paths
* `targeting` is a JSON string specifying targeting info of your ad
See [Targeting Specs](https://developers.facebook.com/docs/marketing-api/targeting-specs)
for details.
* `optimization_goal` the optimization goal for the adsets
* `billing_event` what you want to pay for
* `bid_amount` how much you want to pay per billing event
* `daily_budget` is integer number in your currency. E.g., if your
currency is USD, dailybudget=1000 says your budget is 1000 USD
* `lifetime_budget` lifetime budget for created ads
* `start_time` when the campaign should start
* `end_time` when the campaign should end
"""
my_account = AdAccount(fbid=accountid)
"""
Take different title body url and image paths, create a batch of
ads based on the permutation of these elements
"""
# Check for bad specs
if daily_budget is None:
if lifetime_budget is None:
raise TypeError(
'One of daily_budget or lifetime_budget must be defined.')
elif end_time is None:
raise TypeError(
'If lifetime_budget is defined, end_time must be defined.')
"""
Step 1: Create new campaign with WEBSITE_CLICKS objective
See
[Campaign Group](https://developers.facebook.com/docs/marketing-api/reference/ad-campaign-group)
for further details on the API used here.
"""
campaign = Campaign(parent_id=accountid)
campaign[Campaign.Field.name] = name + ' Campaign'
campaign[Campaign.Field.objective] = \
Campaign.Objective.link_clicks
campaign.remote_create(params={
'status': Campaign.Status.paused,
})
"""
Step 2: Create AdSet using specified optimization goal, billing event
and bid.
See
[AdSet](https://developers.facebook.com/docs/marketing-api/reference/ad-campaign)
for further details on the API used here.
"""
# Create ad set
ad_set = AdSet(parent_id=accountid)
ad_set[AdSet.Field.campaign_id] = campaign.get_id_assured()
ad_set[AdSet.Field.name] = name + ' AdSet'
ad_set[AdSet.Field.optimization_goal] = optimization_goal
ad_set[AdSet.Field.billing_event] = billing_event
ad_set[AdSet.Field.bid_amount] = bid_amount
if daily_budget:
ad_set[AdSet.Field.daily_budget] = daily_budget
else:
ad_set[AdSet.Field.lifetime_budget] = lifetime_budget
if end_time:
ad_set[AdSet.Field.end_time] = end_time
if start_time:
ad_set[AdSet.Field.start_time] = start_time
ad_set[AdSet.Field.targeting] = targeting
ad_set.remote_create()
"""
Step 3: Upload images and get image hashes for use in ad creative.
See
[Ad Image](https://developers.facebook.com/docs/marketing-api/reference/ad-image#Creating)
for further details on the API used here.
"""
# Upload the images first one by one
image_hashes = []
for image_path in image_paths:
img = AdImage(parent_id=accountid)
img[AdImage.Field.filename] = image_path
img.remote_create()
image_hashes.append(img.get_hash())
ADGROUP_BATCH_CREATE_LIMIT = 10
ads_created = []
def callback_failure(response):
raise response.error()
"""
Step 4: Using itertools.product get combinations of creative
elements.
"""
for creative_info_batch in generate_batches(
itertools.product(titles, bodies, urls, image_hashes),
ADGROUP_BATCH_CREATE_LIMIT):
"""
Step 5: Create an API batch so we can create all
ad creatives with one HTTP request.
See
[Batch Requests](https://developers.facebook.com/docs/graph-api/making-multiple-requests#simple)
for further details on batching API calls.
"""
api_batch = my_account.get_api_assured().new_batch()
for title, body, url, image_hash in creative_info_batch:
# Create the ad
"""
Step 6: For each combination of creative elements,
add to the batch an API call to create a new Ad
and specify the creative inline.
See
[AdGroup](https://developers.facebook.com/docs/marketing-api/adgroup/)
for further details on creating Ads.
"""
ad = Ad(parent_id=accountid)
ad[Ad.Field.name] = name + ' Ad'
ad[Ad.Field.adset_id] = ad_set.get_id_assured()
ad[Ad.Field.creative] = {
AdCreative.Field.object_story_spec: {
"page_id": pageid,
"link_data": {
"message": body,
"link": url,
"caption": title,
"image_hash": image_hash
}
},
}
ad.remote_create(batch=api_batch, failure=callback_failure)
ads_created.append(ad)
"""
Step 7: Execute the batched API calls
See
[Batch Requests](https://developers.facebook.com/docs/graph-api/making-multiple-requests#simple)
for further details on batching API calls.
"""
api_batch.execute()
return [campaign, ad_set, ads_created]
def main(_):
# Load config
config = utils.load_config(FLAGS.config)
ckpt_dir = os.path.join(config['training']['ckpt_dir'],
config['experiment_name'])
ckpt = os.path.join(ckpt_dir, 'av.ckpt')
log_dir = os.path.join(config['training']['log_dir'],
config['experiment_name'])
if not os.path.exists(ckpt_dir):
tf.logging.info('Creating checkpoint directory: %s' % ckpt_dir)
os.mkdir(ckpt_dir)
if not os.path.exists(log_dir):
tf.logging.info('Creating log directory: %s' % log_dir)
g = tf.Graph()
with g.as_default():
tf.logging.info('Creating graph')
build_graph(config)
global_step = tf.train.get_or_create_global_step()
saver = tf.train.Saver(max_to_keep=5)
init_fn = get_init_fn(config)
total_loss = tf.losses.get_total_loss()
optimizer = tf.train.AdamOptimizer(config['training']['learning_rate'])
train_op = slim.learning.create_train_op(
total_loss,
optimizer,
clip_gradient_norm=config['training']['gradient_clipping'],
summarize_gradients=True)
summary_op = tf.summary.merge_all()
eval_logger = tf.summary.FileWriter(log_dir)
metric_op = tf.get_collection('metrics')
update_op = tf.get_collection('updates')
streaming_vars = [
i for i in tf.local_variables() if 'streaming' in i.name
]
reset_op = [tf.variables_initializer(streaming_vars)]
with tf.Session() as sess:
sess.run([
tf.global_variables_initializer(),
tf.local_variables_initializer()
])
init_fn(sess) # Responsible for restoring variables / warm starts
# Evaluate on test data.
for feed_dict, uris in utils.generate_batches('val', config):
sess.run(update_op, feed_dict=feed_dict)
print(sess.run(metric_op))
# Write summaries.
summary = sess.run(summary_op, feed_dict=feed_dict)
eval_logger.add_summary(summary, 0)
# Generate data loop.
for feed_dict, uris in utils.generate_batches('train', config):
try:
i, _ = sess.run([global_step, train_op],
feed_dict=feed_dict)
except tf.errors.InvalidArgumentError: # Encountered a bad JPEG
continue
if not i % config['training']['log_frequency']:
loss = sess.run(total_loss, feed_dict=feed_dict)
tf.logging.info('Iteration %i - Loss: %0.4f' % (i, loss))
if not i % config['training']['save_frequency']:
tf.logging.info('Saving checkpoint for iteration %i' % i)
saver.save(sess, ckpt)
sess.run(reset_op)
# Evaluate on test data.
for feed_dict, uris in utils.generate_batches(
'val', config):
sess.run(update_op, feed_dict=feed_dict)
print(sess.run(metric_op))
# Write summaries.
summary = sess.run(summary_op, feed_dict=feed_dict)
eval_logger.add_summary(summary, i)
if i >= config['training']['max_steps']:
tf.logging.info('Training complete')
sys.exit(0)
n_test = len(test_index)
adj_train = [adj[i] for i in train_index]
features_train = [features[i] for i in train_index]
y_train = [y[i] for i in train_index]
adj_val = [adj[i] for i in val_index]
features_val = [features[i] for i in val_index]
y_val = [y[i] for i in val_index]
adj_test = [adj[i] for i in test_index]
features_test = [features[i] for i in test_index]
y_test = [y[i] for i in test_index]
adj_train, features_train, batch_n_graphs_train, y_train = generate_batches(
adj_train, features_train, y_train, args.batch_size,
args.use_master_node)
adj_val, features_val, batch_n_graphs_val, y_val = generate_batches(
adj_val, features_val, y_val, args.batch_size, args.use_master_node)
adj_test, features_test, batch_n_graphs_test, y_test = generate_batches(
adj_test, features_test, y_test, args.batch_size, args.use_master_node)
n_train_batches = ceil(n_train / args.batch_size)
n_val_batches = ceil(n_val / args.batch_size)
n_test_batches = ceil(n_test / args.batch_size)
# Model and optimizer
model = MPAD(embeddings.shape[1], args.message_passing_layers, args.hidden,
args.penultimate, nclass, args.dropout, embeddings,
args.use_master_node)
def train_model(
classifier,
dataset,
loss_func,
optimizer,
args,
):
train_state = make_train_state(args)
for epoch_index in range(args.num_epochs):
train_state['epoch_index'] = epoch_index
dataset.set_split('train')
batch_generator = generate_batches(dataset,
batch_size=args.batch_size,
device=args.device)
running_loss = 0.0
running_acc = 0.0
classifier.train()
batch_generator = tqdm(batch_generator)
batch_generator.set_description(
f"E:({epoch_index+1}/{args.num_epochs})")
for batch_index, batch_dict in enumerate(batch_generator):
optimizer.zero_grad()
y_pred = classifier(batch_dict['x_data'].float())
loss = loss_func(y_pred, batch_dict['y_target'].float())
loss_batch = loss.item()
running_loss += (loss_batch - running_loss) / (batch_index + 1)
loss.backward()
optimizer.step()
acc_batch = compute_accuracy(y_pred, batch_dict['y_target'])
running_acc += (acc_batch - running_acc) / (batch_index + 1)
batch_generator.set_postfix(train_acc=running_acc,
train_loss=running_loss)
train_state['train_acc'].append(running_acc)
train_state['train_loss'].append(running_loss)
dataset.set_split('valid')
batch_generator = generate_batches(dataset,
batch_size=args.batch_size,
device=args.device)
batch_generator = tqdm(batch_generator)
batch_generator.set_description(
f"E:({epoch_index+1}/{args.num_epochs})")
running_loss = 0.0
running_acc = 0.0
classifier.eval()
for batch_index, batch_dict in enumerate(batch_generator):
y_pred = classifier(batch_dict['x_data'].float())
loss = loss_func(y_pred, batch_dict['y_target'].float())
loss_batch = loss.item()
running_loss += (loss_batch - running_loss) / (batch_index + 1)
acc_batch = compute_accuracy(y_pred, batch_dict['y_target'])
running_acc += (acc_batch - running_acc) / (batch_index + 1)
batch_generator.set_postfix(val_acc=running_acc,
val_loss=running_loss)
train_state['val_acc'].append(running_acc)
train_state['val_loss'].append(running_loss)
if args.save_model:
if not os.path.isdir(args.model_save_dir):
os.makedirs(args.model_save_dir)
now = datetime.now()
file_path = os.path.join(args.model_save_dir,
datetime.strftime(now, '%y%m%d_%H%M%S.pth'))
torch.save(classifier.state_dict(), file_path)
print(f"Model has been saved: {file_path}")
def predict(input_path, output_path, resources_path):
"""
This is the skeleton of the prediction function.
The predict function will build your model, load the weights from the checkpoint and write a new file (output_path)
with your predictions in the BIES format.
The resources folder should contain everything you need to make the predictions. It is the "resources" folder in your submission.
N.B. DO NOT HARD CODE PATHS IN HERE. Use resource_path instead, otherwise we will not be able to run the code.
:param input_path: the path of the input file to predict.
:param output_path: the path of the output file (where you save your predictions)
:param resources_path: the path of the resources folder containing your model and stuff you might need.
:return: None
"""
print("Loading embeddings...")
emb_path_1grams = resources_path + "/train/embeddings_1grams.utf8"
emb_path_2grams = resources_path + "/train/embeddings_2grams.utf8"
word_to_idx_1grams, idx_to_word_1grams, emb_matrix_1grams = u.read_embeddings(
emb_path_1grams)
word_to_idx_2grams, idx_to_word_2grams, emb_matrix_2grams = u.read_embeddings(
emb_path_2grams)
labels_to_idx, idx_to_labels = u.get_label_dictionaries()
print("Done.")
tf.reset_default_graph()
x_1grams, x_2grams, y, \
keep_pr, recurrent_keep_pr, \
lengths, train, \
loss, preds = m.get_layered_model(pretrained_emb_1grams=emb_matrix_1grams,
pretrained_emb_2grams=emb_matrix_2grams,
hidden_size=96,
layers=1,
y_size=len(labels_to_idx),
learning_rate=0.005)
model_path = resources_path + "/2layers_model_cityu/base_model.ckpt"
print("Loading model saved in path: %s" % model_path)
saver = tf.train.Saver()
with tf.Session() as sess:
saver.restore(sess, model_path)
with open(output_path, mode='w', encoding='utf-8') as preds_file:
pass
print("\nGenerating predictions...")
predictions = []
with open(output_path, mode='a', encoding='utf-8') as preds_file:
for batch_inputs, \
batch_labels, \
batch_lengths in u.generate_batches(dataset_input=input_path,
dataset_label="",
batch_size=32,
label_to_idx=labels_to_idx,
ngram_features=[1, 2],
word_to_idx=[word_to_idx_1grams, word_to_idx_2grams],
to_shuffle=False,
testing=True):
preds_val = sess.run(
[preds],
feed_dict={
x_1grams: batch_inputs[0],
x_2grams: batch_inputs[1],
lengths: batch_lengths,
keep_pr: 1.0,
recurrent_keep_pr: 1.0
})
for p in preds_val[0]:
p = p[1:np.count_nonzero(p) - 1]
p = p.tolist()
# default to "S" if some special tag (either '-' or '<PAD>') is predicted
p = [
idx_to_labels[c] if c > 1 else idx_to_labels[5]
for c in p
]
predictions.append(p)
if len(predictions) == 128:
preds_file.writelines("%s\n" % ''.join(p)
for p in predictions)
predictions = []
if len(predictions) > 0:
preds_file.writelines("%s\n" % ''.join(p) for p in predictions)
print("Done.\nYour predictions have been stored in path: %s" % output_path)
