def train(data_path, model_file):
data_loader = DataLoader(data_path)
data_loader()
weights = load_weights(model_file) if model_file else None
classifier = model(data_loader.train_x,
data_loader.train_y,
data_loader.valid_x,
data_loader.valid_y,
threshold=0.4,
weights=weights,
learning_rate=0.003)
classifier(epoch=10000)
save_model(classifier)
def train(config):
mnist = mnist_data.read_data_sets(config.data_dir, one_hot=True)
x = tf.placeholder(tf.float32, [None, 784])
label = tf.placeholder(tf.float32, [None, 10])
pred_y = model(x)
with tf.name_scope('loss'):
cross_entropy = tf.nn.softmax_cross_entropy_with_logits(labels=label,
logits=pred_y)
cross_entropy = tf.reduce_mean(cross_entropy)
tf.summary.scalar('loss', cross_entropy)
with tf.name_scope('adam_optimizer'):
train_step = tf.train.AdamOptimizer(config.lr).minimize(cross_entropy)
with tf.name_scope('accuracy'):
correct_prediction = tf.equal(tf.argmax(pred_y, 1),
tf.argmax(label, 1))
correct_prediction = tf.cast(correct_prediction, tf.float32)
accuracy = tf.reduce_mean(correct_prediction)
tf.summary.scalar('accuracy', accuracy)
with tf.Session(config=tf.ConfigProto(gpu_options=gpu_options)) as sess:
sess.run(tf.global_variables_initializer())
# merged = tf.summary.merge_all()
# writer = tf.summary.FileWriter("logs/", sess.graph)
#sess.run(tf.global_variables_initializer())
for i in range(config.epoches):
loss = 0
for j in range(int(train_data_size / config.batch_size)):
batch = mnist.train.next_batch(config.batch_size)
train_step.run(feed_dict={x: batch[0], label: batch[1]})
loss += sess.run(
cross_entropy, feed_dict={
x: batch[0],
label: batch[1]
}
) #sess.run(tf.reduce_mean(sess.run(cross_entropy, feed_dict={x: batch[0], label: batch[1]})))
if j % 5 == 0:
train_accuracy = accuracy.eval(feed_dict={
x: batch[0],
label: batch[1]
})
print(
'[Epoch: %d Sample: %d] training accuracy: %g loss: %s'
% (i,
(j + 1) * config.batch_size, train_accuracy, loss /
(j + 1)))
train_accuracy = accuracy.eval(feed_dict={
x: mnist.test.images,
label: mnist.test.labels
})
print("[Epoch: %s] Test Accuracy: %s" % (i + 1, train_accuracy))
def train_val(X_trainval, y_trainval, temp_dir, current_K, test_count,
hyper_count):
# constructing the validataion set by stratified cross-validataion
skf_train_val = StratifiedKFold(n_splits=VAL_FOLD,
random_state=RANDOM_STATE,
shuffle=True)
skf_train_val.get_n_splits(X_trainval, y_trainval)
fold_count = 1
all_train_acc = []
all_train_loss = []
all_val_loss = []
all_val_acc = []
all_val_f1 = []
all_val_precision = []
all_val_recall = []
all_num_train = []
all_num_val = []
all_num_under_val = []
for train_index, val_index in skf_train_val.split(X_trainval, y_trainval):
X_train = X_trainval.iloc[train_index]
y_train = y_trainval.iloc[train_index]
X_val = X_trainval.iloc[val_index]
y_val = y_trainval.iloc[val_index]
# undersampling val set by resampling
sample_val = pd.concat([y_val, X_val], axis=1)
num_class0, num_class1 = y_val.value_counts()
all_num_val.append([num_class0, num_class1])
# calculating the required US samples
num_us_instance = num_class0 - num_class1
under_sample_val = undersampling(sample_val, num_us_instance)
num_class0, num_class1 = under_sample_val['y'].value_counts()
all_num_under_val.append([num_class0, num_class1])
# pre-sampling undersampled val data by pretrained BERT with PCA
print("-" * 70)
print("START PROCESSING VAL SET")
under_sample_val_emb = preprocessor(under_sample_val, MAX_LENGTH,
PCA_DIMEN)
under_sample_val_emb = to_array_list(under_sample_val_emb)
# saving undersampled val data
under_val_path = temp_dir + '/sample_val_under_emb.tsv'
under_sample_val_emb.to_csv(under_val_path,
sep='\t',
encoding="utf-8",
index=False,
header=False)
del X_val, y_val
del sample_val
del under_sample_val
del under_sample_val_emb
# preprocessing train data
print("-" * 70)
print("START PROCESSING TRAIN SET")
sample_train = pd.concat([y_train, X_train], axis=1)
# pre-sampling train data by pretrained BERT with PCA
sample_train_emb = preprocessor(sample_train, MAX_LENGTH, PCA_DIMEN)
# performing undersampling by NearMiss
print("-" * 70)
print("START SAMPLING TRAIN SET [{}] ".format(fold_count))
start_time = time.time()
sample_train_nearmiss = nearmiss(sample_train_emb, current_K)
end_time = time.time()
print("FINISH SAMPLING TRAIN SET [{}]: {}".format(
fold_count, (end_time - start_time)))
# saving undersampled train data
train_path = temp_dir + '/sample_train_nearmiss.tsv'
sample_train_nearmiss.to_csv(train_path,
sep='\t',
encoding="utf-8",
index=False,
header=False)
num_class0, num_class1 = sample_train_nearmiss['y'].value_counts()
all_num_train.append([num_class0, num_class1])
del X_train, y_train, sample_train
# processing to model classifier (preceptron or KNN)
history = model(train_path, under_val_path, test_count, \
hyper_count, fold_count, model='KNN', test=False)
# calculating average train loss and accuracy
all_train_loss.append(history['train_loss'])
all_train_acc.append(history['train_acc'])
# calculating average val loss, accuracy, F1, precision and recall
all_val_loss.append(history['val_loss'])
all_val_acc.append(history['val_acc'])
all_val_f1.append(history['val_f1'])
all_val_precision.append(history['val_precision'])
all_val_recall.append(history['val_recall'])
# increamenting the fold index, and repeat above process
fold_count = fold_count + 1
# return back the ICV log
results = {
'final_all_train_acc': all_train_acc,
'final_all_train_loss': all_train_loss,
'final_all_val_loss': all_val_loss,
'final_all_val_acc': all_val_acc,
'final_all_val_f1': all_val_f1,
'final_all_val_precision': all_val_precision,
'final_all_val_recall': all_val_recall,
'final_avg_val_loss': np.mean(all_val_loss),
'train_distribution': all_num_train,
'val_distribution': all_num_val,
'under val val_distribution': all_num_under_val
}
return results
print("FINISH SAMPLING TRAINVAL SET [{}]: {}".format(
test_count, (end_time - start_time)))
# saving undersampled trainval data
trainval_path = temp_dir + '/sample_trainval_nearmiss.tsv'
sample_trainval_nearmiss.to_csv(trainval_path,
sep='\t',
encoding="utf-8",
index=False,
header=False)
num_class0, num_class1 = sample_trainval_nearmiss['y'].value_counts()
all_num_trainval.append([num_class0, num_class1])
# processing to model classifier (preceptron or KNN)
history = model(trainval_path, under_test_path, test_count, \
best_hyper, best_fold, model='Perceptron', test=True)
# calculating average trainval loss and accuracy
train_loss.append(history['train_loss'])
train_acc.append(history['train_acc'])
# calculating average test loss accuracy, F1, precision and recall
test_loss.append(history['val_loss'])
test_acc.append(history['val_acc'])
test_f1.append(history['val_f1'])
test_precision.append(history['val_precision'])
test_recall.append(history['val_recall'])
test_count = test_count + 1
# creating the resulted dict for saving as json file
default=512,
help="height of images in set")
parser.add_argument("--classifier",
action="store_true",
help="use classifier?")
args = parser.parse_args()
return args
if __name__ == "__main__":
args = get_args()
print(args)
classifier = model()
classifier.load_weights("model.h5")
f = open(args.output_file, "w")
for i, item in enumerate(os.listdir(args.directory)):
img = cv2.imread(os.path.join(args.directory, item))
max_width = img.shape[1] - args.width
max_height = img.shape[0] - args.height
coords = []
flag = True
while flag:
widths = np.random.randint(max_width, size=2 * args.count)
heights = np.random.randint(max_height, size=2 * args.count)
batch = []
for e in range(args.count * 2):
image = img[heights[e]:heights[e] + args.height,
def main():
mo = classifier.model('img/pear.jpg')
def distributed_model(config):
tf.app.flags.DEFINE_string("job_name", "", "Enter 'ps' or 'worker' ")
tf.app.flags.DEFINE_integer("task_index", 0,
"Index of the task within the jobs")
tf.app.flags.DEFINE_bool("async", True, "Async or Sync Train")
tf.app.flags.DEFINE_string("ps_hosts", "",
"Comma-separated list of hostname:port pairs")
tf.app.flags.DEFINE_string("worker_hosts", "",
"Comma-separated list of hostname:port pairs")
tf.app.flags.DEFINE_string("data_dir", "./data/", "Data directory")
FLAGS = tf.app.flags.FLAGS
ps_hosts = FLAGS.ps_hosts.split(",")
worker_hosts = FLAGS.worker_hosts.split(",")
mnist = mnist_data.read_data_sets(FLAGS.data_dir, one_hot=True)
cluster = tf.train.ClusterSpec({"worker": worker_hosts, "ps": ps_hosts})
server = tf.train.Server(cluster,
job_name=FLAGS.job_name,
task_index=FLAGS.task_index)
if FLAGS.job_name == 'ps':
server.join()
else:
with tf.device(
tf.train.replica_device_setter(
worker_device='/job:worker/task:%d' % FLAGS.task_index,
cluster=cluster)):
global_step = tf.Variable(0, name="global_step", trainable=False)
x = tf.placeholder(tf.float32, [None, 784])
label = tf.placeholder(tf.float32, [None, 10])
pred_y = model(x)
with tf.name_scope('loss'):
cross_entropy = tf.nn.softmax_cross_entropy_with_logits(
labels=label, logits=pred_y)
cross_entropy = tf.reduce_mean(cross_entropy)
tf.summary.scalar('loss', cross_entropy)
with tf.name_scope('adam_optimizer'):
optimizer = tf.train.AdamOptimizer(config.lr)
with tf.name_scope('accuracy'):
correct_prediction = tf.equal(tf.argmax(pred_y, 1),
tf.argmax(label, 1))
correct_prediction = tf.cast(correct_prediction, tf.float32)
accuracy = tf.reduce_mean(correct_prediction)
tf.summary.scalar('accuracy', accuracy)
with tf.name_scope('grads_and_vars'):
grads_and_vars = optimizer.compute_gradients(cross_entropy)
if FLAGS. async:
# 异步模式
train_op = optimizer.apply_gradients(grads_and_vars)
else:
rep_op = tf.train.SyncReplicasOptimizer(
optimizer,
replicas_to_aggregate=len(worker_hosts),
total_num_replicas=len(worker_hosts),
use_locking=True)
train_op = rep_op.apply_gradients(grads_and_vars,
global_step=global_step)
init_token_op = rep_op.get_init_tokens_op()
chief_queue_runner = rep_op.get_chief_queue_runner()
init_op = tf.global_variables_initializer()
saver = tf.train.Saver()
summary_op = tf.summary.merge_all()
sv = tf.train.Supervisor(is_chief=(FLAGS.task_index == 0),
logdir='./summary_log/',
init_op=init_op,
summary_op=None,
saver=saver,
global_step=global_step,
save_model_secs=60)
summary_writer = tf.summary.FileWriter('./summary_log/')
with sv.prepare_or_wait_for_session(server.target) as sess:
if FLAGS.task_index == 0 and not FLAGS. async:
sv.start_queue_runners(sess, [chief_queue_runner])
sess.run(init_token_op)
for i in xrange(config.epoches):
loss = 0
for j in xrange(train_data_size / config.batch_size):
batch = mnist.train.next_batch(config.batch_size)
sess.run(train_op,
feed_dict={
x: batch[0],
label: batch[1]
})
loss += sess.run(cross_entropy,
feed_dict={
x: batch[0],
label: batch[1]
})
if j % 5 == 0:
train_accuracy = accuracy.eval(session=sess,
feed_dict={
x: batch[0],
label: batch[1]
})
print(
'[Epoch: %d Sample: %d] training accuracy: %g loss: %s'
% (i, (j + 1) * config.batch_size,
train_accuracy, loss / (j + 1)))
summary = sess.run(summary_op)
summary_writer.add_summary(summary, global_step)
train_accuracy = accuracy.eval(session=sess,
feed_dict={
x: mnist.test.images,
label: mnist.test.labels
})
print("[Epoch: %s] Test Accuracy: %s" %
(i + 1, train_accuracy))
sv.stop()