def train(model):
history_score = []
for i in range(num_round):
fetches = [model.optimizer, model.loss]
if batch_size > 0:
ls = []
for j in range(train_size / batch_size + 1):
X_i, y_i = utils.slice(train_data, j * batch_size, batch_size)
_, l = model.run(fetches, X_i, y_i)
ls.append(l)
elif batch_size == -1:
X_i, y_i = utils.slice(train_data)
_, l = model.run(fetches, X_i, y_i)
ls = [l]
train_preds = model.run(model.y_prob, utils.slice(train_data)[0])
test_preds = model.run(model.y_prob, utils.slice(test_data)[0])
train_score = roc_auc_score(train_data[1], train_preds)
test_score = roc_auc_score(test_data[1], test_preds)
print('[%d]\tloss (with l2 norm):%f\ttrain-auc: %f\teval-auc: %f' %
(i, np.mean(ls), train_score, test_score))
history_score.append(test_score)
if i > min_round and i > early_stop_round:
if np.argmax(
history_score) == i - early_stop_round and history_score[
-1] - history_score[-1 * early_stop_round] < 1e-5:
print('early stop\nbest iteration:\n[%d]\teval-auc: %f' %
(np.argmax(history_score), np.max(history_score)))
break
def train(model):
history_score = []
for i in range(num_round):
fetches = [model.optimizer, model.loss]
# if model.model_file is not None:
# model.saver.restore(model.sess,model.model_file)
if batch_size > 0:
ls = []
print('[%d]\ttraining...' % i)
for j in range(iters):
X_i, y_i = utils.slice(train_data, j * batch_size, batch_size)
_, l = model.run(fetches, X_i, y_i)
ls.append(l)
# if(j == 0):
# print('---------------------------')
# print(X_i[0])
# print('---------------------------')
elif batch_size == -1:
X_i, y_i = utils.slice(train_data)
print('xi.shape', X_i.shape)
print('yi.shape', y_i.shape)
_, l = model.run(fetches, X_i, y_i)
ls = [l]
# model.saver.save(model.sess,'./Save/train.ckpt')
train_preds = []
print('[%d]\tevaluating...' % i)
for j in range(int(train_size / 10000 + 1)):
X_i, _ = utils.slice(train_data, j * 10000, 10000)
preds = model.run(model.y_prob, X_i, mode='test')
train_preds.extend(preds)
test_preds = []
for j in range(int(test_size / 10000 + 1)):
X_i, _ = utils.slice(test_data, j * 10000, 10000)
preds = model.run(model.y_prob, X_i, mode='test')
test_preds.extend(preds)
train_score = roc_auc_score(train_data[1], train_preds)
test_score = roc_auc_score(test_data[1], test_preds)
print('[%d]\tloss (with l2 norm):%f\ttrain-auc: %f\teval-auc: %f' %
(i, np.mean(ls), train_score, test_score))
history_score.append(test_score)
if i > min_round and i > early_stop_round:
if np.argmax(
history_score) == i - early_stop_round and history_score[
-1] - history_score[-1 * early_stop_round] < 1e-5:
print('early stop\nbest iteration:\n[%d]\teval-auc: %f' %
(np.argmax(history_score), np.max(history_score)))
break
# var_map = {}
# for name, var in model.vars.items():
# print(name, var)
# ans = model.run(var)
# var_map[name] = ans
# print(var_map[name])
if algo == 'fm':
model.dump(pkl_path)
def __splitScene(self, scene, layer, offset, size):
# self.__logger.debug('splitting scene...')
ret = {}
bn = self.__batch_n
bh = self.__batch_h
bw = self.__batch_w
bc = self.__batch_c
sfeatures = self.__sfeatures
ifeatures = self.__ifeatures
ret[1] = ret[3] = np.zeros([1, bh, bw, bc])
# make sure ih and iw's value
scene[1] = np.reshape(scene[1], [-1, self.__ih, self.__iw, sfeatures])
scene[3] = np.reshape(scene[3], [-1, self.__ih, self.__iw, sfeatures])
sh = offset[0]
sw = offset[1]
cols = 3
ret[0] = utils.slice(scene[0], [0, sh, sw, 0],
[1, bh, bw, cols]) # img-0
ret[1][:, :, :, :sfeatures] = utils.slice(
scene[1], [layer, sh, sw, 0], [1, bh, bw, sfeatures]) # txt-0
ret[1][:, :, :,
sfeatures + 0:sfeatures + 1] = utils.getLuminance(ret[0])
ret[1][:, :, :,
sfeatures + 1:sfeatures + 2] = utils.getMagnitude(ret[0])
ret[2] = utils.slice(scene[2], [0, sh, sw, 0],
[1, bh, bw, cols]) # img-1
ret[3][:, :, :, :sfeatures] = utils.slice(
scene[3], [layer, sh, sw, 0], [1, bh, bw, sfeatures]) # txt-1
ret[3][:, :, :,
sfeatures + 0:sfeatures + 1] = utils.getLuminance(ret[2])
ret[3][:, :, :,
sfeatures + 1:sfeatures + 2] = utils.getMagnitude(ret[2])
ret[4] = utils.slice(scene[4], [0, sh, sw, 0],
[1, bh, bw, cols]) # truth
# preprocess data
# ret[1] = self.__Sigmoid(ret[1])
# ret[3] = self.__Sigmoid(ret[3])
# base = np.max(ret[1][:, :, :, 3]) - np.min(ret[1][:, :, :, 3])
# if base == 0: base = np.sum(ret[1][:, :, :, 3])
# ret[1][:, :, :, 3] = (ret[1][:, :, :, 3] - np.min(ret[1][:, :, :, 3])) / (base)
# base = np.max(ret[1][:, :, :, 7]) - np.min(ret[1][:, :, :, 7])
# if base == 0: base = np.sum(ret[1][:, :, :, 7])
# ret[1][:, :, :, 7] = (ret[1][:, :, :, 7] - np.min(ret[1][:, :, :, 7])) / (base)
# base = np.max(ret[3][:, :, :, 3]) - np.min(ret[3][:, :, :, 3])
# if base == 0: base = np.sum(ret[3][:, :, :, 3])
# ret[3][:, :, :, 3] = (ret[3][:, :, :, 3] - np.min(ret[3][:, :, :, 3])) / (base)
# base = np.max(ret[3][:, :, :, 7]) - np.min(ret[3][:, :, :, 7])
# if base == 0: base = np.sum(ret[3][:, :, :, 7])
# ret[3][:, :, :, 7] = (ret[3][:, :, :, 7] - np.min(ret[3][:, :, :, 7])) / (base)
return ret
def cut():
# utils.slice('organized_sound/wav/british/bo156.wav', 'test.wav', 0, 3000)
base_dir = 'organized_sound/wav/'
language_dirs = ['american', 'british']
for language in language_dirs:
lang_dir_path = base_dir + language + '/'
for filename in os.listdir(base_dir + language):
if filename.endswith('.wav'):
filepath = lang_dir_path + filename
outpath = lang_dir_path + 's_' + filename
infile = wave.open(filepath)
utils.slice(infile, outpath, 0, 3000)
def train(model):
history_score = []
for i in range(num_round):
fetches = [model.optimizer, model.loss]
if batch_size > 0:
ls = []
bar = progressbar.ProgressBar()
print('[%d]\ttraining...' % i)
for j in bar(range(int(train_size / batch_size + 1))):
X_i, y_i = utils.slice(train_data, j * batch_size, batch_size)
_, l = model.run(fetches, X_i, y_i)
ls.append(l)
elif batch_size == -1:
X_i, y_i = utils.slice(train_data)
_, l = model.run(fetches, X_i, y_i)
ls = [l]
train_preds = []
print('[%d]\tevaluating...' % i)
bar = progressbar.ProgressBar()
for j in bar(range(int(train_size / 10000 + 1))):
X_i, _ = utils.slice(train_data, j * 10000, 10000)
preds = model.run(model.y_prob, X_i, mode='test')
train_preds.extend(preds)
test_preds = []
bar = progressbar.ProgressBar()
for j in bar(range(int(test_size / 10000 + 1))):
X_i, _ = utils.slice(test_data, j * 10000, 10000)
preds = model.run(model.y_prob, X_i, mode='test')
test_preds.extend(preds)
train_score = roc_auc_score(train_data[1], train_preds)
train_loss = log_loss(train_data[1], train_preds)
test_score = roc_auc_score(test_data[1], test_preds)
test_loss = log_loss(test_data[1], test_preds)
print('[%d]\tloss (with l2 norm):%f\ttrain-auc: %f\teval-auc: %f' %
(i, np.mean(ls), train_score, test_score))
print('train_loss:{0}\ttest_loss:{1}'.format(train_loss, test_loss))
history_score.append(test_score)
if i > min_round and i > early_stop_round:
if np.argmax(
history_score) == i - early_stop_round and history_score[
-1] - history_score[-1 * early_stop_round] < 1e-5:
print('early stop\nbest iteration:\n[%d]\teval-auc: %f' %
(np.argmax(history_score), np.max(history_score)))
break
def train(model):
history_score = []
for i in range(num_round):
fetches = [model.optimizer, model.loss]
#if model.model_file is not None:
# model.saver.restore(model.sess,model.model_file)
if batch_size > 0:
ls = []
print('[%d]\ttraining...' % i)
for j in range(int(train_size / batch_size + 1)):
X_i, y_i = utils.slice(train_data, j * batch_size, batch_size)
_, l = model.run(fetches, X_i, y_i)
ls.append(l)
elif batch_size == -1:
X_i, y_i = utils.slice(train_data)
print(X_i.shape)
print(y_i.shape)
_, l = model.run(fetches, X_i, y_i)
ls = [l]
#model.saver.save(model.sess,'./Save/train.ckpt')
train_preds = []
print('[%d]\tevaluating...' % i)
for j in range(int(train_size / 10000 + 1)):
X_i, _ = utils.slice(train_data, j * 10000, 10000)
preds = model.run(model.y_prob, X_i, mode='test')
train_preds.extend(preds)
test_preds = []
for j in range(int(test_size / 10000 + 1)):
X_i, _ = utils.slice(test_data, j * 10000, 10000)
preds = model.run(model.y_prob, X_i, mode='test')
test_preds.extend(preds)
train_score = roc_auc_score(train_data[1], train_preds)
test_score = roc_auc_score(test_data[1], test_preds)
#for i in train_preds:
# print(i)
print('[%d]\tloss (with l2 norm):%f\ttrain-auc: %f\teval-auc: %f' %
(i, np.mean(ls), train_score, test_score))
history_score.append(test_score)
if i > min_round and i > early_stop_round:
if np.argmax(
history_score) == i - early_stop_round and history_score[
-1] - history_score[-1 * early_stop_round] < 1e-5:
print('early stop\nbest iteration:\n[%d]\teval-auc: %f' %
(np.argmax(history_score), np.max(history_score)))
break
def step(self, rxSym, H = None, method = 'MMSE'):
noise = utils.cplxRandn(rxSym.shape, self.N0)
rx = rxSym + noise
if method == 'MMSE':
assert (not H is None)
M = np.dot(utils.hermitian(H), np.linalg.inv(np.dot(H, utils.hermitian(H)) + np.eye(H.shape[0]) * self.N0))
est = np.dot(M, rx)
return utils.slice(est, self.modSyms)
def train(model):
history_score = []
start_time = time.time()
print 'epochs\tloss\ttrain-auc\teval-auc\ttime'
sys.stdout.flush()
for i in range(num_round):
fetches = [model.optimizer, model.loss]
if batch_size > 0:
ls = []
for j in range(train_size / batch_size + 1):
X_i, y_i = utils.slice(train_data, j * batch_size, batch_size)
_, l = model.run(fetches, X_i, y_i)
ls.append(l)
elif batch_size == -1:
X_i, y_i = utils.slice(train_data)
_, l = model.run(fetches, X_i, y_i)
ls = [l]
lst_train_pred = []
lst_test_pred = []
if batch_size > 0:
for j in range(train_size / batch_size + 1):
X_i, y_i = utils.slice(train_data, j * batch_size, batch_size)
#X_i = utils.libsvm_2_coo(X_i, (len(X_i), input_dim)).tocsr()
_train_preds = model.run(model.y_prob, X_i)
lst_train_pred.append(_train_preds)
for j in range(test_size / batch_size + 1):
X_i, y_i = utils.slice(test_data, j * batch_size, batch_size)
#X_i = utils.libsvm_2_coo(X_i, (len(X_i), input_dim)).tocsr()
_test_preds = model.run(model.y_prob, X_i)
lst_test_pred.append(_test_preds)
train_preds = np.concatenate(lst_train_pred)
test_preds = np.concatenate(lst_test_pred)
train_score = roc_auc_score(train_data[1], train_preds)
test_score = roc_auc_score(test_data[1], test_preds)
print '%d\t%f\t%f\t%f\t%f' % (i, np.mean(ls), train_score, test_score, time.time() - start_time)
sys.stdout.flush()
history_score.append(test_score)
if i > min_round and i > early_stop_round:
if np.argmax(history_score) == i - early_stop_round and history_score[-1] - history_score[
-1 * early_stop_round] < 1e-5:
print 'early stop\nbest iteration:\n[%d]\teval-auc: %f' % (
np.argmax(history_score), np.max(history_score))
sys.stdout.flush()
break
def train(model):
history_score = []
for i in range(num_round):
fetches = [model.optimizer, model.loss]
if batch_size > 0:
ls = []
bar = progressbar.ProgressBar()
print('[%d]\ttraining...' % i)
for j in bar(range(int(train_size / batch_size + 1))):
X_i, y_i = utils.slice(train_data, j * batch_size, batch_size)
_, l = model.run(fetches, X_i, y_i)
ls.append(l)
elif batch_size == -1:
X_i, y_i = utils.slice(train_data)
_, l = model.run(fetches, X_i, y_i)
ls = [l]
train_preds = []
print('[%d]\tevaluating...' % i)
bar = progressbar.ProgressBar()
for j in bar(range(int(train_size / 10000 + 1))):
X_i, _ = utils.slice(train_data, j * 10000, 10000)
preds = model.run(model.y_prob, X_i, mode='test')
train_preds.extend(preds)
test_preds = []
bar = progressbar.ProgressBar()
for j in bar(range(int(test_size / 10000 + 1))):
X_i, _ = utils.slice(test_data, j * 10000, 10000)
preds = model.run(model.y_prob, X_i, mode='test')
test_preds.extend(preds)
train_score = roc_auc_score(train_data[1], train_preds)
train_loss = log_loss(train_data[1], train_preds)
test_score = roc_auc_score(test_data[1], test_preds)
test_loss = log_loss(test_data[1], test_preds)
print('[%d]\tloss (with l2 norm):%f\ttrain-auc: %f\teval-auc: %f' % (i, np.mean(ls), train_score, test_score))
print('train_loss:{0}\ttest_loss:{1}'.format(train_loss, test_loss))
history_score.append(test_score)
if i > min_round and i > early_stop_round:
if np.argmax(history_score) == i - early_stop_round and history_score[-1] - history_score[
-1 * early_stop_round] < 1e-5:
print('early stop\nbest iteration:\n[%d]\teval-auc: %f' % (
np.argmax(history_score), np.max(history_score)))
break
'kernel_l2': 0,
'random_seed': 0
}
model = PNN1(**pnn1_params)
elif algo == 'pnn2':
pnn2_params = {
'layer_sizes': [field_sizes, 10, 1],
'layer_acts': ['tanh', 'none'],
'drop_out': [0, 0],
'opt_algo': 'gd',
'learning_rate': 0.01,
'layer_l2': [0, 0],
'kernel_l2': 0,
'random_seed': 0
}
model = PNN2(**pnn2_params)
if algo in {'fnn', 'ccpm', 'pnn1', 'pnn2'}:
train_data = utils.split_data(train_data)
test_data = utils.split_data(test_data)
train(model)
X_i, y_i = utils.slice(train_data, 0, 100)
fetches = [model.tmp1, model.tmp2]
tmp1, tmp2 = model.run(fetches, X_i, y_i)
print(tmp1.shape)
print(tmp2.shape)
def train(model, name):
global batch_size
global time_run
global time_process
history_score = []
start_time = time.time()
print 'epochs\tloss\ttrain-auc\teval-auc\ttime'
sys.stdout.flush()
for i in range(num_round):
fetches = [model.optimizer, model.loss]
if batch_size > 0:
ls = []
for j in range(train_size / batch_size + 1):
start_process = time.time()
X_i, y_i = utils.slice(train_data, j * batch_size, batch_size)
time_process += time.time() - start_process
start_run = time.time()
_, l = model.run(fetches, X_i, y_i)
time_run += time.time() - start_run
ls.append(l)
'''
f = open(train_file, 'r')
lst_lines = []
for line in f:
if len(lst_lines) < batch_size:
lst_lines.append(line)
else:
X_i, y_i = utils.slice(utils.process_lines(lst_lines, name), 0, -1)
print 'type(X_i)', type(X_i)
print 'type(X_i[0])', type(X_i[0])
_, l = model.run(fetches, X_i, y_i)
ls.append(l)
lst_lines = [line]
f.close()
if len(lst_lines) > 0:
X_i, y_i = utils.slice(utils.process_lines(lst_lines, name), 0, -1)
_, l = model.run(fetches, X_i, y_i)
ls.append(l)
'''
'''
while True:
lines_gen = list(islice(f, batch_size * bb))
if not lines_gen:
break
for ib in range(bb):
X_i, y_i = utils.slice(utils.process_lines(lines_gen[batch_size * ib : batch_size * (ib+1)], name), 0, -1)
_, l = model.run(fetches, X_i, y_i)
ls.append(l)
'''
elif batch_size == -1:
pass
"""
X_i, y_i = utils.slice(train_data)
_, l = model.run(fetches, X_i, y_i)
ls = [l]
"""
lst_train_pred = []
lst_test_pred = []
if batch_size > 0:
f = open(train_file, 'r')
lst_lines = []
for line in f:
if len(lst_lines) < batch_size:
lst_lines.append(line)
else:
X_i, y_i = utils.slice(
utils.process_lines(lst_lines, name), 0, -1)
_train_preds = model.run(model.y_prob, X_i)
lst_train_pred.append(_train_preds)
lst_lines = [line]
f.close()
if len(lst_lines) > 0:
X_i, y_i = utils.slice(utils.process_lines(lst_lines, name), 0,
-1)
_train_preds = model.run(model.y_prob, X_i)
lst_train_pred.append(_train_preds)
'''
while True:
lines_gen = list(islice(f, batch_size * bb))
if not lines_gen:
break
for ib in range(bb):
X_i, y_i = utils.slice(utils.process_lines(lines_gen[batch_size * ib : batch_size * (ib+1)], name), 0, -1)
_train_preds = model.run(model.y_prob, X_i)
lst_train_pred.append(_train_preds)
'''
"""
for j in range(train_size / batch_size + 1):
X_i, y_i = utils.slice(train_data, j * batch_size, batch_size)
#X_i = utils.libsvm_2_coo(X_i, (len(X_i), input_dim)).tocsr()
_train_preds = model.run(model.y_prob, X_i)
lst_train_pred.append(_train_preds)
"""
f = open(test_file, 'r')
lst_lines = []
for line in f:
if len(lst_lines) < batch_size:
lst_lines.append(line)
else:
X_i, y_i = utils.slice(
utils.process_lines(lst_lines, name), 0, -1)
_test_preds = model.run(model.y_prob, X_i)
lst_test_pred.append(_test_preds)
lst_lines = [line]
f.close()
if len(lst_lines) > 0:
X_i, y_i = utils.slice(utils.process_lines(lst_lines, name), 0,
-1)
_test_preds = model.run(model.y_prob, X_i)
lst_test_pred.append(_test_preds)
'''
while True:
lines_gen = list(islice(f, batch_size * bb))
if not lines_gen:
break
for ib in range(bb):
X_i, y_i = utils.slice(utils.process_lines(lines_gen[batch_size * ib : batch_size * (ib+1)], name), 0, -1)
_test_preds = model.run(model.y_prob, X_i)
lst_test_pred.append(_test_preds)
'''
"""
for j in range(test_size / batch_size + 1):
X_i, y_i = utils.slice(test_data, j * batch_size, batch_size)
#X_i = utils.libsvm_2_coo(X_i, (len(X_i), input_dim)).tocsr()
_test_preds = model.run(model.y_prob, X_i)
lst_test_pred.append(_test_preds)
"""
train_preds = np.concatenate(lst_train_pred)
test_preds = np.concatenate(lst_test_pred)
train_score = roc_auc_score(train_label, train_preds)
test_score = roc_auc_score(test_label, test_preds)
print '%d\t%f\t%f\t%f\t%f\t%s' % (
i, np.mean(ls), train_score, test_score, time.time() - start_time,
strftime("%Y-%m-%d %H:%M:%S", gmtime()))
print 'time_run', time_run
print 'time_process', time_process
# Save the model to local files
'''
path_model = 'model/' + str(name) + '_epoch_' + str(i)
model.dump(path_model)
d_label_score = {}
d_label_score['label'] = test_label
d_label_score['score'] = test_preds
#path_label_score = 'model/label_score_' + str(name) + '_epoch_' + str(i)
#pkl.dump(d_label_score, open(path_label_score, 'wb'))
sys.stdout.flush()
'''
history_score.append(test_score)
if i > min_round and i > early_stop_round:
i_max = np.argmax(history_score)
# Early stop
if i - i_max >= early_stop_round:
print 'early stop\nbest iteration:\n[%d]\teval-auc: %f' % (
np.argmax(history_score), np.max(history_score))
sys.stdout.flush()
break
'''
T[i] = map(int, filter(lambda c: c != '', T[i].split(' ')))
WIDTH = len(T)
HEIGHT = len(T[0])
log.info('Image size: %sx%s' % (WIDTH, HEIGHT))
if __name__ == "__main__":
from itertools import islice
p = Pool(processes=len(users) / 3)
BLOCK_SIZE = 16
try:
while True:
fetched = utils.fetch()
if not fetched:
log.fatal('Failed to fetch canvas.')
exit(-1)
D = utils.slice(utils.fetch(), WIDTH, HEIGHT)
log.info('Fetched canvas.')
cnt = 0
for idx in p.imap_unordered(utils.draw, utils.diff(D, T,
provider)):
taken[idx] = False
cnt += 1
# for idx in p.imap_unordered(utils.draw, islice(utils.diff(D, T, provider), 0, BLOCK_SIZE)):
# taken[idx] = False
log.info('Rendered %s pixel(s).' % cnt)
if cnt:
time.sleep(0.1)
else:
time.sleep(60)
except:
exc_type, exc_value, exc_traceback = sys.exc_info()
def train(self,
sample_dir,
ckpt_dir='ckpt',
training_epoches=1000000,
batch_size=512):
fig_count = 0
self.sess.run(tf.global_variables_initializer())
train_size = train_data[0].shape[0]
# test_size = test_data[0].shape[0]
for epoch in range(training_epoches):
# update D
for j in range(int(train_size / batch_size + 1)):
X_b, y_b = utils.slice(train_data, j * batch_size, batch_size)
from keras.utils import to_categorical
y_b = to_categorical(y_b, 2)
# X_b, y_b = self.data(batch_size)
self.sess.run(self.D_solver,
feed_dict={
self.X: X_b,
self.y: y_b,
self.z: sample_z(len(y_b), self.z_dim)
})
# update G
k = 1
for _ in range(k):
self.sess.run(self.G_solver,
feed_dict={
self.y: y_b,
self.z: sample_z(len(y_b), self.z_dim)
})
# save img, model. print loss
if epoch % 100 == 0 or epoch < 100:
D_loss_curr = self.sess.run(self.D_loss,
feed_dict={
self.X:
X_b,
self.y:
y_b,
self.z:
sample_z(len(y_b), self.z_dim)
})
G_loss_curr = self.sess.run(self.G_loss,
feed_dict={
self.y:
y_b,
self.z:
sample_z(len(y_b), self.z_dim)
})
print('Iter: {}; D loss: {:.4}; G_loss: {:.4}'.format(
epoch, D_loss_curr, G_loss_curr))
if epoch % 1000 == 0:
y_s = sample_y(16, self.y_dim, fig_count % 10)
samples = self.sess.run(self.G_sample,
feed_dict={
self.y: y_s,
self.z:
sample_z(16, self.z_dim)
})
print(y_s.shape)
print(samples.shape)
def do_inference(hostport, test_data, concurrency, num_tests, batch_size):
"""Tests PredictionService with concurrent-batched requests.
Args:
hostport: Host:port address of the PredictionService.
test_data: The full path to the test data set.
concurrency: Maximum number of concurrent requests.
num_tests: Number of test tensors to use.
batch_size: Number of tests to include in each query
Returns:
The results of the queries
Raises:
IOError: An error occurred processing test data set.
"""
host, port = hostport.split(':')
channel = implementations.insecure_channel(host, int(port))
stub = prediction_service_pb2.beta_create_PredictionService_stub(channel)
result_counter = _ResultCounter(concurrency)
num_field = 15
with open(test_data, 'r') as f:
data = f.read().split('\n')
requests = []
t0 = time.time()
#Create batches of requests
for i in range(num_batches):
data_batch = data[:batch_size]
data = data[batch_size:]
index_list = [[] for _ in range(num_field)]
values_list = [[] for _ in range(num_field)]
for j in range(batch_size):
X, y = utils.slice(
utils.process_lines([data_batch[j]], 'fwfm', INPUT_DIM,
FIELD_OFFSETS), 0, -1)
for idx in range(num_field):
index_list[idx].append(X[idx][0].tolist())
values_list[idx].append(1)
requests.append(predict_pb2.PredictRequest())
requests[i].model_spec.name = 'serve'
requests[i].model_spec.signature_name = 'model'
requests[i].output_filter.append('outputs')
for idx in range(num_field):
requests[i].inputs["field_" + str(idx) + "_values"].CopyFrom(
tf.contrib.util.make_tensor_proto(
values_list[idx],
shape=[len(values_list[idx])],
dtype=tf.int64))
requests[i].inputs["field_" + str(idx) + "_indices"].CopyFrom(
tf.contrib.util.make_tensor_proto(
index_list[idx],
shape=[len(index_list[idx]), 2],
dtype=tf.float32))
requests[i].inputs["field_" + str(idx) +
"_dense_shape"].CopyFrom(
tf.contrib.util.make_tensor_proto(
[batch_size, total_features],
shape=[2],
dtype=tf.int64))
t1 = time.time()
#Query server
for i in range(num_batches):
result_counter.throttle()
result = stub.Predict.future(requests[i],
100.0) # 100 secs timeout
result.add_done_callback(_create_rpc_callback(i, result_counter))
t2 = time.time()
#Synchronize on comleted queries
result_counter.get_complete()
t3 = time.time()
full_results = []
for values in results:
full_results.extend(values)
print("Elapsed time for ", num_tests, " request creations: ",
(t1 - t0))
print("Elapsed time for ", num_batches, " batch submissions: ",
(t2 - t1))
print("Elapsed time for ", num_tests, " inferences: ", (t3 - t1))
return full_results
net.load_state_dict(torch.load(args.model, map_location='cpu'))
print("Using CPU version of the net, this may be very slow")
print("Model loaded !")
dataset = HelioDataset('./data/sidc/SIDC_dataset.csv', 'data/dpd/', 1)
data_loader = DataLoader(dataset)
pred_mask_slices = []
input = None
true_mask = None
for _, obs in enumerate(data_loader):
input = np.array(obs["img"][0])
true_mask = np.array(obs["mask"][0])
obs = slice(obs, args.window, args.window)
for idx in range(0, len(obs['imgs']), args.batch_size):
print("\nPredicting images {0} - {1} ...".format(
idx, idx + args.batch_size))
imgs = obs['imgs'][idx:idx + args.batch_size].float()
masks = predict_img(net=net,
imgs=imgs,
out_threshold=args.mask_threshold,
use_dense_crf=args.crf,
use_gpu=not args.cpu)
pred_mask_slices.extend(masks.squeeze())
pred_mask_slices = np.array(pred_mask_slices)
n = input[0].shape[0] // args.window
def train(model, name):
history_score = []
start_time = time.time()
print 'epochs\tloss\ttrain-auc\teval-auc\ttime'
sys.stdout.flush()
for i in range(num_round):
fetches = [model.optimizer, model.loss]
if batch_size > 0:
ls = []
f = open(train_file, 'r')
while True:
lines_gen = list(islice(f, batch_size * bb))
if not lines_gen:
break
for ib in range(bb):
X_i, y_i = utils.slice(utils.process_lines(lines_gen[batch_size * ib : batch_size * (ib+1)]), 0, -1)
_, l = model.run(fetches, X_i, y_i)
ls.append(l)
elif batch_size == -1:
pass
"""
X_i, y_i = utils.slice(train_data)
_, l = model.run(fetches, X_i, y_i)
ls = [l]
"""
lst_train_pred = []
lst_test_pred = []
if batch_size > 0:
f = open(train_file, 'r')
while True:
lines_gen = list(islice(f, batch_size * bb))
if not lines_gen:
break
for ib in range(bb):
X_i, y_i = utils.slice(utils.process_lines(lines_gen[batch_size * ib : batch_size * (ib+1)]), 0, -1)
_train_preds = model.run(model.y_prob, X_i)
lst_train_pred.append(_train_preds)
"""
for j in range(train_size / batch_size + 1):
X_i, y_i = utils.slice(train_data, j * batch_size, batch_size)
#X_i = utils.libsvm_2_coo(X_i, (len(X_i), input_dim)).tocsr()
_train_preds = model.run(model.y_prob, X_i)
lst_train_pred.append(_train_preds)
"""
f = open(test_file, 'r')
while True:
lines_gen = list(islice(f, batch_size * bb))
if not lines_gen:
break
for ib in range(bb):
X_i, y_i = utils.slice(utils.process_lines(lines_gen[batch_size * ib : batch_size * (ib+1)]), 0, -1)
_test_preds = model.run(model.y_prob, X_i)
lst_test_pred.append(_test_preds)
"""
for j in range(test_size / batch_size + 1):
X_i, y_i = utils.slice(test_data, j * batch_size, batch_size)
#X_i = utils.libsvm_2_coo(X_i, (len(X_i), input_dim)).tocsr()
_test_preds = model.run(model.y_prob, X_i)
lst_test_pred.append(_test_preds)
"""
train_preds = np.concatenate(lst_train_pred)
test_preds = np.concatenate(lst_test_pred)
train_score = roc_auc_score(train_label, train_preds)
test_score = roc_auc_score(test_label, test_preds)
print '%d\t%f\t%f\t%f\t%f\t%s' % (i, np.mean(ls), train_score, test_score, time.time() - start_time, strftime("%Y-%m-%d %H:%M:%S", gmtime()))
path_model = 'model/' + str(name) + '_epoch_' + str(i)
path_label_score = 'model/label_score_' + str(name) + '_epoch_' + str(i)
#model.dump(path_model)
d_label_score = {}
d_label_score['label'] = test_label
d_label_score['score'] = test_preds
#pkl.dump(d_label_score, open(path_label_score, 'wb'))
sys.stdout.flush()
history_score.append(test_score)
if i > min_round and i > early_stop_round:
#if np.argmax(history_score) == i - early_stop_round and history_score[-1] - history_score[
# -1 * early_stop_round] < 1e-5:
i_max = np.argmax(history_score)
if i - i_max >= early_stop_round:
print 'early stop\nbest iteration:\n[%d]\teval-auc: %f' % (
np.argmax(history_score), np.max(history_score))
sys.stdout.flush()
break
def __call__(self, m_, x_, h_, c_, a_, ct_, pctx_):
"""
Each variable is one time slice of the LSTM
m_ - (mask),
x_- (previous word),
h_- (hidden state),
c_- (lstm memory),
a_ - (alpha distribution [eq (5)]),
as_- (sample from alpha dist),
ct_- (context),
pctx_ (projected context),
"""
# attention computation
# [described in equations (4), (5), (6) in
# section "3.1.2 Decoder: Long Short Term Memory Network]
# f_att(a_i, h_t-1) This part is concatenation between a_i and h_t-1
pstate_ = tensor.dot(h_, self.Wd_att)
pctx_ = pctx_ + pstate_[:, None, :]
pctx_list = [pctx_]
pctx_ = tensor.tanh(pctx_)
#equation 4
alpha = tensor.dot(pctx_, self.U_att) + self.c_att
#Eqn 5
alpha = tensor.nnet.softmax(
alpha.reshape([alpha.shape[0], alpha.shape[1]])) # softmax
#Eqn 6
ctx_ = (self.context * alpha[:, :, None]).sum(
1) # current context, aka zt in paper
#selector
sel_ = tensor.nnet.sigmoid(tensor.dot(h_, self.W_sel + self.b_sel))
sel_ = sel_.reshape([sel_.shape[0]])
ctx_ = sel_[:, None] * ctx_
#Accumulation to compute different gate outputs
preact = tensor.dot(h_, self.U)
preact += x_
preact += tensor.dot(ctx_, self.Wc)
dim = self.U.shape[0]
# Recover the activations to the lstm gates
# [equation (1)]
i = utils.slice(preact, 0, dim)
f = utils.slice(preact, 1, dim)
o = utils.slice(preact, 2, dim)
i = tensor.nnet.sigmoid(i)
f = tensor.nnet.sigmoid(f)
o = tensor.nnet.sigmoid(o)
c = tensor.tanh(utils.slice(preact, 3, dim))
# compute the new memory/hidden state
# if the mask is 0, just copy the previous state
#Eqn 2
c = f * c_ + i * c
c = m_[:, None] * c + (1. - m_)[:, None] * c_
#Eqn 3
h = o * tensor.tanh(c)
h = m_[:, None] * h + (1. - m_)[:, None] * h_
return [h, c, alpha, ctx_]
def next_batch(self, seed):
np.random.seed(seed)
assert (self.__isTrain)
self.__logger.debug('Size of Cache:%dM' % self.__cache.getSize())
self.__logger.debug('Eta of Cache:%.2f%%' %
(self.__cache.getEta() * 100.0))
cols = self.__cols
bn = self.__batch_n
bh = self.__batch_h
bw = self.__batch_w
bc = self.__batch_c
sfeatures = self.__sfeatures
# x1 input image,x2 txt features,y ground truth
x1 = np.zeros([bn, bh, bw, cols])
x2 = np.zeros([bn, bh, bw, bc])
y = np.zeros([bn, bh, bw, cols])
# select a scene randomly
id = np.random.randint(0, len(self.__directories))
while self.__directories[id] in self.__abandons:
id = np.random.randint(0, len(self.__directories))
self.__logger.debug(
'something is wrong in this scene,looking for another one...')
# iter each layer of batchs
for n in range(bn):
# get input data
a = b = c = None
imgpath = self.__directories[id] + self.__img_input
txtpath = self.__directories[id] + self.__txt_input
confpath = self.__directories[id] + self.__conf_input
# exit(-1)
# print(imgpath,txtpath,confpath)
isMatch = self.checkMatch(imgpath, txtpath, confpath)
# checkMatch before putting them into Cache
# skip current scene when md5 mismatch
if not isMatch:
n -= 1
self.__abandons.append(self.__directories[id])
self.__logger.warn(
'md5 of scene [%s] mismatch,abandon this one!' %
self.__directories[id])
continue
try:
a = self.__cache.get(self.__directories[id] + self.__img_input)
except:
a = utils.readIMG(self.__directories[id] + self.__img_input)
self.__cache.add(self.__directories[id] + self.__img_input, a)
try:
b = self.__cache.get(self.__directories[id] + self.__txt_input)
except:
b = utils.readTXT(self.__directories[id] + self.__txt_input)
self.__cache.add(self.__directories[id] + self.__txt_input, b)
try:
c = self.__cache.get(self.__directories[id] +
self.__ground_truth)
except:
c = utils.readIMG(self.__directories[id] + self.__ground_truth)
self.__cache.add(self.__directories[id] + self.__ground_truth,
c)
value = {'input': a, 'data': b, 'truth': c}
# some assertions about input data
assert (len(value['input'].shape) == 4)
assert (len(value['truth'].shape) == 4)
assert (value['input'].shape[1] == value['truth'].shape[1])
assert (value['input'].shape[2] == value['truth'].shape[2])
# height & width of the input image
ih = value['input'].shape[1]
iw = value['input'].shape[2]
value['data'] = np.reshape(value['data'], [-1, ih, iw, sfeatures])
# pick a sample pos(left&up corner) randomly
sh = np.random.randint(0, ih - bh)
sw = np.random.randint(0, iw - bw)
sh = sw = 400
# part of input image(x1) & ground truth(y)
px1 = utils.slice(value['input'], [0, sh, sw, 0],
[1, bh, bw, cols])
py = utils.slice(value['truth'], [0, sh, sw, 0], [1, bh, bw, cols])
# select a layer of txt randomly
idx = np.random.randint(0, value['data'].shape[0])
# self.__logger.debug('%d %d %d %d' % (id,idx,sh,sw))
# idx = 0
# part of input txt(px2)
px2 = np.zeros([1, bh, bw, bc])
# [ 0:features]:features stored in txt
px2[:, :, :, :sfeatures] = utils.slice(value['data'],
[idx, sh, sw, 0],
[1, bh, bw, sfeatures])
# [features:]:features extracted from img
assert (self.__ifeatures == 2)
px2[:, :, :, sfeatures + 0:sfeatures + 1] = utils.getLuminance(px1)
px2[:, :, :, sfeatures + 1:sfeatures + 2] = utils.getMagnitude(px1)
# px2[:,:,:,features + 2:features + 5] =
# np.reshape(px1,[bh,bw,cols])
px2 = np.reshape(px2, [-1, bc])
# reshape these parts
x1[n] = np.reshape(px1, [bh, bw, cols])
x2[n] = np.reshape(px2, [bh, bw, bc])
y[n] = np.reshape(py, [bh, bw, cols])
# normalize data before return them
x2 = self.normalize(x2)
# utils.displayImage(x1)
return x1, x2, y
def train(model, name, in_memory = True, flag_MTL = False):
#builder = tf.saved_model.builder.SavedModelBuilder('model')
global batch_size, time_run, time_read, time_process
history_score = []
best_score = -1
best_epoch = -1
start_time = time.time()
print 'epochs\tloss\ttrain-auc\teval-auc\ttime'
sys.stdout.flush()
if in_memory:
train_data = utils.read_data(path_train, INPUT_DIM)
validation_data = utils.read_data(path_validation, INPUT_DIM)
test_data = utils.read_data(path_test, INPUT_DIM)
model_name = name.split('_')[0]
if model_name in set(['lr', 'fm']):
train_data_tmp = utils.split_data(train_data, FIELD_OFFSETS)
validation_data_tmp = utils.split_data(validation_data, FIELD_OFFSETS)
test_data_tmp = utils.split_data(test_data, FIELD_OFFSETS)
else:
train_data = utils.split_data(train_data, FIELD_OFFSETS)
validation_data = utils.split_data(validation_data, FIELD_OFFSETS)
test_data = utils.split_data(test_data, FIELD_OFFSETS)
for i in range(num_round):
fetches = [model.optimizer, model.loss]
if batch_size > 0:
ls = []
if in_memory:
for j in range(train_size / batch_size + 1):
X_i, y_i = utils.slice(train_data, j * batch_size, batch_size)
_, l = model.run(fetches, X_i, y_i)
ls.append(l)
else:
f = open(path_train, 'r')
lst_lines = []
for line in f:
if len(lst_lines) < batch_size:
lst_lines.append(line)
else:
X_i, y_i = utils.slice(utils.process_lines(lst_lines, name, INPUT_DIM, FIELD_OFFSETS), 0, -1) # type of X_i, X_i[0], X_i[0][0] is list, tuple and np.ndarray respectively.
_, l = model.run(fetches, X_i, y_i)
ls.append(l)
lst_lines = [line]
f.close()
if len(lst_lines) > 0:
X_i, y_i = utils.slice(utils.process_lines(lst_lines, name, INPUT_DIM, FIELD_OFFSETS), 0, -1)
_, l = model.run(fetches, X_i, y_i)
ls.append(l)
elif batch_size == -1:
pass
model.dump('model/' + name + '_epoch_' + str(i))
if in_memory:
lst_train_preds = []
lst_validation_preds = []
lst_test_preds = []
for j in range(train_size / batch_size + 1):
X_i, y_i = utils.slice(train_data, j * batch_size, batch_size)
p = model.run(model.y_prob, X_i, y_i)
lst_train_preds.append(p)
for j in range(validation_size / batch_size + 1):
X_i, y_i = utils.slice(validation_data, j * batch_size, batch_size)
p = model.run(model.y_prob, X_i, y_i)
lst_validation_preds.append(p)
for j in range(test_size / batch_size + 1):
X_i, y_i = utils.slice(test_data, j * batch_size, batch_size)
p = model.run(model.y_prob, X_i, y_i)
lst_test_preds.append(p)
train_preds = np.concatenate(lst_train_preds)
validation_preds = np.concatenate(lst_validation_preds)
test_preds = np.concatenate(lst_test_preds)
#train_preds = model.run(model.y_prob, utils.slice(train_data)[0])
#test_preds = model.run(model.y_prob, utils.slice(test_data)[0])
train_score = roc_auc_score(train_data[1], train_preds)
validation_score = roc_auc_score(validation_data[1], validation_preds)
test_score = roc_auc_score(test_data[1], test_preds)
train_score_sum = 0
train_score_weight = 0
validation_score_sum = 0
validation_score_weight = 0
test_score_sum = 0
test_score_weight = 0
#print '[%d]\tloss:%f\ttrain-auc: %f\teval-auc: %f' % (i, np.mean(ls), train_score, test_score)
print '%d\t%f\t%f\t%f\t%f\t%f\t%s' % (i, np.mean(ls), train_score, validation_score, test_score, time.time() - start_time, strftime("%Y-%m-%d %H:%M:%S", gmtime()))
if flag_MTL:
d_index_task_label_pred_train = {}
d_index_task_label_pred_validation = {}
d_index_task_label_pred_test = {}
if model_name in set(['lr', 'fm']):
index_task_train = train_data_tmp[0][-1].indices
index_task_validation = validation_data_tmp[0][-1].indices
index_task_test = test_data_tmp[0][-1].indices
else:
index_task_train = train_data[0][-1].indices
index_task_validation = validation_data[0][-1].indices
index_task_test = test_data[0][-1].indices
for index_tmp in range(len(index_task_train)):
index_task = index_task_train[index_tmp]
d_index_task_label_pred_train.setdefault(index_task, [[],[]])
d_index_task_label_pred_train[index_task][0].append(train_data[1][index_tmp])
d_index_task_label_pred_train[index_task][1].append(train_preds[index_tmp])
for index_task in sorted(list(set(index_task_train))):
auc = roc_auc_score(d_index_task_label_pred_train[index_task][0], d_index_task_label_pred_train[index_task][1])
num_samples = len(d_index_task_label_pred_train[index_task][0])
train_score_sum += auc * num_samples
train_score_weight += num_samples
print 'train, index_type: %d, number of samples: %d, AUC: %f' % (index_task, len(d_index_task_label_pred_train[index_task][0]), auc)
for index_tmp in range(len(index_task_validation)):
index_task = index_task_validation[index_tmp]
d_index_task_label_pred_validation.setdefault(index_task, [[],[]])
d_index_task_label_pred_validation[index_task][0].append(validation_data[1][index_tmp])
d_index_task_label_pred_validation[index_task][1].append(validation_preds[index_tmp])
for index_task in sorted(list(set(index_task_validation))):
auc = roc_auc_score(d_index_task_label_pred_validation[index_task][0], d_index_task_label_pred_validation[index_task][1])
num_samples = len(d_index_task_label_pred_validation[index_task][0])
validation_score_sum += auc * num_samples
validation_score_weight += num_samples
print 'validation, index_type: %d, number of samples: %d, AUC: %f' % (index_task, num_samples, auc)
for index_tmp in range(len(index_task_test)):
index_task = index_task_test[index_tmp]
d_index_task_label_pred_test.setdefault(index_task, [[],[]])
d_index_task_label_pred_test[index_task][0].append(test_data[1][index_tmp])
d_index_task_label_pred_test[index_task][1].append(test_preds[index_tmp])
for index_task in sorted(list(set(index_task_test))):
auc = roc_auc_score(d_index_task_label_pred_test[index_task][0], d_index_task_label_pred_test[index_task][1])
num_samples = len(d_index_task_label_pred_test[index_task][0])
test_score_sum += auc * num_samples
test_score_weight += num_samples
print 'test, index_type: %d, number of samples: %d, AUC: %f' % (index_task, len(d_index_task_label_pred_test[index_task][0]), auc)
weighted_train_score = train_score_sum / train_score_weight
print 'weighted_train_score', weighted_train_score
weighted_validation_score = validation_score_sum / validation_score_weight
print 'weighted_validation_score', weighted_validation_score
weighted_test_score = test_score_sum / test_score_weight
print 'weighted_test_score', weighted_test_score
history_score.append(weighted_validation_score)
if weighted_validation_score < best_score and (i - best_epoch) >= 3:
break
if weighted_validation_score > best_score:
best_score = weighted_validation_score
best_epoch = i
sys.stdout.flush()
else:
lst_train_pred = []
lst_test_pred = []
if batch_size > 0:
f = open(path_train, 'r')
lst_lines = []
for line in f:
if len(lst_lines) < batch_size:
lst_lines.append(line)
else:
X_i, y_i = utils.slice(utils.process_lines(lst_lines, name, INPUT_DIM, FIELD_OFFSETS), 0, -1)
_train_preds = model.run(model.y_prob, X_i)
lst_train_pred.append(_train_preds)
lst_lines = [line]
f.close()
if len(lst_lines) > 0:
X_i, y_i = utils.slice(utils.process_lines(lst_lines, name, INPUT_DIM, FIELD_OFFSETS), 0, -1)
_train_preds = model.run(model.y_prob, X_i)
lst_train_pred.append(_train_preds)
f = open(path_test, 'r')
lst_lines = []
for line in f:
if len(lst_lines) < batch_size:
lst_lines.append(line)
else:
X_i, y_i = utils.slice(utils.process_lines(lst_lines, name, INPUT_DIM, FIELD_OFFSETS), 0, -1)
_test_preds = model.run(model.y_prob, X_i)
lst_test_pred.append(_test_preds)
lst_lines = [line]
f.close()
if len(lst_lines) > 0:
X_i, y_i = utils.slice(utils.process_lines(lst_lines, name, INPUT_DIM, FIELD_OFFSETS), 0, -1)
_test_preds = model.run(model.y_prob, X_i)
lst_test_pred.append(_test_preds)
train_preds = np.concatenate(lst_train_pred)
test_preds = np.concatenate(lst_test_pred)
print 'np.shape(train_preds)', np.shape(train_preds)
train_score = roc_auc_score(train_label, train_preds)
test_score = roc_auc_score(test_label, test_preds)
print '%d\t%f\t%f\t%f\t%f\t%s' % (i, np.mean(ls), train_score, test_score, time.time() - start_time, strftime("%Y-%m-%d %H:%M:%S", gmtime()))
sys.stdout.flush()
'''
