def train(model): history_score = [] for i in range(num_round): fetches = [model.optimizer, model.loss] if batch_size > 0: ls = [] for j in range(train_size / batch_size + 1): X_i, y_i = utils.slice(train_data, j * batch_size, batch_size) _, l = model.run(fetches, X_i, y_i) ls.append(l) elif batch_size == -1: X_i, y_i = utils.slice(train_data) _, l = model.run(fetches, X_i, y_i) ls = [l] train_preds = model.run(model.y_prob, utils.slice(train_data)[0]) test_preds = model.run(model.y_prob, utils.slice(test_data)[0]) train_score = roc_auc_score(train_data[1], train_preds) test_score = roc_auc_score(test_data[1], test_preds) print('[%d]\tloss (with l2 norm):%f\ttrain-auc: %f\teval-auc: %f' % (i, np.mean(ls), train_score, test_score)) history_score.append(test_score) if i > min_round and i > early_stop_round: if np.argmax( history_score) == i - early_stop_round and history_score[ -1] - history_score[-1 * early_stop_round] < 1e-5: print('early stop\nbest iteration:\n[%d]\teval-auc: %f' % (np.argmax(history_score), np.max(history_score))) break
def train(model): history_score = [] for i in range(num_round): fetches = [model.optimizer, model.loss] # if model.model_file is not None: # model.saver.restore(model.sess,model.model_file) if batch_size > 0: ls = [] print('[%d]\ttraining...' % i) for j in range(iters): X_i, y_i = utils.slice(train_data, j * batch_size, batch_size) _, l = model.run(fetches, X_i, y_i) ls.append(l) # if(j == 0): # print('---------------------------') # print(X_i[0]) # print('---------------------------') elif batch_size == -1: X_i, y_i = utils.slice(train_data) print('xi.shape', X_i.shape) print('yi.shape', y_i.shape) _, l = model.run(fetches, X_i, y_i) ls = [l] # model.saver.save(model.sess,'./Save/train.ckpt') train_preds = [] print('[%d]\tevaluating...' % i) for j in range(int(train_size / 10000 + 1)): X_i, _ = utils.slice(train_data, j * 10000, 10000) preds = model.run(model.y_prob, X_i, mode='test') train_preds.extend(preds) test_preds = [] for j in range(int(test_size / 10000 + 1)): X_i, _ = utils.slice(test_data, j * 10000, 10000) preds = model.run(model.y_prob, X_i, mode='test') test_preds.extend(preds) train_score = roc_auc_score(train_data[1], train_preds) test_score = roc_auc_score(test_data[1], test_preds) print('[%d]\tloss (with l2 norm):%f\ttrain-auc: %f\teval-auc: %f' % (i, np.mean(ls), train_score, test_score)) history_score.append(test_score) if i > min_round and i > early_stop_round: if np.argmax( history_score) == i - early_stop_round and history_score[ -1] - history_score[-1 * early_stop_round] < 1e-5: print('early stop\nbest iteration:\n[%d]\teval-auc: %f' % (np.argmax(history_score), np.max(history_score))) break # var_map = {} # for name, var in model.vars.items(): # print(name, var) # ans = model.run(var) # var_map[name] = ans # print(var_map[name]) if algo == 'fm': model.dump(pkl_path)
def __splitScene(self, scene, layer, offset, size): # self.__logger.debug('splitting scene...') ret = {} bn = self.__batch_n bh = self.__batch_h bw = self.__batch_w bc = self.__batch_c sfeatures = self.__sfeatures ifeatures = self.__ifeatures ret[1] = ret[3] = np.zeros([1, bh, bw, bc]) # make sure ih and iw's value scene[1] = np.reshape(scene[1], [-1, self.__ih, self.__iw, sfeatures]) scene[3] = np.reshape(scene[3], [-1, self.__ih, self.__iw, sfeatures]) sh = offset[0] sw = offset[1] cols = 3 ret[0] = utils.slice(scene[0], [0, sh, sw, 0], [1, bh, bw, cols]) # img-0 ret[1][:, :, :, :sfeatures] = utils.slice( scene[1], [layer, sh, sw, 0], [1, bh, bw, sfeatures]) # txt-0 ret[1][:, :, :, sfeatures + 0:sfeatures + 1] = utils.getLuminance(ret[0]) ret[1][:, :, :, sfeatures + 1:sfeatures + 2] = utils.getMagnitude(ret[0]) ret[2] = utils.slice(scene[2], [0, sh, sw, 0], [1, bh, bw, cols]) # img-1 ret[3][:, :, :, :sfeatures] = utils.slice( scene[3], [layer, sh, sw, 0], [1, bh, bw, sfeatures]) # txt-1 ret[3][:, :, :, sfeatures + 0:sfeatures + 1] = utils.getLuminance(ret[2]) ret[3][:, :, :, sfeatures + 1:sfeatures + 2] = utils.getMagnitude(ret[2]) ret[4] = utils.slice(scene[4], [0, sh, sw, 0], [1, bh, bw, cols]) # truth # preprocess data # ret[1] = self.__Sigmoid(ret[1]) # ret[3] = self.__Sigmoid(ret[3]) # base = np.max(ret[1][:, :, :, 3]) - np.min(ret[1][:, :, :, 3]) # if base == 0: base = np.sum(ret[1][:, :, :, 3]) # ret[1][:, :, :, 3] = (ret[1][:, :, :, 3] - np.min(ret[1][:, :, :, 3])) / (base) # base = np.max(ret[1][:, :, :, 7]) - np.min(ret[1][:, :, :, 7]) # if base == 0: base = np.sum(ret[1][:, :, :, 7]) # ret[1][:, :, :, 7] = (ret[1][:, :, :, 7] - np.min(ret[1][:, :, :, 7])) / (base) # base = np.max(ret[3][:, :, :, 3]) - np.min(ret[3][:, :, :, 3]) # if base == 0: base = np.sum(ret[3][:, :, :, 3]) # ret[3][:, :, :, 3] = (ret[3][:, :, :, 3] - np.min(ret[3][:, :, :, 3])) / (base) # base = np.max(ret[3][:, :, :, 7]) - np.min(ret[3][:, :, :, 7]) # if base == 0: base = np.sum(ret[3][:, :, :, 7]) # ret[3][:, :, :, 7] = (ret[3][:, :, :, 7] - np.min(ret[3][:, :, :, 7])) / (base) return ret
def cut(): # utils.slice('organized_sound/wav/british/bo156.wav', 'test.wav', 0, 3000) base_dir = 'organized_sound/wav/' language_dirs = ['american', 'british'] for language in language_dirs: lang_dir_path = base_dir + language + '/' for filename in os.listdir(base_dir + language): if filename.endswith('.wav'): filepath = lang_dir_path + filename outpath = lang_dir_path + 's_' + filename infile = wave.open(filepath) utils.slice(infile, outpath, 0, 3000)
def train(model): history_score = [] for i in range(num_round): fetches = [model.optimizer, model.loss] if batch_size > 0: ls = [] bar = progressbar.ProgressBar() print('[%d]\ttraining...' % i) for j in bar(range(int(train_size / batch_size + 1))): X_i, y_i = utils.slice(train_data, j * batch_size, batch_size) _, l = model.run(fetches, X_i, y_i) ls.append(l) elif batch_size == -1: X_i, y_i = utils.slice(train_data) _, l = model.run(fetches, X_i, y_i) ls = [l] train_preds = [] print('[%d]\tevaluating...' % i) bar = progressbar.ProgressBar() for j in bar(range(int(train_size / 10000 + 1))): X_i, _ = utils.slice(train_data, j * 10000, 10000) preds = model.run(model.y_prob, X_i, mode='test') train_preds.extend(preds) test_preds = [] bar = progressbar.ProgressBar() for j in bar(range(int(test_size / 10000 + 1))): X_i, _ = utils.slice(test_data, j * 10000, 10000) preds = model.run(model.y_prob, X_i, mode='test') test_preds.extend(preds) train_score = roc_auc_score(train_data[1], train_preds) train_loss = log_loss(train_data[1], train_preds) test_score = roc_auc_score(test_data[1], test_preds) test_loss = log_loss(test_data[1], test_preds) print('[%d]\tloss (with l2 norm):%f\ttrain-auc: %f\teval-auc: %f' % (i, np.mean(ls), train_score, test_score)) print('train_loss:{0}\ttest_loss:{1}'.format(train_loss, test_loss)) history_score.append(test_score) if i > min_round and i > early_stop_round: if np.argmax( history_score) == i - early_stop_round and history_score[ -1] - history_score[-1 * early_stop_round] < 1e-5: print('early stop\nbest iteration:\n[%d]\teval-auc: %f' % (np.argmax(history_score), np.max(history_score))) break
def train(model): history_score = [] for i in range(num_round): fetches = [model.optimizer, model.loss] #if model.model_file is not None: # model.saver.restore(model.sess,model.model_file) if batch_size > 0: ls = [] print('[%d]\ttraining...' % i) for j in range(int(train_size / batch_size + 1)): X_i, y_i = utils.slice(train_data, j * batch_size, batch_size) _, l = model.run(fetches, X_i, y_i) ls.append(l) elif batch_size == -1: X_i, y_i = utils.slice(train_data) print(X_i.shape) print(y_i.shape) _, l = model.run(fetches, X_i, y_i) ls = [l] #model.saver.save(model.sess,'./Save/train.ckpt') train_preds = [] print('[%d]\tevaluating...' % i) for j in range(int(train_size / 10000 + 1)): X_i, _ = utils.slice(train_data, j * 10000, 10000) preds = model.run(model.y_prob, X_i, mode='test') train_preds.extend(preds) test_preds = [] for j in range(int(test_size / 10000 + 1)): X_i, _ = utils.slice(test_data, j * 10000, 10000) preds = model.run(model.y_prob, X_i, mode='test') test_preds.extend(preds) train_score = roc_auc_score(train_data[1], train_preds) test_score = roc_auc_score(test_data[1], test_preds) #for i in train_preds: # print(i) print('[%d]\tloss (with l2 norm):%f\ttrain-auc: %f\teval-auc: %f' % (i, np.mean(ls), train_score, test_score)) history_score.append(test_score) if i > min_round and i > early_stop_round: if np.argmax( history_score) == i - early_stop_round and history_score[ -1] - history_score[-1 * early_stop_round] < 1e-5: print('early stop\nbest iteration:\n[%d]\teval-auc: %f' % (np.argmax(history_score), np.max(history_score))) break
def step(self, rxSym, H = None, method = 'MMSE'): noise = utils.cplxRandn(rxSym.shape, self.N0) rx = rxSym + noise if method == 'MMSE': assert (not H is None) M = np.dot(utils.hermitian(H), np.linalg.inv(np.dot(H, utils.hermitian(H)) + np.eye(H.shape[0]) * self.N0)) est = np.dot(M, rx) return utils.slice(est, self.modSyms)
def train(model): history_score = [] start_time = time.time() print 'epochs\tloss\ttrain-auc\teval-auc\ttime' sys.stdout.flush() for i in range(num_round): fetches = [model.optimizer, model.loss] if batch_size > 0: ls = [] for j in range(train_size / batch_size + 1): X_i, y_i = utils.slice(train_data, j * batch_size, batch_size) _, l = model.run(fetches, X_i, y_i) ls.append(l) elif batch_size == -1: X_i, y_i = utils.slice(train_data) _, l = model.run(fetches, X_i, y_i) ls = [l] lst_train_pred = [] lst_test_pred = [] if batch_size > 0: for j in range(train_size / batch_size + 1): X_i, y_i = utils.slice(train_data, j * batch_size, batch_size) #X_i = utils.libsvm_2_coo(X_i, (len(X_i), input_dim)).tocsr() _train_preds = model.run(model.y_prob, X_i) lst_train_pred.append(_train_preds) for j in range(test_size / batch_size + 1): X_i, y_i = utils.slice(test_data, j * batch_size, batch_size) #X_i = utils.libsvm_2_coo(X_i, (len(X_i), input_dim)).tocsr() _test_preds = model.run(model.y_prob, X_i) lst_test_pred.append(_test_preds) train_preds = np.concatenate(lst_train_pred) test_preds = np.concatenate(lst_test_pred) train_score = roc_auc_score(train_data[1], train_preds) test_score = roc_auc_score(test_data[1], test_preds) print '%d\t%f\t%f\t%f\t%f' % (i, np.mean(ls), train_score, test_score, time.time() - start_time) sys.stdout.flush() history_score.append(test_score) if i > min_round and i > early_stop_round: if np.argmax(history_score) == i - early_stop_round and history_score[-1] - history_score[ -1 * early_stop_round] < 1e-5: print 'early stop\nbest iteration:\n[%d]\teval-auc: %f' % ( np.argmax(history_score), np.max(history_score)) sys.stdout.flush() break
def train(model): history_score = [] for i in range(num_round): fetches = [model.optimizer, model.loss] if batch_size > 0: ls = [] bar = progressbar.ProgressBar() print('[%d]\ttraining...' % i) for j in bar(range(int(train_size / batch_size + 1))): X_i, y_i = utils.slice(train_data, j * batch_size, batch_size) _, l = model.run(fetches, X_i, y_i) ls.append(l) elif batch_size == -1: X_i, y_i = utils.slice(train_data) _, l = model.run(fetches, X_i, y_i) ls = [l] train_preds = [] print('[%d]\tevaluating...' % i) bar = progressbar.ProgressBar() for j in bar(range(int(train_size / 10000 + 1))): X_i, _ = utils.slice(train_data, j * 10000, 10000) preds = model.run(model.y_prob, X_i, mode='test') train_preds.extend(preds) test_preds = [] bar = progressbar.ProgressBar() for j in bar(range(int(test_size / 10000 + 1))): X_i, _ = utils.slice(test_data, j * 10000, 10000) preds = model.run(model.y_prob, X_i, mode='test') test_preds.extend(preds) train_score = roc_auc_score(train_data[1], train_preds) train_loss = log_loss(train_data[1], train_preds) test_score = roc_auc_score(test_data[1], test_preds) test_loss = log_loss(test_data[1], test_preds) print('[%d]\tloss (with l2 norm):%f\ttrain-auc: %f\teval-auc: %f' % (i, np.mean(ls), train_score, test_score)) print('train_loss:{0}\ttest_loss:{1}'.format(train_loss, test_loss)) history_score.append(test_score) if i > min_round and i > early_stop_round: if np.argmax(history_score) == i - early_stop_round and history_score[-1] - history_score[ -1 * early_stop_round] < 1e-5: print('early stop\nbest iteration:\n[%d]\teval-auc: %f' % ( np.argmax(history_score), np.max(history_score))) break
'kernel_l2': 0, 'random_seed': 0 } model = PNN1(**pnn1_params) elif algo == 'pnn2': pnn2_params = { 'layer_sizes': [field_sizes, 10, 1], 'layer_acts': ['tanh', 'none'], 'drop_out': [0, 0], 'opt_algo': 'gd', 'learning_rate': 0.01, 'layer_l2': [0, 0], 'kernel_l2': 0, 'random_seed': 0 } model = PNN2(**pnn2_params) if algo in {'fnn', 'ccpm', 'pnn1', 'pnn2'}: train_data = utils.split_data(train_data) test_data = utils.split_data(test_data) train(model) X_i, y_i = utils.slice(train_data, 0, 100) fetches = [model.tmp1, model.tmp2] tmp1, tmp2 = model.run(fetches, X_i, y_i) print(tmp1.shape) print(tmp2.shape)
def train(model, name): global batch_size global time_run global time_process history_score = [] start_time = time.time() print 'epochs\tloss\ttrain-auc\teval-auc\ttime' sys.stdout.flush() for i in range(num_round): fetches = [model.optimizer, model.loss] if batch_size > 0: ls = [] for j in range(train_size / batch_size + 1): start_process = time.time() X_i, y_i = utils.slice(train_data, j * batch_size, batch_size) time_process += time.time() - start_process start_run = time.time() _, l = model.run(fetches, X_i, y_i) time_run += time.time() - start_run ls.append(l) ''' f = open(train_file, 'r') lst_lines = [] for line in f: if len(lst_lines) < batch_size: lst_lines.append(line) else: X_i, y_i = utils.slice(utils.process_lines(lst_lines, name), 0, -1) print 'type(X_i)', type(X_i) print 'type(X_i[0])', type(X_i[0]) _, l = model.run(fetches, X_i, y_i) ls.append(l) lst_lines = [line] f.close() if len(lst_lines) > 0: X_i, y_i = utils.slice(utils.process_lines(lst_lines, name), 0, -1) _, l = model.run(fetches, X_i, y_i) ls.append(l) ''' ''' while True: lines_gen = list(islice(f, batch_size * bb)) if not lines_gen: break for ib in range(bb): X_i, y_i = utils.slice(utils.process_lines(lines_gen[batch_size * ib : batch_size * (ib+1)], name), 0, -1) _, l = model.run(fetches, X_i, y_i) ls.append(l) ''' elif batch_size == -1: pass """ X_i, y_i = utils.slice(train_data) _, l = model.run(fetches, X_i, y_i) ls = [l] """ lst_train_pred = [] lst_test_pred = [] if batch_size > 0: f = open(train_file, 'r') lst_lines = [] for line in f: if len(lst_lines) < batch_size: lst_lines.append(line) else: X_i, y_i = utils.slice( utils.process_lines(lst_lines, name), 0, -1) _train_preds = model.run(model.y_prob, X_i) lst_train_pred.append(_train_preds) lst_lines = [line] f.close() if len(lst_lines) > 0: X_i, y_i = utils.slice(utils.process_lines(lst_lines, name), 0, -1) _train_preds = model.run(model.y_prob, X_i) lst_train_pred.append(_train_preds) ''' while True: lines_gen = list(islice(f, batch_size * bb)) if not lines_gen: break for ib in range(bb): X_i, y_i = utils.slice(utils.process_lines(lines_gen[batch_size * ib : batch_size * (ib+1)], name), 0, -1) _train_preds = model.run(model.y_prob, X_i) lst_train_pred.append(_train_preds) ''' """ for j in range(train_size / batch_size + 1): X_i, y_i = utils.slice(train_data, j * batch_size, batch_size) #X_i = utils.libsvm_2_coo(X_i, (len(X_i), input_dim)).tocsr() _train_preds = model.run(model.y_prob, X_i) lst_train_pred.append(_train_preds) """ f = open(test_file, 'r') lst_lines = [] for line in f: if len(lst_lines) < batch_size: lst_lines.append(line) else: X_i, y_i = utils.slice( utils.process_lines(lst_lines, name), 0, -1) _test_preds = model.run(model.y_prob, X_i) lst_test_pred.append(_test_preds) lst_lines = [line] f.close() if len(lst_lines) > 0: X_i, y_i = utils.slice(utils.process_lines(lst_lines, name), 0, -1) _test_preds = model.run(model.y_prob, X_i) lst_test_pred.append(_test_preds) ''' while True: lines_gen = list(islice(f, batch_size * bb)) if not lines_gen: break for ib in range(bb): X_i, y_i = utils.slice(utils.process_lines(lines_gen[batch_size * ib : batch_size * (ib+1)], name), 0, -1) _test_preds = model.run(model.y_prob, X_i) lst_test_pred.append(_test_preds) ''' """ for j in range(test_size / batch_size + 1): X_i, y_i = utils.slice(test_data, j * batch_size, batch_size) #X_i = utils.libsvm_2_coo(X_i, (len(X_i), input_dim)).tocsr() _test_preds = model.run(model.y_prob, X_i) lst_test_pred.append(_test_preds) """ train_preds = np.concatenate(lst_train_pred) test_preds = np.concatenate(lst_test_pred) train_score = roc_auc_score(train_label, train_preds) test_score = roc_auc_score(test_label, test_preds) print '%d\t%f\t%f\t%f\t%f\t%s' % ( i, np.mean(ls), train_score, test_score, time.time() - start_time, strftime("%Y-%m-%d %H:%M:%S", gmtime())) print 'time_run', time_run print 'time_process', time_process # Save the model to local files ''' path_model = 'model/' + str(name) + '_epoch_' + str(i) model.dump(path_model) d_label_score = {} d_label_score['label'] = test_label d_label_score['score'] = test_preds #path_label_score = 'model/label_score_' + str(name) + '_epoch_' + str(i) #pkl.dump(d_label_score, open(path_label_score, 'wb')) sys.stdout.flush() ''' history_score.append(test_score) if i > min_round and i > early_stop_round: i_max = np.argmax(history_score) # Early stop if i - i_max >= early_stop_round: print 'early stop\nbest iteration:\n[%d]\teval-auc: %f' % ( np.argmax(history_score), np.max(history_score)) sys.stdout.flush() break '''
T[i] = map(int, filter(lambda c: c != '', T[i].split(' '))) WIDTH = len(T) HEIGHT = len(T[0]) log.info('Image size: %sx%s' % (WIDTH, HEIGHT)) if __name__ == "__main__": from itertools import islice p = Pool(processes=len(users) / 3) BLOCK_SIZE = 16 try: while True: fetched = utils.fetch() if not fetched: log.fatal('Failed to fetch canvas.') exit(-1) D = utils.slice(utils.fetch(), WIDTH, HEIGHT) log.info('Fetched canvas.') cnt = 0 for idx in p.imap_unordered(utils.draw, utils.diff(D, T, provider)): taken[idx] = False cnt += 1 # for idx in p.imap_unordered(utils.draw, islice(utils.diff(D, T, provider), 0, BLOCK_SIZE)): # taken[idx] = False log.info('Rendered %s pixel(s).' % cnt) if cnt: time.sleep(0.1) else: time.sleep(60) except: exc_type, exc_value, exc_traceback = sys.exc_info()
def train(self, sample_dir, ckpt_dir='ckpt', training_epoches=1000000, batch_size=512): fig_count = 0 self.sess.run(tf.global_variables_initializer()) train_size = train_data[0].shape[0] # test_size = test_data[0].shape[0] for epoch in range(training_epoches): # update D for j in range(int(train_size / batch_size + 1)): X_b, y_b = utils.slice(train_data, j * batch_size, batch_size) from keras.utils import to_categorical y_b = to_categorical(y_b, 2) # X_b, y_b = self.data(batch_size) self.sess.run(self.D_solver, feed_dict={ self.X: X_b, self.y: y_b, self.z: sample_z(len(y_b), self.z_dim) }) # update G k = 1 for _ in range(k): self.sess.run(self.G_solver, feed_dict={ self.y: y_b, self.z: sample_z(len(y_b), self.z_dim) }) # save img, model. print loss if epoch % 100 == 0 or epoch < 100: D_loss_curr = self.sess.run(self.D_loss, feed_dict={ self.X: X_b, self.y: y_b, self.z: sample_z(len(y_b), self.z_dim) }) G_loss_curr = self.sess.run(self.G_loss, feed_dict={ self.y: y_b, self.z: sample_z(len(y_b), self.z_dim) }) print('Iter: {}; D loss: {:.4}; G_loss: {:.4}'.format( epoch, D_loss_curr, G_loss_curr)) if epoch % 1000 == 0: y_s = sample_y(16, self.y_dim, fig_count % 10) samples = self.sess.run(self.G_sample, feed_dict={ self.y: y_s, self.z: sample_z(16, self.z_dim) }) print(y_s.shape) print(samples.shape)
def do_inference(hostport, test_data, concurrency, num_tests, batch_size): """Tests PredictionService with concurrent-batched requests. Args: hostport: Host:port address of the PredictionService. test_data: The full path to the test data set. concurrency: Maximum number of concurrent requests. num_tests: Number of test tensors to use. batch_size: Number of tests to include in each query Returns: The results of the queries Raises: IOError: An error occurred processing test data set. """ host, port = hostport.split(':') channel = implementations.insecure_channel(host, int(port)) stub = prediction_service_pb2.beta_create_PredictionService_stub(channel) result_counter = _ResultCounter(concurrency) num_field = 15 with open(test_data, 'r') as f: data = f.read().split('\n') requests = [] t0 = time.time() #Create batches of requests for i in range(num_batches): data_batch = data[:batch_size] data = data[batch_size:] index_list = [[] for _ in range(num_field)] values_list = [[] for _ in range(num_field)] for j in range(batch_size): X, y = utils.slice( utils.process_lines([data_batch[j]], 'fwfm', INPUT_DIM, FIELD_OFFSETS), 0, -1) for idx in range(num_field): index_list[idx].append(X[idx][0].tolist()) values_list[idx].append(1) requests.append(predict_pb2.PredictRequest()) requests[i].model_spec.name = 'serve' requests[i].model_spec.signature_name = 'model' requests[i].output_filter.append('outputs') for idx in range(num_field): requests[i].inputs["field_" + str(idx) + "_values"].CopyFrom( tf.contrib.util.make_tensor_proto( values_list[idx], shape=[len(values_list[idx])], dtype=tf.int64)) requests[i].inputs["field_" + str(idx) + "_indices"].CopyFrom( tf.contrib.util.make_tensor_proto( index_list[idx], shape=[len(index_list[idx]), 2], dtype=tf.float32)) requests[i].inputs["field_" + str(idx) + "_dense_shape"].CopyFrom( tf.contrib.util.make_tensor_proto( [batch_size, total_features], shape=[2], dtype=tf.int64)) t1 = time.time() #Query server for i in range(num_batches): result_counter.throttle() result = stub.Predict.future(requests[i], 100.0) # 100 secs timeout result.add_done_callback(_create_rpc_callback(i, result_counter)) t2 = time.time() #Synchronize on comleted queries result_counter.get_complete() t3 = time.time() full_results = [] for values in results: full_results.extend(values) print("Elapsed time for ", num_tests, " request creations: ", (t1 - t0)) print("Elapsed time for ", num_batches, " batch submissions: ", (t2 - t1)) print("Elapsed time for ", num_tests, " inferences: ", (t3 - t1)) return full_results
net.load_state_dict(torch.load(args.model, map_location='cpu')) print("Using CPU version of the net, this may be very slow") print("Model loaded !") dataset = HelioDataset('./data/sidc/SIDC_dataset.csv', 'data/dpd/', 1) data_loader = DataLoader(dataset) pred_mask_slices = [] input = None true_mask = None for _, obs in enumerate(data_loader): input = np.array(obs["img"][0]) true_mask = np.array(obs["mask"][0]) obs = slice(obs, args.window, args.window) for idx in range(0, len(obs['imgs']), args.batch_size): print("\nPredicting images {0} - {1} ...".format( idx, idx + args.batch_size)) imgs = obs['imgs'][idx:idx + args.batch_size].float() masks = predict_img(net=net, imgs=imgs, out_threshold=args.mask_threshold, use_dense_crf=args.crf, use_gpu=not args.cpu) pred_mask_slices.extend(masks.squeeze()) pred_mask_slices = np.array(pred_mask_slices) n = input[0].shape[0] // args.window
def train(model, name): history_score = [] start_time = time.time() print 'epochs\tloss\ttrain-auc\teval-auc\ttime' sys.stdout.flush() for i in range(num_round): fetches = [model.optimizer, model.loss] if batch_size > 0: ls = [] f = open(train_file, 'r') while True: lines_gen = list(islice(f, batch_size * bb)) if not lines_gen: break for ib in range(bb): X_i, y_i = utils.slice(utils.process_lines(lines_gen[batch_size * ib : batch_size * (ib+1)]), 0, -1) _, l = model.run(fetches, X_i, y_i) ls.append(l) elif batch_size == -1: pass """ X_i, y_i = utils.slice(train_data) _, l = model.run(fetches, X_i, y_i) ls = [l] """ lst_train_pred = [] lst_test_pred = [] if batch_size > 0: f = open(train_file, 'r') while True: lines_gen = list(islice(f, batch_size * bb)) if not lines_gen: break for ib in range(bb): X_i, y_i = utils.slice(utils.process_lines(lines_gen[batch_size * ib : batch_size * (ib+1)]), 0, -1) _train_preds = model.run(model.y_prob, X_i) lst_train_pred.append(_train_preds) """ for j in range(train_size / batch_size + 1): X_i, y_i = utils.slice(train_data, j * batch_size, batch_size) #X_i = utils.libsvm_2_coo(X_i, (len(X_i), input_dim)).tocsr() _train_preds = model.run(model.y_prob, X_i) lst_train_pred.append(_train_preds) """ f = open(test_file, 'r') while True: lines_gen = list(islice(f, batch_size * bb)) if not lines_gen: break for ib in range(bb): X_i, y_i = utils.slice(utils.process_lines(lines_gen[batch_size * ib : batch_size * (ib+1)]), 0, -1) _test_preds = model.run(model.y_prob, X_i) lst_test_pred.append(_test_preds) """ for j in range(test_size / batch_size + 1): X_i, y_i = utils.slice(test_data, j * batch_size, batch_size) #X_i = utils.libsvm_2_coo(X_i, (len(X_i), input_dim)).tocsr() _test_preds = model.run(model.y_prob, X_i) lst_test_pred.append(_test_preds) """ train_preds = np.concatenate(lst_train_pred) test_preds = np.concatenate(lst_test_pred) train_score = roc_auc_score(train_label, train_preds) test_score = roc_auc_score(test_label, test_preds) print '%d\t%f\t%f\t%f\t%f\t%s' % (i, np.mean(ls), train_score, test_score, time.time() - start_time, strftime("%Y-%m-%d %H:%M:%S", gmtime())) path_model = 'model/' + str(name) + '_epoch_' + str(i) path_label_score = 'model/label_score_' + str(name) + '_epoch_' + str(i) #model.dump(path_model) d_label_score = {} d_label_score['label'] = test_label d_label_score['score'] = test_preds #pkl.dump(d_label_score, open(path_label_score, 'wb')) sys.stdout.flush() history_score.append(test_score) if i > min_round and i > early_stop_round: #if np.argmax(history_score) == i - early_stop_round and history_score[-1] - history_score[ # -1 * early_stop_round] < 1e-5: i_max = np.argmax(history_score) if i - i_max >= early_stop_round: print 'early stop\nbest iteration:\n[%d]\teval-auc: %f' % ( np.argmax(history_score), np.max(history_score)) sys.stdout.flush() break
def __call__(self, m_, x_, h_, c_, a_, ct_, pctx_): """ Each variable is one time slice of the LSTM m_ - (mask), x_- (previous word), h_- (hidden state), c_- (lstm memory), a_ - (alpha distribution [eq (5)]), as_- (sample from alpha dist), ct_- (context), pctx_ (projected context), """ # attention computation # [described in equations (4), (5), (6) in # section "3.1.2 Decoder: Long Short Term Memory Network] # f_att(a_i, h_t-1) This part is concatenation between a_i and h_t-1 pstate_ = tensor.dot(h_, self.Wd_att) pctx_ = pctx_ + pstate_[:, None, :] pctx_list = [pctx_] pctx_ = tensor.tanh(pctx_) #equation 4 alpha = tensor.dot(pctx_, self.U_att) + self.c_att #Eqn 5 alpha = tensor.nnet.softmax( alpha.reshape([alpha.shape[0], alpha.shape[1]])) # softmax #Eqn 6 ctx_ = (self.context * alpha[:, :, None]).sum( 1) # current context, aka zt in paper #selector sel_ = tensor.nnet.sigmoid(tensor.dot(h_, self.W_sel + self.b_sel)) sel_ = sel_.reshape([sel_.shape[0]]) ctx_ = sel_[:, None] * ctx_ #Accumulation to compute different gate outputs preact = tensor.dot(h_, self.U) preact += x_ preact += tensor.dot(ctx_, self.Wc) dim = self.U.shape[0] # Recover the activations to the lstm gates # [equation (1)] i = utils.slice(preact, 0, dim) f = utils.slice(preact, 1, dim) o = utils.slice(preact, 2, dim) i = tensor.nnet.sigmoid(i) f = tensor.nnet.sigmoid(f) o = tensor.nnet.sigmoid(o) c = tensor.tanh(utils.slice(preact, 3, dim)) # compute the new memory/hidden state # if the mask is 0, just copy the previous state #Eqn 2 c = f * c_ + i * c c = m_[:, None] * c + (1. - m_)[:, None] * c_ #Eqn 3 h = o * tensor.tanh(c) h = m_[:, None] * h + (1. - m_)[:, None] * h_ return [h, c, alpha, ctx_]
def next_batch(self, seed): np.random.seed(seed) assert (self.__isTrain) self.__logger.debug('Size of Cache:%dM' % self.__cache.getSize()) self.__logger.debug('Eta of Cache:%.2f%%' % (self.__cache.getEta() * 100.0)) cols = self.__cols bn = self.__batch_n bh = self.__batch_h bw = self.__batch_w bc = self.__batch_c sfeatures = self.__sfeatures # x1 input image,x2 txt features,y ground truth x1 = np.zeros([bn, bh, bw, cols]) x2 = np.zeros([bn, bh, bw, bc]) y = np.zeros([bn, bh, bw, cols]) # select a scene randomly id = np.random.randint(0, len(self.__directories)) while self.__directories[id] in self.__abandons: id = np.random.randint(0, len(self.__directories)) self.__logger.debug( 'something is wrong in this scene,looking for another one...') # iter each layer of batchs for n in range(bn): # get input data a = b = c = None imgpath = self.__directories[id] + self.__img_input txtpath = self.__directories[id] + self.__txt_input confpath = self.__directories[id] + self.__conf_input # exit(-1) # print(imgpath,txtpath,confpath) isMatch = self.checkMatch(imgpath, txtpath, confpath) # checkMatch before putting them into Cache # skip current scene when md5 mismatch if not isMatch: n -= 1 self.__abandons.append(self.__directories[id]) self.__logger.warn( 'md5 of scene [%s] mismatch,abandon this one!' % self.__directories[id]) continue try: a = self.__cache.get(self.__directories[id] + self.__img_input) except: a = utils.readIMG(self.__directories[id] + self.__img_input) self.__cache.add(self.__directories[id] + self.__img_input, a) try: b = self.__cache.get(self.__directories[id] + self.__txt_input) except: b = utils.readTXT(self.__directories[id] + self.__txt_input) self.__cache.add(self.__directories[id] + self.__txt_input, b) try: c = self.__cache.get(self.__directories[id] + self.__ground_truth) except: c = utils.readIMG(self.__directories[id] + self.__ground_truth) self.__cache.add(self.__directories[id] + self.__ground_truth, c) value = {'input': a, 'data': b, 'truth': c} # some assertions about input data assert (len(value['input'].shape) == 4) assert (len(value['truth'].shape) == 4) assert (value['input'].shape[1] == value['truth'].shape[1]) assert (value['input'].shape[2] == value['truth'].shape[2]) # height & width of the input image ih = value['input'].shape[1] iw = value['input'].shape[2] value['data'] = np.reshape(value['data'], [-1, ih, iw, sfeatures]) # pick a sample pos(left&up corner) randomly sh = np.random.randint(0, ih - bh) sw = np.random.randint(0, iw - bw) sh = sw = 400 # part of input image(x1) & ground truth(y) px1 = utils.slice(value['input'], [0, sh, sw, 0], [1, bh, bw, cols]) py = utils.slice(value['truth'], [0, sh, sw, 0], [1, bh, bw, cols]) # select a layer of txt randomly idx = np.random.randint(0, value['data'].shape[0]) # self.__logger.debug('%d %d %d %d' % (id,idx,sh,sw)) # idx = 0 # part of input txt(px2) px2 = np.zeros([1, bh, bw, bc]) # [ 0:features]:features stored in txt px2[:, :, :, :sfeatures] = utils.slice(value['data'], [idx, sh, sw, 0], [1, bh, bw, sfeatures]) # [features:]:features extracted from img assert (self.__ifeatures == 2) px2[:, :, :, sfeatures + 0:sfeatures + 1] = utils.getLuminance(px1) px2[:, :, :, sfeatures + 1:sfeatures + 2] = utils.getMagnitude(px1) # px2[:,:,:,features + 2:features + 5] = # np.reshape(px1,[bh,bw,cols]) px2 = np.reshape(px2, [-1, bc]) # reshape these parts x1[n] = np.reshape(px1, [bh, bw, cols]) x2[n] = np.reshape(px2, [bh, bw, bc]) y[n] = np.reshape(py, [bh, bw, cols]) # normalize data before return them x2 = self.normalize(x2) # utils.displayImage(x1) return x1, x2, y
def train(model, name, in_memory = True, flag_MTL = False): #builder = tf.saved_model.builder.SavedModelBuilder('model') global batch_size, time_run, time_read, time_process history_score = [] best_score = -1 best_epoch = -1 start_time = time.time() print 'epochs\tloss\ttrain-auc\teval-auc\ttime' sys.stdout.flush() if in_memory: train_data = utils.read_data(path_train, INPUT_DIM) validation_data = utils.read_data(path_validation, INPUT_DIM) test_data = utils.read_data(path_test, INPUT_DIM) model_name = name.split('_')[0] if model_name in set(['lr', 'fm']): train_data_tmp = utils.split_data(train_data, FIELD_OFFSETS) validation_data_tmp = utils.split_data(validation_data, FIELD_OFFSETS) test_data_tmp = utils.split_data(test_data, FIELD_OFFSETS) else: train_data = utils.split_data(train_data, FIELD_OFFSETS) validation_data = utils.split_data(validation_data, FIELD_OFFSETS) test_data = utils.split_data(test_data, FIELD_OFFSETS) for i in range(num_round): fetches = [model.optimizer, model.loss] if batch_size > 0: ls = [] if in_memory: for j in range(train_size / batch_size + 1): X_i, y_i = utils.slice(train_data, j * batch_size, batch_size) _, l = model.run(fetches, X_i, y_i) ls.append(l) else: f = open(path_train, 'r') lst_lines = [] for line in f: if len(lst_lines) < batch_size: lst_lines.append(line) else: X_i, y_i = utils.slice(utils.process_lines(lst_lines, name, INPUT_DIM, FIELD_OFFSETS), 0, -1) # type of X_i, X_i[0], X_i[0][0] is list, tuple and np.ndarray respectively. _, l = model.run(fetches, X_i, y_i) ls.append(l) lst_lines = [line] f.close() if len(lst_lines) > 0: X_i, y_i = utils.slice(utils.process_lines(lst_lines, name, INPUT_DIM, FIELD_OFFSETS), 0, -1) _, l = model.run(fetches, X_i, y_i) ls.append(l) elif batch_size == -1: pass model.dump('model/' + name + '_epoch_' + str(i)) if in_memory: lst_train_preds = [] lst_validation_preds = [] lst_test_preds = [] for j in range(train_size / batch_size + 1): X_i, y_i = utils.slice(train_data, j * batch_size, batch_size) p = model.run(model.y_prob, X_i, y_i) lst_train_preds.append(p) for j in range(validation_size / batch_size + 1): X_i, y_i = utils.slice(validation_data, j * batch_size, batch_size) p = model.run(model.y_prob, X_i, y_i) lst_validation_preds.append(p) for j in range(test_size / batch_size + 1): X_i, y_i = utils.slice(test_data, j * batch_size, batch_size) p = model.run(model.y_prob, X_i, y_i) lst_test_preds.append(p) train_preds = np.concatenate(lst_train_preds) validation_preds = np.concatenate(lst_validation_preds) test_preds = np.concatenate(lst_test_preds) #train_preds = model.run(model.y_prob, utils.slice(train_data)[0]) #test_preds = model.run(model.y_prob, utils.slice(test_data)[0]) train_score = roc_auc_score(train_data[1], train_preds) validation_score = roc_auc_score(validation_data[1], validation_preds) test_score = roc_auc_score(test_data[1], test_preds) train_score_sum = 0 train_score_weight = 0 validation_score_sum = 0 validation_score_weight = 0 test_score_sum = 0 test_score_weight = 0 #print '[%d]\tloss:%f\ttrain-auc: %f\teval-auc: %f' % (i, np.mean(ls), train_score, test_score) print '%d\t%f\t%f\t%f\t%f\t%f\t%s' % (i, np.mean(ls), train_score, validation_score, test_score, time.time() - start_time, strftime("%Y-%m-%d %H:%M:%S", gmtime())) if flag_MTL: d_index_task_label_pred_train = {} d_index_task_label_pred_validation = {} d_index_task_label_pred_test = {} if model_name in set(['lr', 'fm']): index_task_train = train_data_tmp[0][-1].indices index_task_validation = validation_data_tmp[0][-1].indices index_task_test = test_data_tmp[0][-1].indices else: index_task_train = train_data[0][-1].indices index_task_validation = validation_data[0][-1].indices index_task_test = test_data[0][-1].indices for index_tmp in range(len(index_task_train)): index_task = index_task_train[index_tmp] d_index_task_label_pred_train.setdefault(index_task, [[],[]]) d_index_task_label_pred_train[index_task][0].append(train_data[1][index_tmp]) d_index_task_label_pred_train[index_task][1].append(train_preds[index_tmp]) for index_task in sorted(list(set(index_task_train))): auc = roc_auc_score(d_index_task_label_pred_train[index_task][0], d_index_task_label_pred_train[index_task][1]) num_samples = len(d_index_task_label_pred_train[index_task][0]) train_score_sum += auc * num_samples train_score_weight += num_samples print 'train, index_type: %d, number of samples: %d, AUC: %f' % (index_task, len(d_index_task_label_pred_train[index_task][0]), auc) for index_tmp in range(len(index_task_validation)): index_task = index_task_validation[index_tmp] d_index_task_label_pred_validation.setdefault(index_task, [[],[]]) d_index_task_label_pred_validation[index_task][0].append(validation_data[1][index_tmp]) d_index_task_label_pred_validation[index_task][1].append(validation_preds[index_tmp]) for index_task in sorted(list(set(index_task_validation))): auc = roc_auc_score(d_index_task_label_pred_validation[index_task][0], d_index_task_label_pred_validation[index_task][1]) num_samples = len(d_index_task_label_pred_validation[index_task][0]) validation_score_sum += auc * num_samples validation_score_weight += num_samples print 'validation, index_type: %d, number of samples: %d, AUC: %f' % (index_task, num_samples, auc) for index_tmp in range(len(index_task_test)): index_task = index_task_test[index_tmp] d_index_task_label_pred_test.setdefault(index_task, [[],[]]) d_index_task_label_pred_test[index_task][0].append(test_data[1][index_tmp]) d_index_task_label_pred_test[index_task][1].append(test_preds[index_tmp]) for index_task in sorted(list(set(index_task_test))): auc = roc_auc_score(d_index_task_label_pred_test[index_task][0], d_index_task_label_pred_test[index_task][1]) num_samples = len(d_index_task_label_pred_test[index_task][0]) test_score_sum += auc * num_samples test_score_weight += num_samples print 'test, index_type: %d, number of samples: %d, AUC: %f' % (index_task, len(d_index_task_label_pred_test[index_task][0]), auc) weighted_train_score = train_score_sum / train_score_weight print 'weighted_train_score', weighted_train_score weighted_validation_score = validation_score_sum / validation_score_weight print 'weighted_validation_score', weighted_validation_score weighted_test_score = test_score_sum / test_score_weight print 'weighted_test_score', weighted_test_score history_score.append(weighted_validation_score) if weighted_validation_score < best_score and (i - best_epoch) >= 3: break if weighted_validation_score > best_score: best_score = weighted_validation_score best_epoch = i sys.stdout.flush() else: lst_train_pred = [] lst_test_pred = [] if batch_size > 0: f = open(path_train, 'r') lst_lines = [] for line in f: if len(lst_lines) < batch_size: lst_lines.append(line) else: X_i, y_i = utils.slice(utils.process_lines(lst_lines, name, INPUT_DIM, FIELD_OFFSETS), 0, -1) _train_preds = model.run(model.y_prob, X_i) lst_train_pred.append(_train_preds) lst_lines = [line] f.close() if len(lst_lines) > 0: X_i, y_i = utils.slice(utils.process_lines(lst_lines, name, INPUT_DIM, FIELD_OFFSETS), 0, -1) _train_preds = model.run(model.y_prob, X_i) lst_train_pred.append(_train_preds) f = open(path_test, 'r') lst_lines = [] for line in f: if len(lst_lines) < batch_size: lst_lines.append(line) else: X_i, y_i = utils.slice(utils.process_lines(lst_lines, name, INPUT_DIM, FIELD_OFFSETS), 0, -1) _test_preds = model.run(model.y_prob, X_i) lst_test_pred.append(_test_preds) lst_lines = [line] f.close() if len(lst_lines) > 0: X_i, y_i = utils.slice(utils.process_lines(lst_lines, name, INPUT_DIM, FIELD_OFFSETS), 0, -1) _test_preds = model.run(model.y_prob, X_i) lst_test_pred.append(_test_preds) train_preds = np.concatenate(lst_train_pred) test_preds = np.concatenate(lst_test_pred) print 'np.shape(train_preds)', np.shape(train_preds) train_score = roc_auc_score(train_label, train_preds) test_score = roc_auc_score(test_label, test_preds) print '%d\t%f\t%f\t%f\t%f\t%s' % (i, np.mean(ls), train_score, test_score, time.time() - start_time, strftime("%Y-%m-%d %H:%M:%S", gmtime())) sys.stdout.flush() '''