def tag2(self, sent): tagged = [[self._normalize(word), pos, chk, None] for word, pos, chk in sent] nword = len(sent) ntag = len(self.role_set) pi = [[[[0, None, None] for k in range(ntag)] for j in range(ntag)] for i in range(nword)] for i, (word, tag, chk, role) in enumerate(tagged): pword1, ptag1, pchk1 = ("START1_WORD", "START1_TAG", "START1_CHK") if i <= 0 else tagged[i - 1][:3] pword2, ptag2, pchk2 = ("START2_WORD", "START2_TAG", "START2_CHK") if i <= 1 else tagged[i - 2][:3] word, tag, chk = tagged[i][:3] fword1, ftag1, fchk1 = ("END1_WORD", "END1_TAG", "END1_CHK") if i >= len(tagged) - 1 else tagged[i + 1][:3] fword2, ftag2, fchk2 = ("END2_WORD", "END2_TAG", "END2_CHK") if i >= len(tagged) - 2 else tagged[i + 2][:3] for j, u in enumerate(self.role_set): prole2 = "START2_ROLE" if i <= 0 else u for k, v in enumerate(self.role_set): prole1 = "START1_ROLE" if i <= 1 else v for t, role in enumerate(self.role_set): score = 0 if i <= 0 else pi[i - 1][t][j][0] score += self.perceptron.get_score( self._make_features( (word, tag, chk, role), (pword1, ptag1, pchk1, prole1), (pword2, ptag2, pchk2, prole2), (fword1, ftag1, fchk1), (fword2, ftag2, fchk2), ), role, ) if score > pi[i][j][k][0]: pi[i][j][k][0] = score pi[i][j][k][1] = role pi[i][j][k][2] = t i = len(tagged) - 1 t, j = None, None for j, u in enumerate(self.role_set): for k, v in enumerate(self.role_set): tag, t = pi[i][j][k][1:3] tagged[i][3] = tag i -= 1 while i >= 0: tagged[i][3] = pi[i][t][j][1] j = t t = pi[i][t][j][2] i -= 1 printc(tagged) return tagged
def tag2(self, sent): tagged = [[self._normalize(word), None] for word in sent] nword = len(sent) ntag = len(self.chk_set) pi = [[[[0, None, None] for k in range(ntag)] for j in range(ntag)] for i in range(nword)] for i, (word, tag) in enumerate(tagged): pword1 = 'START1_WORD' if i <= 0 else tagged[i - 1][0] pword2 = 'START2_WORD' if i <= 1 else tagged[i - 2][0] fword1 = 'END1_WORD' if i >= len(sent) - 1 else tagged[i + 1][0] fword2 = 'END2_WORD' if i >= len(sent) - 2 else tagged[i + 2][0] for j, u in enumerate(self.chk_set): ptag2 = 'START1_TAG' if i <= 0 else u for k, v in enumerate(self.chk_set): ptag1 = 'START2_TAG' if i <= 1 else v for t, tag in enumerate(self.chk_set): score = 0 if i <= 0 else pi[i - 1][t][j][0] score += self.perceptron.get_score( self._make_features((word, tag), (pword1, ptag1), (pword2, ptag2), fword1, fword2), tag) if score > pi[i][j][k][0]: pi[i][j][k][0] = score pi[i][j][k][1] = tag pi[i][j][k][2] = t i = len(tagged) - 1 t, j = None, None for j, u in enumerate(self.chk_set): for k, v in enumerate(self.chk_set): tag, t = pi[i][j][k][1:3] tagged[i][1] = tag i -= 1 while i >= 0: tagged[i][1] = pi[i][t][j][1] j = t t = pi[i][t][j][2] i -= 1 printc(tagged) return tagged
def get_result(img_path, classID, time, iso): in_path = img_path ref_path = '00000000000' # taking the place of ground truth path, ignore it X_img, _, wb = ic._main_pre_process(in_path, ref_path, 00000000, classID=classID, time=time, iso=iso) #image channel order: R G1 B G2 X_img = np.expand_dims(X_img, axis=0) #from file's Image Array wb = np.expand_dims(wb, axis=0) #from file's EXIF info iso = np.expand_dims(iso, axis=0) #from file's EXIF info time = np.expand_dims(time, axis=0) #from file's EXIF info #execute y_hat = sess.run( [Yhat_im_output_Graph], feed_dict={ img_input_Graph: X_img, wb_graph: wb, inp_iso_Graph: iso, inp_t_Graph: time, }) #show and save output = np.minimum(np.maximum(y_hat[0], 0), 1) output = cv2.cvtColor(output[0, :, :, :], cv2.COLOR_RGB2BGR) print('note: there have to be a test_result1 folder!') cv2.imwrite( './test_result1/' + os.path.basename(img_path) + 'enhanced.jpg', output * 255) hp.printc( './test_result1/' + os.path.basename(img_path) + 'enhanced.jpg\n', 'green') cv2.namedWindow('OUT', cv2.WINDOW_NORMAL) cv2.imshow('OUT', output) cv2.waitKey(1000)
#step three load network parameters logging.info("step three load network parameters") epoch_pickup = int(hp.read_epoch()) sess.run(tf.global_variables_initializer()) saver = tf.train.Saver() ckpt = tf.train.get_checkpoint_state(checkpoint_dir) if ckpt: print('loaded ' + ckpt.model_checkpoint_path) saver.restore(sess, ckpt.model_checkpoint_path) else: hp.update_epoch(0) epoch_pickup = 0 printc('new model,epoch 0\n','green') decend_log = open('decend_log.txt','a') so.init_show_window() #step four start trainning and save logging.info("step four start trainning and save") one_epoch_time = -1.0 start_lr = 0.000200 end_lr = 0.000010 for epoch in range(epoch_pickup,pa.epoch_togo): time_start=time.time() logging.info("epoch:%d starting",epoch)
ckpt = tf.train.get_checkpoint_state(checkpoint_dir) if ckpt: # pick up from where it quit print('loaded ' + ckpt.model_checkpoint_path) saver.restore(sess, ckpt.model_checkpoint_path) else: # first time to run hp.update_epoch(0) epoch_pickup = 0 variables_to_restore = tf.contrib.framework.get_variables_to_restore( include=['prefix_net', 'quality_pri_net']) saver_temp = tf.train.Saver(variables_to_restore) ckpt2 = tf.train.get_checkpoint_state('./checkpoint/ESN_only/') saver_temp.restore(sess, ckpt2.model_checkpoint_path) printc('partial parameter loaded\n', 'green') saver.save(sess, checkpoint_dir + 'model.ckpt') printc('new model,epoch 0\n', 'green') decend_log = open('decend_log_step2.txt', 'a') so.init_show_window() #step four start trainning and save logging.info("step four start trainning and save") one_epoch_time = -1.0 start_lr = 0.000100 end_lr = 0.000010 for epoch in range(epoch_pickup, pa.epoch_togo): time_start = time.time() logging.info("epoch:%d starting", epoch)
def token2props(token_path, word_path, dest_path): wordfile = open(word_path, 'r') tokenfile = open(token_path, 'r') destfile = open(dest_path, 'w') sents = [] words = [] while True: line = wordfile.readline() if line == '': break word = line.rstrip('\n') if word == '': sents.append(words) words = [] else: words.append(word) print len(sents) records = [] tagged_sents = [] nline = 0 while True: line = tokenfile.readline() nline += 1 if line == '': break line = line.rstrip('\n') if line == '': if len(tagged_sents) == 0: tagged_sents.append([records]) else: last = tagged_sents[-1][-1] same = True if len(last) != len(records): same = False else: if last[0][0] == '特区' and records[0][0] == '保税区': same = False print 'here' printc(records) else: for idx,record in enumerate(last): if record[2] != records[idx][2]: same = False break if same: tagged_sents[-1].append(records) else: tagged_sents.append([records]) records = [] else: word, pos, chunk, role = line.split('\t') records.append([word, role, chunk]) print len(tagged_sents) if len(tagged_sents) != len(sents): for idx, (sent1, sent2) in enumerate(zip(sents, tagged_sents)): # sys.stdin.readline() # print idx # print 1, # printc(sent1) # print 2, # printc([word for (word, _, _) in sent2[0]]) pass # printc(tagged_sents[297][1]) # printc(tagged_sents[297][2]) # printc(tagged_sents[0:10]) finals = [] for idx, tsents in enumerate(tagged_sents): final = [[] for i in range(len(tsents))] for idx2, sent in enumerate(tsents): i, j = 0, 0 # printc(sent) while i < len(sent): word, role = sent[i][:2] if role != 'O': role = role[2:] first = False while True: if sents[idx][j] == word: final[idx2].append(role) j += 1 break else: if role == 'V': final[idx2].append('O') else: final[idx2].append(role) j += 1 i += 1 for k in range(j, len(sents[idx])): final[idx2].append('O') # print len(sents[idx]), len(final[idx2]) # printc(sents[idx]) # print final[idx2] assert(len(sents[idx]) == len(final[idx2])) finals.append(final) # print idx2 # for f, v in zip(finals[:10], sents[:10]): # printc(f) # printc(v) print len(finals) for idx, sent in enumerate(sents): for idx2, word in enumerate(sent): lastone = [] nextone = [] thisone = [roles[idx2] for roles in finals[idx]] if idx2-1 < 0: lastone = ['START'] * len(finals[idx]) else: lastone = [roles[idx2-1] for roles in finals[idx]] if idx2+1 >= len(finals[idx][0]): nextone = ['END'] * len(finals[idx]) else: nextone = [roles[idx2+1] for roles in finals[idx]] for k, one in enumerate(thisone): if one == 'O': thisone[k] = '*' else: if lastone[k] != one: thisone[k] = '(' + one + '*' if nextone[k] != one: thisone[k] += one + ')' else: if nextone[k] != one: thisone[k] = '*' + ')' else: thisone[k] = '*' s = '\t'.join(thisone) word = word if '(V*V)' in s else '-' destfile.write('%s\t%s\n' % (word, s)) destfile.write('\n') destfile.close() wordfile.close() tokenfile.close()