def tag2(self, sent):
tagged = [[self._normalize(word), pos, chk, None] for word, pos, chk in sent]
nword = len(sent)
ntag = len(self.role_set)
pi = [[[[0, None, None] for k in range(ntag)] for j in range(ntag)] for i in range(nword)]
for i, (word, tag, chk, role) in enumerate(tagged):
pword1, ptag1, pchk1 = ("START1_WORD", "START1_TAG", "START1_CHK") if i <= 0 else tagged[i - 1][:3]
pword2, ptag2, pchk2 = ("START2_WORD", "START2_TAG", "START2_CHK") if i <= 1 else tagged[i - 2][:3]
word, tag, chk = tagged[i][:3]
fword1, ftag1, fchk1 = ("END1_WORD", "END1_TAG", "END1_CHK") if i >= len(tagged) - 1 else tagged[i + 1][:3]
fword2, ftag2, fchk2 = ("END2_WORD", "END2_TAG", "END2_CHK") if i >= len(tagged) - 2 else tagged[i + 2][:3]
for j, u in enumerate(self.role_set):
prole2 = "START2_ROLE" if i <= 0 else u
for k, v in enumerate(self.role_set):
prole1 = "START1_ROLE" if i <= 1 else v
for t, role in enumerate(self.role_set):
score = 0 if i <= 0 else pi[i - 1][t][j][0]
score += self.perceptron.get_score(
self._make_features(
(word, tag, chk, role),
(pword1, ptag1, pchk1, prole1),
(pword2, ptag2, pchk2, prole2),
(fword1, ftag1, fchk1),
(fword2, ftag2, fchk2),
),
role,
)
if score > pi[i][j][k][0]:
pi[i][j][k][0] = score
pi[i][j][k][1] = role
pi[i][j][k][2] = t
i = len(tagged) - 1
t, j = None, None
for j, u in enumerate(self.role_set):
for k, v in enumerate(self.role_set):
tag, t = pi[i][j][k][1:3]
tagged[i][3] = tag
i -= 1
while i >= 0:
tagged[i][3] = pi[i][t][j][1]
j = t
t = pi[i][t][j][2]
i -= 1
printc(tagged)
return tagged
def tag2(self, sent):
tagged = [[self._normalize(word), None] for word in sent]
nword = len(sent)
ntag = len(self.chk_set)
pi = [[[[0, None, None] for k in range(ntag)] for j in range(ntag)]
for i in range(nword)]
for i, (word, tag) in enumerate(tagged):
pword1 = 'START1_WORD' if i <= 0 else tagged[i - 1][0]
pword2 = 'START2_WORD' if i <= 1 else tagged[i - 2][0]
fword1 = 'END1_WORD' if i >= len(sent) - 1 else tagged[i + 1][0]
fword2 = 'END2_WORD' if i >= len(sent) - 2 else tagged[i + 2][0]
for j, u in enumerate(self.chk_set):
ptag2 = 'START1_TAG' if i <= 0 else u
for k, v in enumerate(self.chk_set):
ptag1 = 'START2_TAG' if i <= 1 else v
for t, tag in enumerate(self.chk_set):
score = 0 if i <= 0 else pi[i - 1][t][j][0]
score += self.perceptron.get_score(
self._make_features((word, tag), (pword1, ptag1),
(pword2, ptag2), fword1,
fword2), tag)
if score > pi[i][j][k][0]:
pi[i][j][k][0] = score
pi[i][j][k][1] = tag
pi[i][j][k][2] = t
i = len(tagged) - 1
t, j = None, None
for j, u in enumerate(self.chk_set):
for k, v in enumerate(self.chk_set):
tag, t = pi[i][j][k][1:3]
tagged[i][1] = tag
i -= 1
while i >= 0:
tagged[i][1] = pi[i][t][j][1]
j = t
t = pi[i][t][j][2]
i -= 1
printc(tagged)
return tagged
def get_result(img_path, classID, time, iso):
in_path = img_path
ref_path = '00000000000' # taking the place of ground truth path, ignore it
X_img, _, wb = ic._main_pre_process(in_path,
ref_path,
00000000,
classID=classID,
time=time,
iso=iso)
#image channel order: R G1 B G2
X_img = np.expand_dims(X_img, axis=0) #from file's Image Array
wb = np.expand_dims(wb, axis=0) #from file's EXIF info
iso = np.expand_dims(iso, axis=0) #from file's EXIF info
time = np.expand_dims(time, axis=0) #from file's EXIF info
#execute
y_hat = sess.run(
[Yhat_im_output_Graph],
feed_dict={
img_input_Graph: X_img,
wb_graph: wb,
inp_iso_Graph: iso,
inp_t_Graph: time,
})
#show and save
output = np.minimum(np.maximum(y_hat[0], 0), 1)
output = cv2.cvtColor(output[0, :, :, :], cv2.COLOR_RGB2BGR)
print('note: there have to be a test_result1 folder!')
cv2.imwrite(
'./test_result1/' + os.path.basename(img_path) + 'enhanced.jpg',
output * 255)
hp.printc(
'./test_result1/' + os.path.basename(img_path) + 'enhanced.jpg\n',
'green')
cv2.namedWindow('OUT', cv2.WINDOW_NORMAL)
cv2.imshow('OUT', output)
cv2.waitKey(1000)
#step three load network parameters
logging.info("step three load network parameters")
epoch_pickup = int(hp.read_epoch())
sess.run(tf.global_variables_initializer())
saver = tf.train.Saver()
ckpt = tf.train.get_checkpoint_state(checkpoint_dir)
if ckpt:
print('loaded ' + ckpt.model_checkpoint_path)
saver.restore(sess, ckpt.model_checkpoint_path)
else:
hp.update_epoch(0)
epoch_pickup = 0
printc('new model,epoch 0\n','green')
decend_log = open('decend_log.txt','a')
so.init_show_window()
#step four start trainning and save
logging.info("step four start trainning and save")
one_epoch_time = -1.0
start_lr = 0.000200
end_lr = 0.000010
for epoch in range(epoch_pickup,pa.epoch_togo):
time_start=time.time()
logging.info("epoch:%d starting",epoch)
ckpt = tf.train.get_checkpoint_state(checkpoint_dir)
if ckpt: # pick up from where it quit
print('loaded ' + ckpt.model_checkpoint_path)
saver.restore(sess, ckpt.model_checkpoint_path)
else: # first time to run
hp.update_epoch(0)
epoch_pickup = 0
variables_to_restore = tf.contrib.framework.get_variables_to_restore(
include=['prefix_net', 'quality_pri_net'])
saver_temp = tf.train.Saver(variables_to_restore)
ckpt2 = tf.train.get_checkpoint_state('./checkpoint/ESN_only/')
saver_temp.restore(sess, ckpt2.model_checkpoint_path)
printc('partial parameter loaded\n', 'green')
saver.save(sess, checkpoint_dir + 'model.ckpt')
printc('new model,epoch 0\n', 'green')
decend_log = open('decend_log_step2.txt', 'a')
so.init_show_window()
#step four start trainning and save
logging.info("step four start trainning and save")
one_epoch_time = -1.0
start_lr = 0.000100
end_lr = 0.000010
for epoch in range(epoch_pickup, pa.epoch_togo):
time_start = time.time()
logging.info("epoch:%d starting", epoch)
def token2props(token_path, word_path, dest_path):
wordfile = open(word_path, 'r')
tokenfile = open(token_path, 'r')
destfile = open(dest_path, 'w')
sents = []
words = []
while True:
line = wordfile.readline()
if line == '':
break
word = line.rstrip('\n')
if word == '':
sents.append(words)
words = []
else:
words.append(word)
print len(sents)
records = []
tagged_sents = []
nline = 0
while True:
line = tokenfile.readline()
nline += 1
if line == '':
break
line = line.rstrip('\n')
if line == '':
if len(tagged_sents) == 0:
tagged_sents.append([records])
else:
last = tagged_sents[-1][-1]
same = True
if len(last) != len(records):
same = False
else:
if last[0][0] == '特区' and records[0][0] == '保税区':
same = False
print 'here'
printc(records)
else:
for idx,record in enumerate(last):
if record[2] != records[idx][2]:
same = False
break
if same:
tagged_sents[-1].append(records)
else:
tagged_sents.append([records])
records = []
else:
word, pos, chunk, role = line.split('\t')
records.append([word, role, chunk])
print len(tagged_sents)
if len(tagged_sents) != len(sents):
for idx, (sent1, sent2) in enumerate(zip(sents, tagged_sents)):
# sys.stdin.readline()
# print idx
# print 1,
# printc(sent1)
# print 2,
# printc([word for (word, _, _) in sent2[0]])
pass
# printc(tagged_sents[297][1])
# printc(tagged_sents[297][2])
# printc(tagged_sents[0:10])
finals = []
for idx, tsents in enumerate(tagged_sents):
final = [[] for i in range(len(tsents))]
for idx2, sent in enumerate(tsents):
i, j = 0, 0
# printc(sent)
while i < len(sent):
word, role = sent[i][:2]
if role != 'O':
role = role[2:]
first = False
while True:
if sents[idx][j] == word:
final[idx2].append(role)
j += 1
break
else:
if role == 'V':
final[idx2].append('O')
else:
final[idx2].append(role)
j += 1
i += 1
for k in range(j, len(sents[idx])):
final[idx2].append('O')
# print len(sents[idx]), len(final[idx2])
# printc(sents[idx])
# print final[idx2]
assert(len(sents[idx]) == len(final[idx2]))
finals.append(final)
# print idx2
# for f, v in zip(finals[:10], sents[:10]):
# printc(f)
# printc(v)
print len(finals)
for idx, sent in enumerate(sents):
for idx2, word in enumerate(sent):
lastone = []
nextone = []
thisone = [roles[idx2] for roles in finals[idx]]
if idx2-1 < 0:
lastone = ['START'] * len(finals[idx])
else:
lastone = [roles[idx2-1] for roles in finals[idx]]
if idx2+1 >= len(finals[idx][0]):
nextone = ['END'] * len(finals[idx])
else:
nextone = [roles[idx2+1] for roles in finals[idx]]
for k, one in enumerate(thisone):
if one == 'O':
thisone[k] = '*'
else:
if lastone[k] != one:
thisone[k] = '(' + one + '*'
if nextone[k] != one:
thisone[k] += one + ')'
else:
if nextone[k] != one:
thisone[k] = '*' + ')'
else:
thisone[k] = '*'
s = '\t'.join(thisone)
word = word if '(V*V)' in s else '-'
destfile.write('%s\t%s\n' % (word, s))
destfile.write('\n')
destfile.close()
wordfile.close()
tokenfile.close()
