def __init__(self):
self.config = Config()
self.reader = Reader()
self.layer = self.setup_layer()
self.loss = self.setup_loss()
self.metrics = self.setup_metrics()
self.merged = tf.summary.merge_all()
self.train_writer = tf.summary.FileWriter(self.config.root + '/event_detect/summary/cnn')
def __init__(self, g, saveFile, init_emb_file=None):
self.config = Config()
self.g = g
self.init_emb_file = init_emb_file
self.saveFile = saveFile
self.reader = Reader(self.g)
self.layer = self.setup_layer()
self.train_op = self.setup_train_op()
self.test_metrics = self.get_test_metrics()
self.loss_train_merged = tf.summary.merge(tf.get_collection('loss_train_summary'))
self.test_merged = tf.summary.merge(tf.get_collection('multi_label_classification'))
self.train_writer = tf.summary.FileWriter('model/summary/{}'.format(self.saveFile))
def __init__(self):
self.config = Config()
self.step_size = self.config.cblstm_step_size
self.reader = Reader()
self.layer = self.setup_layer()
self.loss = self.setup_loss()
self.train_op = self.setup_train_op()
self.train_metrics = self.setup_metrics(True)
self.test_metrics = self.setup_metrics(False)
self.train_merged = tf.summary.merge(
tf.get_collection('train_summary'))
self.train_metrics_merged = tf.summary.merge(
tf.get_collection('train_metrics_summary'))
self.test_metrics_merged = tf.summary.merge(
tf.get_collection('test_metrics_summary'))
self.train_writer = tf.summary.FileWriter('summary/cblstm')
def load_model():
#
cropper = Cropper()
#
detector = Detector(config.DETECTOR_CFG, config.DETECTOR_WEIGHT, config.DETECTOR_LABELS)
#
reader_config = Cfg.load_config_from_file(config.READER_CFG)
reader_config['weights'] = config.READER_WEIGHT
reader_config['device'] = config.DEVICE
reader = Reader(reader_config)
#
return cropper, detector, reader
def test():
path = DEFAULT_PATH
logging.basicConfig(filename='run.log', filemode='w', level=logging.DEBUG)
logger = logging.getLogger('packet_extractor')
parser = argparse.ArgumentParser()
parser.add_argument('-v',
'--verbose',
help='increase output verbosity',
action='store_true')
parser.add_argument('-p', '--port', help='port on which esp is running')
args = parser.parse_args()
if args.verbose:
logger.setLevel(logging.DEBUG)
logger.debug('Verbose mode activated')
if args.port:
path = args.port
reader = Reader(path=path)
reader.open_reader()
logger.info("Starting read")
with open("trace.txt", 'w') as tracer:
while 1:
try:
data = reader.get_data()
print(data.decode("utf-8"))
tracer.write(data.decode("utf-8"))
except KeyboardInterrupt:
logging.error("Keyboard quit")
break
except IndexError:
tracer.flush()
logging.error("Logger error")
pass
except serial.serialutil.SerialException as err:
tracer.flush()
logger.error("Serial exception")
logger.error(err)
pass
reader.close_reader()
logging.info('Closing read')
def main(model_name, new_scan=False, preprocess=True):
reader = Reader()
config = Config()
reader.aftername.sort()
if new_scan == True:
print('start new scan!')
file_list = reader.aftername
start_point = 0
else:
with open('detect_result/' + model_name + '/checkpoint') as file:
start_point = int(file.readline())
file_list = reader.aftername[start_point:]
print('restart from {}'.format(file_list[0]))
if model_name == 'cnn':
from event_detect.cnn import CNN
import tensorflow as tf
from tflib.models import Model
model = CNN()
sess = tf.Session()
saver, global_step = Model.continue_previous_session(
sess, model_file='cnn', ckpt_file='saver/cnn/checkpoint')
if model_name == 'cldnn':
from event_detect.cldnn import CLDNN
import tensorflow as tf
from tflib.models import Model
model = CLDNN()
sess = tf.Session()
saver, global_step = Model.continue_previous_session(
sess, model_file='cldnn', ckpt_file='saver/cldnn/checkpoint')
for file in file_list:
begin = datetime.datetime.now()
traces = obspy.read(file[0])
traces = traces + obspy.read(file[1])
traces = traces + obspy.read(file[2])
if not (traces[0].stats.starttime == traces[1].stats.starttime
and traces[0].stats.starttime == traces[2].stats.starttime):
starttime = max([
traces[0].stats.starttime, traces[1].stats.starttime,
traces[2].stats.starttime
])
for j in range(3):
traces[j] = traces[j].slice(starttime=starttime)
if not (traces[0].stats.endtime == traces[1].stats.endtime
and traces[0].stats.endtime == traces[2].stats.endtime):
endtime = min([
traces[0].stats.endtime, traces[1].stats.endtime,
traces[2].stats.endtime
])
for j in range(3):
traces[j] = traces[j].slice(endtime=endtime)
start_flag = -1
end_flag = -1
event_list = []
for windowed_st in traces.slide(window_length=(config.winsize - 1) /
100.0,
step=config.winlag / 100.0):
data_input = []
for j in range(3):
data_input.append(windowed_st[j].data)
if model_name == 'cnn':
# raw_data = [data_preprocess(d, 'bandpass', False) for d in data_input]
data_input = np.array(data_input).T
if preprocess:
# data_input = sklearn.preprocessing.minmax_scale(data_input)
data_mean = np.mean(data_input, axis=0)
data_input = np.absolute(data_input - data_mean)
data_input = data_input / (np.max(data_input, axis=0) +
np.array([1, 1, 1]))
data_input = np.array([np.array([data_input])])
elif model_name == 'cldnn':
# raw_data = [data_preprocess(d, 'bandpass', False) for d in data_input]
data_input = [data_preprocess(d) for d in data_input]
data_input = np.array(data_input).T
data_input = np.array([data_input])
class_pred, confidence = model.classify(sess=sess,
input_=data_input)
if class_pred == 1:
# plt.subplot(3, 1, 1)
# plt.plot(raw_data[0])
# plt.subplot(3, 1, 2)
# plt.plot(raw_data[1])
# plt.subplot(3, 1, 3)
# plt.plot(raw_data[2])
# plt.show()
if start_flag == -1:
start_flag = windowed_st[0].stats.starttime
end_flag = windowed_st[0].stats.endtime
else:
end_flag = windowed_st[0].stats.endtime
if class_pred == 0 and start_flag != -1 and end_flag < windowed_st[
0].stats.starttime:
event = [
file[0].split('\\')[-1][:-4], start_flag, end_flag,
confidence
]
# print(event)
event_list.append(event)
start_flag = -1
end_flag = -1
if len(event_list) != 0:
with open('detect_result/' + model_name + '/events_test.csv',
mode='a',
newline='') as f:
csvwriter = csv.writer(f)
for event in event_list:
csvwriter.writerow(event)
f.close()
start_point += 1
with open('detect_result/' + model_name + '/checkpoint',
mode='w') as f:
f.write(str(start_point))
end = datetime.datetime.now()
print('{} scanned, num {}, time {}.'.format(
file[0].split('\\')[-1][:-4], start_point, end - begin))
print('checkpoint saved.')
for ent in labels:
start_dic[(ent[0], ent[2])].append(ent)
end_dic[(ent[1], ent[2])].append(ent)
all_num += 1
for k, v in start_dic.items():
if len(v) > 1:
start_num += len(v)
for k, v in end_dic.items():
if len(v) > 1:
end_num += len(v)
print("All {}, start {}, end {}".format(all_num, start_num, end_num))
if __name__ == "__main__":
reader = Reader()
reader.read_and_gen_vectors_glove(config.embed_path)
reader.read_all_data("./data/ace2005/", "ace2005.train", "ace2005.dev",
"ace2005.test")
# print reader.train_sents[0]
train_batches, dev_batches, test_batches = reader.to_batch(
config.batch_size)
f = open(config.train_data_path, 'wb')
pickle.dump(train_batches, f)
f.close()
f = open(config.dev_data_path, 'wb')
pickle.dump(dev_batches, f)
f.close()
for ent in labels:
start_dic[(ent[0], ent[2])].append(ent)
end_dic[(ent[1], ent[2])].append(ent)
all_num += 1
for k, v in start_dic.items():
if len(v) > 1:
start_num += len(v)
for k, v in end_dic.items():
if len(v) > 1:
end_num += len(v)
print("All {}, start {}, end {}".format(all_num, start_num, end_num))
if __name__ == "__main__":
reader = Reader()
reader.read_all_data("./data/ace2005/", "ace2005.train", "ace2005.dev",
"ace2005.test")
# print reader.train_sents[0]
train_batches, dev_batches, test_batches = reader.to_batch(
config.batch_size)
f = open(config.train_data_path, 'wb')
pickle.dump(train_batches, f)
f.close()
f = open(config.dev_data_path, 'wb')
pickle.dump(dev_batches, f)
f.close()
f = open(config.test_data_path, 'wb')
for ent in labels:
start_dic[(ent[0], ent[2])].append(ent)
end_dic[(ent[1], ent[2])].append(ent)
all_num += 1
for k, v in start_dic.items():
if len(v) > 1:
start_num += len(v)
for k, v in end_dic.items():
if len(v) > 1:
end_num += len(v)
print("All {}, start {}, end {}".format(all_num, start_num, end_num))
if __name__ == "__main__":
reader = Reader()
#reader.read_and_gen_vectors_pubmed_word2vec(config.embed_path)
reader.read_all_data("./data/genia/", "genia.train", "genia.dev", "genia.test")
# print reader.train_sents[0]
train_batches, dev_batches, test_batches = reader.to_batch(config.batch_size)
f = open(config.train_data_path, 'wb')
pickle.dump(train_batches, f)
f.close()
f = open(config.dev_data_path, 'wb')
pickle.dump(dev_batches, f)
f.close()
f = open(config.test_data_path, 'wb')
pickle.dump(test_batches, f)
start_num += len(v)
for k, v in end_dic.items():
if len(v) > 1:
end_num += len(v)
print("All {}, start {}, end {}".format(all_num, start_num, end_num))
if __name__ == "__main__":
tokenizer_dir = "tokenization/polish-roberta-large/"
tokenizer = SentencePieceBPETokenizer(f"{tokenizer_dir}/vocab.json",
f"{tokenizer_dir}/merges.txt")
getattr(tokenizer,
"_tokenizer").post_processor = RobertaProcessing(sep=("</s>", 2),
cls=("<s>", 0))
reader = Reader("polish", tokenizer, cls="<s>", sep="</s>", threshold=8)
reader.read_all_data("./data/poleval/", "poleval.train", "poleval.dev",
"poleval.test")
# print reader.train_sents[0]
train_batches, dev_batches, test_batches = reader.to_batch(
config.batch_size)
f = open(config.train_data_path, 'wb')
pickle.dump(train_batches, f)
f.close()
f = open(config.dev_data_path, 'wb')
pickle.dump(dev_batches, f)
f.close()
f = open(config.test_data_path, 'wb')
def main(model_name, new_scan=False, preprocess=True):
reader = Reader()
config = Config()
plot = config.plot
bandpass = config.bandpass
resample = config.resample
confidence0=[]
plot_num = 0
reader.aftername.sort()
if new_scan == True:
print('start new scan!')
file_list = reader.aftername
start_point = 0
else:
with open(config.root + '/event_detect/detect_result/' + model_name + '/checkpoint') as file:
start_point = int(file.readline())
file_list = reader.aftername[start_point:]
print('restart from {}'.format(file_list[0]))
if model_name == 'cnn':
from cnn import CNN
import tensorflow as tf
from tflib.models import Model
model = CNN()
sess = tf.Session(config=tf.ConfigProto(device_count={"CPU":20},inter_op_parallelism_threads=0,intra_op_parallelism_threads=0))
saver, global_step = Model.continue_previous_session(sess,
model_file='cnn',
ckpt_file=config.root + '/event_detect/saver/cnn/checkpoint')
file_list_len = len(file_list)
# print(file_list_len)
try:
os.system('rm -rf %s/event_detect/detect_result/png/*'%config.root)
os.system('rm -rf %s/event_detect/detect_result/cnn/*.csv'%config.root)
except:
pass
for file in file_list:
file=np.array(file)
#print(file)
#file=file.T
#np.random.shuffle(file) #random
#file=file.T
#print(file,'\n')
begin = datetime.datetime.now()
if plot:
plot_traces = obspy.read(file[2][0]) #Z component
sta_num = len(file[0])
trace_len = []
for i in range(3):
for j in range(sta_num):
trace_len.append(obspy.read(file[i][j])[0].stats.npts)
max_len = max(trace_len)
for i in range(3):
for j in range(sta_num): # station number
each_tr = obspy.read(file[i][j])
if each_tr[0].stats.npts < max_len:
zero = np.zeros(max_len-each_tr[0].stats.npts)
each_tr[0].data = np.concatenate([each_tr[0].data,zero])
if i==j==0:
traces = each_tr
else:
traces=traces + each_tr
if i == 2:
if j == 0:
pass
else:
plot_traces = plot_traces + each_tr
if plot:
if resample:
plot_traces = plot_traces.resample(sampling_rate=resample)
plot_traces = plot_traces.filter('bandpass',freqmin=bandpass[0],freqmax=bandpass[1],corners=4,zerophase=True)
if resample:
traces = traces.resample(sampling_rate=resample)
traces = traces.filter('bandpass',freqmin=bandpass[0],freqmax=bandpass[1],corners=4,zerophase=True)
starttime = traces[0].stats.starttime;
endtime = traces[0].stats.endtime;
#print(traces)
start_flag = -1
end_flag = -1
event_list = []
confidence_total=[]
start_total=[]
end_total=[]
samples_trace= 1.0/traces[0].stats.delta;
npts = traces[0].stats.npts
for windowed_st in traces.slide(window_length=(config.winsize-1)/samples_trace,
step=config.winlag / samples_trace):
data_input = [[],[],[]]
for j in range(sta_num):
if len(windowed_st[j].data) < config.winsize:
windowed_st[j].data = np.concatenate([windowed_st[j].data,np.zeros(config.winsize-len(windowed_st[j].data))])
data_input[0].append(windowed_st[j].data[:config.winsize])
for j in range(sta_num,2*sta_num):
if len(windowed_st[j].data) < config.winsize:
windowed_st[j].data = np.concatenate([windowed_st[j].data,np.zeros(config.winsize-len(windowed_st[j].data))])
data_input[1].append(windowed_st[j].data[:config.winsize])
for j in range(2*sta_num,3*sta_num):
if len(windowed_st[j].data) < config.winsize:
windowed_st[j].data = np.concatenate([windowed_st[j].data,np.zeros(config.winsize-len(windowed_st[j].data))])
data_input[2].append(windowed_st[j].data[:config.winsize])
if model_name == 'cnn':
if preprocess:
for i in range(3):
for j in range(sta_num):
data_input[i][j] = data_preprocess(data_input[i][j])
data_input=np.array(data_input)
if len(data_input[0][0])<config.winsize:
concat = np.zeros([3, sta_num, config.winsize - len(data_input[0][0])])
data_input=np.concatenate([data_input,concat],axis=2)
if len(data_input[0][0])>config.winsize:
data_input=data_input[:, :, :config.winsize]
data_input=data_input.transpose((1,2,0))
data_input = np.array([data_input])
#print(event_list)
class_pred, confidence = model.classify(sess=sess, input_=data_input)
confidence0.append(confidence)
print(class_pred,confidence)
if class_pred == 1:
confidence_total.append(confidence)
start_total.append(windowed_st[0].stats.starttime)
end_total.append(windowed_st[0].stats.endtime)
if start_flag == -1:
start_flag = windowed_st[0].stats.starttime
end_flag = windowed_st[0].stats.endtime
else:
end_flag = windowed_st[0].stats.endtime
print(class_pred,start_flag,end_flag,windowed_st[0].stats.starttime)
if class_pred == 0 and start_flag != -1: #end_flag < windowed_st[0].stats.starttime:
confidence = np.max(confidence_total)
for j in range(len(confidence_total)):
if confidence == confidence_total[j]:
break
start_local = start_total[j]
end_local = end_total[j]
a=True
# event = [file[0][0].split('/')[-2], start_flag, end_flag,
# confidence, start_local, end_local]
event = [file[0][0].split('/')[-2], start_flag, end_flag, confidence]
confidence_total=[]
start_total = []
end_total = []
if plot:
plot_num = int(plot_num + 1)
name = config.root + '/event_detect/detect_result/png/' \
+ str(plot_num) + '_' + str(confidence) + '.png'
plot_traces.plot(starttime=start_flag, endtime=end_flag, size=(800, 800),
automerge=False, equal_scale=False, linewidth=0.8, outfile=name)
# print(event)
event_list.append(event)
#print(event_list)
start_flag = -1
end_flag = -1
if class_pred == 1 and end_flag+config.winlag / samples_trace>=endtime:
confidence = np.max(confidence_total)
for j in range(len(confidence_total)):
if confidence == confidence_total[j]:
break
start_local = start_total[j]
end_local = end_total[j]
if plot:
plot_num = int(plot_num + 1)
name = config.root + '/event_detect/detect_result/png/' \
+ str(plot_num) + '_' + str(confidence) + '.png'
plot_traces.plot(starttime=start_flag, endtime=endtime, size=(800, 800),
automerge=False, equal_scale=False, linewidth=0.8, outfile=name)
# event = [file[0][0].split('/')[-2], start_flag, endtime,
# confidence, start_total, end_total]
event = [file[0][0].split('/')[-2], start_flag, endtime, confidence]
event_list.append(event)
start_flag = -1
end_flag = -1
if len(event_list) != 0:
with open(config.root + '/event_detect/detect_result/' + model_name + '/events_list.csv', mode='a', newline='') as f:
csvwriter = csv.writer(f)
for event in event_list:
csvwriter.writerow(event)
f.close()
start_point += 1
with open(config.root + '/event_detect/detect_result/' + model_name + '/checkpoint', mode='w') as f:
f.write(str(start_point))
end = datetime.datetime.now()
print('{} scanned, num {}, time {}.'.format(file[0][0].split('/')[-2], start_point, end - begin))
print('checkpoint saved.')
for ent in labels:
start_dic[(ent[0], ent[2])].append(ent)
end_dic[(ent[1], ent[2])].append(ent)
all_num += 1
for k, v in start_dic.items():
if len(v) > 1:
start_num += len(v)
for k, v in end_dic.items():
if len(v) > 1:
end_num += len(v)
print("All {}, start {}, end {}".format(all_num, start_num, end_num))
if __name__ == "__main__":
reader = Reader()
reader.read_and_gen_vectors_pubmed_word2vec(config.embed_path)
reader.read_all_data("./data/genia_sample/", "train.data", "dev.data", "test.data")
# print reader.train_sents[0]
train_batches, dev_batches, test_batches = reader.to_batch(config.batch_size)
f = open(config.train_data_path, 'wb')
pickle.dump(train_batches, f)
f.close()
f = open(config.dev_data_path, 'wb')
pickle.dump(dev_batches, f)
f.close()
f = open(config.test_data_path, 'wb')
pickle.dump(test_batches, f)
class CNN(object):
def __init__(self):
self.config = Config()
self.reader = Reader()
self.layer = self.setup_layer()
self.loss = self.setup_loss()
self.metrics = self.setup_metrics()
self.merged = tf.summary.merge_all()
self.train_writer = tf.summary.FileWriter(self.config.root +
'/event_detect/summary/cnn')
def setup_layer(self):
layer = dict()
layer['target'] = tf.placeholder(tf.int32, shape=[None], name='target')
layer['input'] = tf.placeholder(
tf.float32,
shape=[None, None, self.config.winsize, 3],
name='input')
layer['conv1'] = layers.conv(
layer['input'],
filter=[3, 3, 3, 8],
# strides=[1, 1, 4, 1],
strides=[1, 1, 1, 1],
padding='SAME',
wd=0.001,
bias=0.0,
name='conv1')
layer['pooling1'] = layers.pool(layer['conv1'],
ksize=[1, 1, 3, 1],
strides=[1, 1, 3, 1],
padding='SAME',
pool_func=tf.nn.max_pool,
name='pooling1')
layer['conv2'] = layers.conv(
layer['pooling1'],
filter=[3, 3, 8, 16],
# strides=[1, 1, 4, 1],
strides=[1, 1, 1, 1],
padding='SAME',
wd=0.001,
bias=0.0,
name='conv2')
layer['pooling2'] = layers.pool(layer['conv2'],
ksize=[1, 1, 3, 1],
strides=[1, 1, 3, 1],
padding='SAME',
pool_func=tf.nn.max_pool,
name='pooling2')
layer['conv3'] = layers.conv(
layer['pooling2'],
filter=[3, 3, 16, 32],
# strides=[1, 1, 4, 1],
strides=[1, 1, 1, 1],
padding='VALID',
wd=0.001,
bias=0.0,
name='conv3')
layer['pooling3'] = layers.pool(layer['conv3'],
ksize=[1, 1, 3, 1],
strides=[1, 1, 3, 1],
padding='SAME',
pool_func=tf.nn.max_pool,
name='pooling3')
layer['conv4'] = layers.conv(
layer['pooling3'],
filter=[1, 3, 32, 32],
# strides=[1, 1, 4, 1],
strides=[1, 1, 1, 1],
padding='SAME',
wd=0.001,
bias=0.0,
name='conv4')
layer['pooling4'] = layers.pool(layer['conv4'],
ksize=[1, 1, 3, 1],
strides=[1, 1, 3, 1],
padding='SAME',
pool_func=tf.nn.max_pool,
name='pooling4')
layer['conv5'] = layers.conv(
layer['pooling4'],
filter=[1, 3, 32, 64],
# strides=[1, 1, 4, 1],
strides=[1, 1, 1, 1],
padding='SAME',
wd=0.001,
bias=0.0,
name='conv5')
layer['pooling5'] = layers.pool(layer['conv5'],
ksize=[1, 1, 3, 1],
strides=[1, 1, 3, 1],
padding='SAME',
pool_func=tf.nn.max_pool,
name='pooling5')
layer['conv6'] = layers.conv(
layer['pooling5'],
filter=[1, 9, 64, 128],
# strides=[1, 1, 4, 1],
strides=[1, 1, 1, 1],
padding='VALID',
wd=0.001,
bias=0.0,
name='conv6')
layer['conv7'] = layers.conv(
layer['conv6'],
filter=[1, 1, 128, 128],
# strides=[1, 1, 4, 1],
strides=[1, 1, 1, 1],
padding='SAME',
wd=0.001,
bias=0.0,
name='conv7')
layer['conv8'] = layers.conv(
layer['conv7'],
filter=[1, 1, 128, 2],
# strides=[1, 1, 4, 1],
strides=[1, 1, 1, 1],
padding='SAME',
wd=0.001,
bias=0.0,
name='conv8')
layer['unfold'] = layers.unfold(layer['conv8'], name='unfold')
#layer['logits'] = tf.reduce_mean(layer['unfold'], 1, name='logits')
layer['class_prob'] = tf.nn.softmax(layer['unfold'], name='class_prob')
layer['class_prediction'] = tf.argmax(layer['class_prob'],
1,
name='class_pred')
return layer
def setup_loss(self):
with tf.name_scope('loss'):
raw_loss = tf.reduce_mean(
tf.nn.sparse_softmax_cross_entropy_with_logits(
logits=self.layer['unfold'], labels=self.layer['target']))
tf.summary.scalar('raw_loss', raw_loss) #tensorboard可视化
tf.add_to_collection('losses', raw_loss) #将raw_loss添加到losses中
loss = tf.add_n(tf.get_collection('losses'),
name='total_loss') #所有元素相加
tf.summary.scalar('total_loss', loss) #tensorboard可视化
return loss
def setup_metrics(self):
metrics = dict()
with tf.variable_scope('metrics'):
metrics['accuracy'] = tf.metrics.accuracy(
labels=self.layer['target'],
predictions=self.layer['class_prediction'],
name='accuracy')[1]
tf.summary.scalar('accuracy', metrics['accuracy'])
metrics['recall'] = tf.metrics.recall(
labels=self.layer['target'],
predictions=self.layer['class_prediction'],
name='recall')[1]
tf.summary.scalar('recall', metrics['recall'])
metrics['precision'] = tf.metrics.precision(
labels=self.layer['target'],
predictions=self.layer['class_prediction'],
name='precision')[1]
tf.summary.scalar('precision', metrics['precision'])
return metrics
def train(self, passes, new_training=True):
with tf.Session() as sess:
global_step = tf.Variable(0, trainable=False)
#learning_rate = tf.train.exponential_decay(0.001, global_step, 200, 0.8, staircase=True)
training = tf.train.AdamOptimizer(
self.config.learning_rate).minimize(self.loss)
if new_training:
saver, global_step = Model.start_new_session(sess)
else:
saver, global_step = Model.continue_previous_session(
sess,
model_file='cnn',
ckpt_file=self.config.root +
'/event_detect/saver/cnn/checkpoint')
sess.run(tf.local_variables_initializer())
self.train_writer.add_graph(sess.graph, global_step=global_step)
test_restlt = []
for step in range(1 + global_step, 1 + passes + global_step):
input, target = self.reader.get_cnn_batch_data('train')
#print(input.shape)
summary, _, acc = sess.run(
[self.merged, training, self.metrics['accuracy']],
feed_dict={
self.layer['input']: input,
self.layer['target']: target
})
self.train_writer.add_summary(summary, step)
if step % 10 == 0:
loss = sess.run(self.loss,
feed_dict={
self.layer['input']: input,
self.layer['target']: target
})
test_restlt.append(loss)
print(
"gobal_step {}, training_loss {}, accuracy {}".format(
step, loss, acc))
if step % 100 == 0:
test_x, text_y = self.reader.get_cnn_batch_data('test')
acc, recall, precision = sess.run([
self.metrics['accuracy'], self.metrics['recall'],
self.metrics['precision']
],
feed_dict={
self.layer['input']:
test_x,
self.layer['target']:
text_y
})
print("test: accuracy {}, recall {}, precision {}".format(
acc, recall, precision))
saver.save(sess,
self.config.root +
'/event_detect/saver/cnn/cnn',
global_step=step)
print('checkpoint saved')
#print(sess.run([self.layer['class_prob']], feed_dict={self.layer['input']: input}))
print(test_restlt)
def classify(self, sess, input_):
class_prediction, confidence = sess.run(
[self.layer['class_prediction'], self.layer['class_prob']],
feed_dict={self.layer['input']: input_})
confidence = confidence[:, 1]
if confidence > self.config.prob:
class_prediction = 1
else:
class_prediction = 0
return class_prediction, confidence
for ent in labels:
start_dic[(ent[0], ent[2])].append(ent)
end_dic[(ent[1], ent[2])].append(ent)
all_num += 1
for k, v in start_dic.items():
if len(v) > 1:
start_num += len(v)
for k, v in end_dic.items():
if len(v) > 1:
end_num += len(v)
print("All {}, start {}, end {}".format(all_num, start_num, end_num))
if __name__ == "__main__":
reader = Reader(config.bert_model)
reader.read_all_data("./data/ace2005/", "ace2005.train", "ace2005.dev",
"ace2005.test")
# print reader.train_sents[0]
train_batches, dev_batches, test_batches = reader.to_batch(
config.batch_size)
f = open(config.train_data_path, 'wb')
pickle.dump(train_batches, f)
f.close()
f = open(config.dev_data_path, 'wb')
pickle.dump(dev_batches, f)
f.close()
f = open(config.test_data_path, 'wb')
for ent in labels:
start_dic[(ent[0], ent[2])].append(ent)
end_dic[(ent[1], ent[2])].append(ent)
all_num += 1
for k, v in start_dic.items():
if len(v) > 1:
start_num += len(v)
for k, v in end_dic.items():
if len(v) > 1:
end_num += len(v)
print("All {}, start {}, end {}".format(all_num, start_num, end_num))
if __name__ == "__main__":
reader = Reader(config)
reader.read_all_data()
# print reader.train_sents[0]
train_batches, dev_batches, test_batches = reader.to_batch()
f = open(config.data_path + "_train.pkl", 'wb')
pickle.dump(train_batches, f)
f.close()
f = open(config.data_path + "_dev.pkl", 'wb')
pickle.dump(dev_batches, f)
f.close()
f = open(config.data_path + "_test.pkl", 'wb')
pickle.dump(test_batches, f)
f.close()
for ent in labels:
start_dic[(ent[0], ent[2])].append(ent)
end_dic[(ent[1], ent[2])].append(ent)
all_num += 1
for k, v in start_dic.items():
if len(v) > 1:
start_num += len(v)
for k, v in end_dic.items():
if len(v) > 1:
end_num += len(v)
print("All {}, start {}, end {}".format(all_num, start_num, end_num))
if __name__ == "__main__":
reader = Reader(config.bert_model)
reader.read_all_data("./data/genia_sample/", "train.data", "dev.data", "test.data")
# print reader.train_sents[0]
train_batches, dev_batches, test_batches = reader.to_batch(config.batch_size)
f = open(config.train_data_path, 'wb')
pickle.dump(train_batches, f)
f.close()
f = open(config.dev_data_path, 'wb')
pickle.dump(dev_batches, f)
f.close()
f = open(config.test_data_path, 'wb')
pickle.dump(test_batches, f)
f.close()
class CNN(object):
def __init__(self):
self.config = Config()
self.reader = Reader()
self.layer = self.setup_layer()
self.loss = self.setup_loss()
self.metrics = self.setup_metrics()
self.merged = tf.summary.merge_all()
self.train_writer = tf.summary.FileWriter('summary/cnn')
def setup_layer(self):
layer = dict()
layer['target'] = tf.placeholder(tf.int32, shape=[None], name='target')
layer['input'] = tf.placeholder(
tf.float32, shape=[None, 1, self.config.winsize, 3], name='input')
layer['conv1'] = layers.conv(
layer['input'],
filter=[1, 6, 3, 8],
# strides=[1, 1, 4, 1],
strides=[1, 1, 1, 1],
padding='SAME',
wd=1e-3,
bias=0.0,
name='conv1')
layer['pooling1'] = layers.pool(layer['conv1'],
ksize=[1, 1, 3, 1],
strides=[1, 1, 3, 1],
padding='SAME',
pool_func=tf.nn.max_pool,
name='pooling1')
layer['conv2'] = layers.conv(
layer['pooling1'],
filter=[1, 6, 8, 16],
# strides=[1, 1, 4, 1],
strides=[1, 1, 1, 1],
padding='SAME',
wd=1e-3,
bias=0.0,
name='conv2')
layer['pooling2'] = layers.pool(layer['conv2'],
ksize=[1, 1, 3, 1],
strides=[1, 1, 3, 1],
padding='SAME',
pool_func=tf.nn.max_pool,
name='pooling2')
layer['conv3'] = layers.conv(
layer['pooling2'],
filter=[1, 6, 16, 32],
# strides=[1, 1, 4, 1],
strides=[1, 1, 1, 1],
padding='SAME',
wd=1e-3,
bias=0.0,
name='conv3')
layer['pooling3'] = layers.pool(layer['conv3'],
ksize=[1, 1, 3, 1],
strides=[1, 1, 3, 1],
padding='SAME',
pool_func=tf.nn.max_pool,
name='pooling3')
layer['conv4'] = layers.conv(
layer['pooling3'],
filter=[1, 6, 32, 32],
# strides=[1, 1, 4, 1],
strides=[1, 1, 1, 1],
padding='SAME',
wd=1e-3,
bias=0.0,
name='conv4')
layer['pooling4'] = layers.pool(layer['conv4'],
ksize=[1, 1, 3, 1],
strides=[1, 1, 3, 1],
padding='SAME',
pool_func=tf.nn.max_pool,
name='pooling4')
layer['conv5'] = layers.conv(
layer['pooling4'],
filter=[1, 6, 32, 32],
# strides=[1, 1, 4, 1],
strides=[1, 1, 1, 1],
padding='SAME',
wd=1e-3,
bias=0.0,
name='conv5')
layer['pooling5'] = layers.pool(layer['conv5'],
ksize=[1, 1, 3, 1],
strides=[1, 1, 3, 1],
padding='SAME',
pool_func=tf.nn.max_pool,
name='pooling5')
layer['conv6'] = layers.conv(
layer['pooling5'],
filter=[1, 6, 32, 32],
# strides=[1, 1, 4, 1],
strides=[1, 1, 1, 1],
padding='SAME',
wd=1e-3,
bias=0.0,
name='conv6')
layer['pooling6'] = layers.pool(layer['conv6'],
ksize=[1, 1, 3, 1],
strides=[1, 1, 3, 1],
padding='SAME',
pool_func=tf.nn.max_pool,
name='pooling6')
layer['unfold'] = layers.Unfold(layer['pooling6'], name='unfold')
layer['logits'] = layers.fc(layer['unfold'], 2, wd=1e-3, name='logits')
layer['class_prob'] = tf.nn.softmax(layer['logits'], name='class_prob')
layer['class_prediction'] = tf.argmax(layer['class_prob'],
1,
name='class_pred')
return layer
def setup_loss(self):
with tf.name_scope('loss'):
raw_loss = tf.reduce_mean(
tf.nn.sparse_softmax_cross_entropy_with_logits(
logits=self.layer['logits'], labels=self.layer['target']))
tf.summary.scalar('raw_loss', raw_loss)
tf.add_to_collection('losses', raw_loss)
loss = tf.add_n(tf.get_collection('losses'), name='total_loss')
tf.summary.scalar('total_loss', loss)
return loss
def setup_metrics(self):
metrics = dict()
with tf.variable_scope('metrics'):
metrics['accuracy'] = tf.metrics.accuracy(
labels=self.layer['target'],
predictions=self.layer['class_prediction'],
name='accuracy')[1]
tf.summary.scalar('accuracy', metrics['accuracy'])
metrics['recall'] = tf.metrics.recall(
labels=self.layer['target'],
predictions=self.layer['class_prediction'],
name='recall')[1]
tf.summary.scalar('recall', metrics['recall'])
metrics['precision'] = tf.metrics.precision(
labels=self.layer['target'],
predictions=self.layer['class_prediction'],
name='precision')[1]
tf.summary.scalar('precision', metrics['precision'])
return metrics
def train(self, passes, new_training=True):
with tf.Session() as sess:
training = tf.train.AdamOptimizer(1e-3).minimize(self.loss)
if new_training:
saver, global_step = Model.start_new_session(sess)
else:
saver, global_step = Model.continue_previous_session(
sess, model_file='cnn', ckpt_file='saver/cnn/checkpoint')
sess.run(tf.local_variables_initializer())
self.train_writer.add_graph(sess.graph, global_step=global_step)
for step in range(1 + global_step, 1 + passes + global_step):
input, target = self.reader.get_cnn_batch_data('train')
summary, _, acc = sess.run(
[self.merged, training, self.metrics['accuracy']],
feed_dict={
self.layer['input']: input,
self.layer['target']: target
})
self.train_writer.add_summary(summary, step)
if step % 10 == 0:
loss = sess.run(self.loss,
feed_dict={
self.layer['input']: input,
self.layer['target']: target
})
print(
"gobal_step {}, training_loss {}, accuracy {}".format(
step, loss, acc))
if step % 100 == 0:
test_x, text_y = self.reader.get_cnn_batch_data('test')
acc, recall, precision = sess.run([
self.metrics['accuracy'], self.metrics['recall'],
self.metrics['precision']
],
feed_dict={
self.layer['input']:
test_x,
self.layer['target']:
text_y
})
print("test: accuracy {}, recall {}, precision {}".format(
acc, recall, precision))
saver.save(sess, 'saver/cnn/cnn', global_step=step)
print('checkpoint saved')
def classify(self, sess, input_):
# with tf.Session() as sess:
# saver, global_step = Model.continue_previous_session(sess,
# model_file='cnn',
# ckpt_file='saver/cnn/checkpoint')
class_prediction, confidence = sess.run(
[self.layer['class_prediction'], self.layer['class_prob']],
feed_dict={self.layer['input']: input_})
confidence = confidence[:, 1]
return class_prediction, confidence
from reader.reader import Reader
reader = Reader("../data/cifar_100_caffe_hdf5")
X,y = reader.getTrainData()
print X
print y
for ent in labels:
start_dic[(ent[0], ent[2])].append(ent)
end_dic[(ent[1], ent[2])].append(ent)
all_num += 1
for k, v in start_dic.items():
if len(v) > 1:
start_num += len(v)
for k, v in end_dic.items():
if len(v) > 1:
end_num += len(v)
print("All {}, start {}, end {}".format(all_num, start_num, end_num))
if __name__ == "__main__":
reader = Reader(config)
reader.read_all_data()
# print reader.train_sents[0]
train_batches, dev_batches, test_batches = reader.to_batch()
f = open(config.data_path + "_train.pkl", 'wb')
pickle.dump(train_batches, f)
f.close()
f = open(config.data_path + "_dev.pkl", 'wb')
pickle.dump(dev_batches, f)
f.close()
f = open(config.data_path + "_test.pkl", 'wb')
pickle.dump(test_batches, f)
f.close()
class CBLSTM(object):
def __init__(self):
self.config = Config()
self.step_size = self.config.cblstm_step_size
self.reader = Reader()
self.layer = self.setup_layer()
self.loss = self.setup_loss()
self.train_op = self.setup_train_op()
self.train_metrics = self.setup_metrics(True)
self.test_metrics = self.setup_metrics(False)
self.train_merged = tf.summary.merge(
tf.get_collection('train_summary'))
self.train_metrics_merged = tf.summary.merge(
tf.get_collection('train_metrics_summary'))
self.test_metrics_merged = tf.summary.merge(
tf.get_collection('test_metrics_summary'))
self.train_writer = tf.summary.FileWriter('summary/cblstm')
def data_padding_preprocess(self, data, data_type):
step_size = self.step_size
sequence_len = list(map(len, data))
sequence_len = [math.ceil(i / float(step_size)) for i in sequence_len]
max_len = max(sequence_len)
if max_len % step_size != 0:
max_len = math.ceil(max_len)
if data_type == 'input':
result = np.zeros([len(data), max_len * step_size, 3],
dtype=np.float32)
for i, example in enumerate(data):
for j, row in enumerate(example):
for k, val in enumerate(row):
result[i][j][k] = val
elif data_type == 'targets':
result = np.zeros([len(data), max_len], dtype=np.int32)
for i, example in enumerate(data):
for step, val in enumerate(example[::step_size]):
result[i][step] = np.max(val)
return result, sequence_len
def setup_layer(self):
# LSTM_units_num = self.config.cblstm_l
lstm_layers_num = self.config.cblstm_lstm_layer_num
class_num = self.config.cblstm_class_num
# batch_size = self.config.cblstm_batch_size
layer = dict()
input_ = tf.placeholder(tf.float32,
shape=[None, None, 3],
name='input')
sequence_length = tf.placeholder(tf.int32,
shape=[None],
name='seq_len')
targets = tf.placeholder(tf.int32, shape=[None, None], name='targets')
keep_prob = tf.placeholder(tf.float32, name='keep_prob')
batch_size = tf.shape(input_)[0]
layer['input'] = input_
layer['seq_len'] = sequence_length
layer['targets'] = targets
layer['keep_prob'] = keep_prob
layer['conv1'] = layers.conv1d(layer['input'],
filter=[4, 3, 8],
strides=1,
padding='SAME',
wd=5e-5,
bias=0.0,
name='conv1')
layer['pooling1'] = layers.pool1d(layer['conv1'],
ksize=[2],
strides=[2],
padding='SAME',
name='pooling1')
layer['conv2'] = layers.conv1d(layer['pooling1'],
filter=[4, 8, 16],
strides=1,
padding='SAME',
wd=5e-5,
bias=0.0,
name='conv2')
layer['pooling2'] = layers.pool1d(layer['conv2'],
ksize=[2],
strides=[2],
padding='SAME',
name='pooling2')
layer['unfold'] = tf.reshape(layer['pooling2'], [batch_size, -1, 400])
layer['unfold'] = tf.reshape(layer['unfold'], [-1, 400])
layer['unfold'] = tf.nn.dropout(layer['unfold'], keep_prob)
layer['dim_red'] = layers.fc(layer['unfold'],
output_dim=100,
wd=5e-5,
name='dim_red')
layer['dim_red'] = tf.reshape(layer['dim_red'], [batch_size, -1, 100])
lstm_cell_fw1 = tf.nn.rnn_cell.LSTMCell(
num_units=100,
forget_bias=1.0,
state_is_tuple=True,
reuse=tf.get_variable_scope().reuse)
lstm_cell_fw2 = tf.nn.rnn_cell.LSTMCell(
num_units=100,
num_proj=50,
forget_bias=1.0,
state_is_tuple=True,
reuse=tf.get_variable_scope().reuse)
lstm_cell_bw1 = tf.nn.rnn_cell.LSTMCell(
num_units=100,
forget_bias=1.0,
state_is_tuple=True,
reuse=tf.get_variable_scope().reuse)
lstm_cell_bw2 = tf.nn.rnn_cell.LSTMCell(
num_units=100,
num_proj=50,
forget_bias=1.0,
state_is_tuple=True,
reuse=tf.get_variable_scope().reuse)
lstm_cell_fw1 = tf.nn.rnn_cell.DropoutWrapper(
lstm_cell_fw1, output_keep_prob=keep_prob)
lstm_cell_fw2 = tf.nn.rnn_cell.DropoutWrapper(
lstm_cell_fw2, output_keep_prob=keep_prob)
lstm_cell_bw1 = tf.nn.rnn_cell.DropoutWrapper(
lstm_cell_bw1, output_keep_prob=keep_prob)
lstm_cell_bw2 = tf.nn.rnn_cell.DropoutWrapper(
lstm_cell_bw2, output_keep_prob=keep_prob)
cell_fw = tf.nn.rnn_cell.MultiRNNCell([lstm_cell_fw1, lstm_cell_fw2],
state_is_tuple=True)
cell_bw = tf.nn.rnn_cell.MultiRNNCell([lstm_cell_bw1, lstm_cell_bw2],
state_is_tuple=True)
layer['dim_red'] = tf.nn.dropout(layer['dim_red'], keep_prob)
with tf.variable_scope('bi_rnn'):
(outputs, _) = tf.nn.bidirectional_dynamic_rnn(
cell_fw=cell_fw,
cell_bw=cell_bw,
inputs=layer['dim_red'],
sequence_length=sequence_length,
dtype=tf.float32)
output = tf.concat(outputs, 2)
layer['birnn'] = tf.reshape(output, [-1, 50 * 2])
layer['birnn'] = tf.nn.dropout(layer['birnn'], keep_prob)
layer['fc'] = layers.fc(layer['birnn'],
output_dim=50,
wd=5e-5,
name='fc')
with tf.variable_scope('softmax'):
softmax_w = tf.get_variable(
name='softmax_w',
shape=[50, class_num],
initializer=tf.truncated_normal_initializer(stddev=0.05),
dtype=tf.float32)
weight_decay = tf.multiply(tf.nn.l2_loss(softmax_w),
5e-5,
name='weight_loss')
tf.add_to_collection('losses', weight_decay)
softmax_b = tf.get_variable(
name='softmax_b',
shape=[class_num],
initializer=tf.constant_initializer(value=0),
dtype=tf.float32)
xw_plus_b = tf.nn.xw_plus_b(layer['fc'], softmax_w, softmax_b)
logits = tf.reshape(xw_plus_b, [batch_size, -1, class_num])
layer['logits'] = logits
class_prob = tf.nn.softmax(logits)
layer['class_prob'] = class_prob
layer['pred_seq'] = tf.cast(tf.argmax(class_prob, axis=2), tf.int32)
return layer
def setup_loss(self):
with tf.name_scope('loss'):
# fake_loss = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=self.training_layer['logits'],
# labels=self.training_layer['targets'])
# mask = tf.cast(tf.sign(self.training_layer['targets']), dtype=tf.float32)
mask = tf.sequence_mask(self.layer['seq_len'], dtype=tf.float32)
# loss_per_example_per_step = tf.multiply(fake_loss, mask)
# loss_per_example_sum = tf.reduce_sum(loss_per_example_per_step, reduction_indices=[1])
# loss_per_example_average = tf.div(x=loss_per_example_sum,
# y=tf.cast(self.training_layer['seq_len'], tf.float32))
raw_loss = tf.contrib.seq2seq.sequence_loss(
self.layer['logits'],
self.layer['targets'],
mask,
average_across_timesteps=True,
average_across_batch=True)
# loss = tf.reduce_mean(loss_per_example_average, name='loss')
# loss_per_example_per_step = tf.multiply(raw_loss, mask)
# mask = tf.reduce_sum(mask, axis=0)
# loss_per_example_per_step = tf.div(x=loss_per_example_per_step,
# y=mask)
raw_loss_summ = tf.summary.scalar('raw_loss', raw_loss)
tf.add_to_collection('losses', raw_loss)
loss = tf.add_n(tf.get_collection('losses'), name='total_loss')
loss_summ = tf.summary.scalar('total_loss', loss)
tf.add_to_collection('train_summary', raw_loss_summ)
tf.add_to_collection('train_summary', loss_summ)
return loss
def setup_train_op(self):
tvars = tf.trainable_variables()
grads, _ = tf.clip_by_global_norm(tf.gradients(self.loss, tvars),
self.config.max_grad_norm)
optimizer = tf.train.AdamOptimizer(5e-4)
return optimizer.apply_gradients(zip(grads, tvars))
# return optimizer.minimize(self.loss)
def setup_metrics(self, is_training):
if is_training:
name_scope = 'training'
summary_name = 'train_metrics_summary'
else:
name_scope = 'test'
summary_name = 'test_metrics_summary'
metrics = dict()
with tf.name_scope(name_scope):
with tf.variable_scope(name_scope):
metrics['acc'] = tf.placeholder(dtype=tf.float32, name='acc')
acc_summ = tf.summary.scalar('acc', metrics['acc'])
metrics['p_error'] = tf.placeholder(dtype=tf.float32,
name='p_error')
p_err_summ = tf.summary.scalar('p_error', metrics['p_error'])
metrics['p_error_max'] = tf.placeholder(dtype=tf.float32,
name='p_error_max')
p_err_max_summ = tf.summary.scalar('p_error_max',
metrics['p_error_max'])
metrics['s_error'] = tf.placeholder(dtype=tf.float32,
name='s_error')
s_err_summ = tf.summary.scalar('s_error', metrics['s_error'])
metrics['s_error_max'] = tf.placeholder(dtype=tf.float32,
name='s_error_max')
s_err_max_summ = tf.summary.scalar('s_error_max',
metrics['s_error_max'])
tf.add_to_collection(summary_name, acc_summ)
tf.add_to_collection(summary_name, p_err_summ)
tf.add_to_collection(summary_name, p_err_max_summ)
tf.add_to_collection(summary_name, s_err_summ)
tf.add_to_collection(summary_name, s_err_max_summ)
return metrics
def train(self, passes, new_training=True):
sess_config = tf.ConfigProto()
sess_config.gpu_options.allow_growth = True
with tf.Session(config=sess_config) as sess:
if new_training:
saver, global_step = Model.start_new_session(sess)
else:
saver, global_step = Model.continue_previous_session(
sess,
model_file='cblstm',
ckpt_file='saver/cblstm/checkpoint')
self.train_writer.add_graph(sess.graph, global_step=global_step)
for step in range(1 + global_step, 1 + passes + global_step):
with tf.variable_scope('Train'):
input_, targets = self.reader.get_birnn_batch_data('train')
input_, seq_len = self.data_padding_preprocess(
input_, 'input')
targets, _ = self.data_padding_preprocess(
targets, 'targets')
_, train_summary, loss, pred_seq = sess.run(
[
self.train_op, self.train_merged, self.loss,
self.layer['pred_seq']
],
feed_dict={
self.layer['input']: input_,
self.layer['targets']: targets,
self.layer['seq_len']: seq_len,
self.layer['keep_prob']: self.config.keep_prob
})
self.train_writer.add_summary(train_summary, step)
train_p_err, train_p_err_max, train_s_err, train_s_err_max = get_p_s_error(
pred_seq, targets, seq_len)
train_acc = get_acc(pred_seq, targets, seq_len)
[train_metrics_summary] = sess.run(
[self.train_metrics_merged],
feed_dict={
self.train_metrics['acc']: train_acc,
self.train_metrics['p_error']: train_p_err,
self.train_metrics['p_error_max']: train_p_err_max,
self.train_metrics['s_error']: train_s_err,
self.train_metrics['s_error_max']: train_s_err_max
})
self.train_writer.add_summary(train_metrics_summary, step)
print("gobal_step {},"
" training_loss {},"
" accuracy {},"
" p_error {},"
" p_err_max {},"
" s_error {},"
" s_err_max {}.".format(step, loss, train_acc,
train_p_err, train_p_err_max,
train_s_err,
train_s_err_max))
if step % 5 == 0:
with tf.variable_scope('Test', reuse=True):
test_input, test_targets = self.reader.get_birnn_batch_data(
'test')
test_input, test_seq_len = self.data_padding_preprocess(
test_input, 'input')
test_targets, _ = self.data_padding_preprocess(
test_targets, 'targets')
[test_pred_seq] = sess.run(
[self.layer['pred_seq']],
feed_dict={
self.layer['input']: test_input,
self.layer['seq_len']: test_seq_len,
self.layer['keep_prob']: 1.0
})
test_p_err, test_p_err_max, test_s_err, test_s_err_max = get_p_s_error(
test_pred_seq, test_targets, test_seq_len)
test_acc = get_acc(test_pred_seq, test_targets,
test_seq_len)
[test_metrics_summary] = sess.run(
[self.test_metrics_merged],
feed_dict={
self.test_metrics['acc']: test_acc,
self.test_metrics['p_error']: test_p_err,
self.test_metrics['p_error_max']:
test_p_err_max,
self.test_metrics['s_error']: test_s_err,
self.test_metrics['s_error_max']:
test_s_err_max
})
self.train_writer.add_summary(test_metrics_summary,
step)
print("test_acc {}, "
"test_p_err {},"
"test_p_err_max {},"
"test_s_err {},"
"test_s_err_max {}.".format(
test_acc, test_p_err, test_p_err_max,
test_s_err, test_s_err_max))
if step % 100 == 0:
saver.save(sess, 'saver/cblstm/cblstm', global_step=step)
print('checkpoint saved')
def pickup_p_s(self, sess, input_, get_pred_seq=False):
# with tf.Session() as sess:
# saver, global_step = Model.continue_previous_session(sess,
# model_file='bi_rnn',
# ckpt_file='saver/bi_rnn/checkpoint')
with tf.variable_scope('model', reuse=True):
input_, seq_len = self.data_padding_preprocess(input_, 'input')
pred_seq, class_prob = sess.run(
[self.layer['pred_seq'], self.layer['class_prob']],
feed_dict={
self.layer['input']: input_,
self.layer['seq_len']: seq_len,
self.layer['keep_prob']: 1.0
})
p_index = [
get_arrival_index(pred_seq[i], seq_len[i], 'P')
for i in range(len(input_))
]
s_index = [
get_arrival_index(pred_seq[i], seq_len[i], 'S')
for i in range(len(input_))
]
if get_pred_seq:
return p_index, s_index, class_prob, pred_seq
else:
return p_index, s_index, class_prob
for ent in labels:
start_dic[(ent[0], ent[2])].append(ent)
end_dic[(ent[1], ent[2])].append(ent)
all_num += 1
for k, v in start_dic.items():
if len(v) > 1:
start_num += len(v)
for k, v in end_dic.items():
if len(v) > 1:
end_num += len(v)
print("All {}, start {}, end {}".format(all_num, start_num, end_num))
if __name__ == "__main__":
reader = Reader()
reader.read_and_gen_vectors_pubmed_word2vec(config.embed_path)
reader.read_all_data("./data/genia/", "genia.train", "genia.dev",
"genia.test")
# print reader.train_sents[0]
train_batches, dev_batches, test_batches = reader.to_batch(
config.batch_size)
f = open(config.train_data_path, 'wb')
pickle.dump(train_batches, f)
f.close()
f = open(config.dev_data_path, 'wb')
pickle.dump(dev_batches, f)
f.close()
class MPRSNE(object):
def __init__(self, g, saveFile, init_emb_file=None):
self.config = Config()
self.g = g
self.init_emb_file = init_emb_file
self.saveFile = saveFile
self.reader = Reader(self.g)
self.layer = self.setup_layer()
self.train_op = self.setup_train_op()
self.test_metrics = self.get_test_metrics()
self.loss_train_merged = tf.summary.merge(tf.get_collection('loss_train_summary'))
self.test_merged = tf.summary.merge(tf.get_collection('multi_label_classification'))
self.train_writer = tf.summary.FileWriter('model/summary/{}'.format(self.saveFile))
def setup_layer(self):
layer = dict()
batch_num = self.config.nodes_seq_batch_num
emb_dim = self.config.emb_dim
neg_sample_num = self.config.loss1_neg_sample_num
walk_len = self.config.walk_length
gru_layer_num = self.config.gru_layer_num
keep_prob = self.config.keep_prob
labels_num = self.g.labels_num
context_size = self.config.context_size
label_l2_loss_wd = self.config.label_l2_loss_wd
emb_l2_loss_wd = self.config.emb_l2_loss_wd
pos_weight = 1.0 / self.config.pos_weight
walk_nodes = tf.placeholder(tf.int32, shape=[batch_num, walk_len], name='walk_nodes')
walk_nodes_labels = tf.placeholder(tf.float32, shape=[batch_num, walk_len, labels_num],
name='walk_nodes_labels')
neg_walk_nodes = tf.placeholder(tf.int32, shape=[batch_num, walk_len, neg_sample_num], name='neg_walk_node')
layer['walk_nodes'] = walk_nodes
layer['walk_nodes_labels'] = walk_nodes_labels
layer['neg_walk_nodes'] = neg_walk_nodes
if self.init_emb_file is not None:
emb = list()
with open(self.init_emb_file) as file:
csv_reader = csv.reader(file)
for row in csv_reader:
emb.append(row)
emb = np.array(emb, dtype=np.float32)
emb_initializer = tf.constant_initializer(emb)
else:
emb_initializer = tf.glorot_uniform_initializer(dtype=tf.float32)
with tf.variable_scope('emb'):
emb = tf.get_variable(name='emb',
shape=[self.g.nodes_num, emb_dim],
initializer=emb_initializer)
layer['emb'] = emb
zeros_vec = tf.constant(0, dtype=tf.float32, shape=[1, emb_dim])
emb = tf.concat([emb, zeros_vec], 0)
with tf.variable_scope('sup_emb'):
sup_emb = tf.get_variable(name='sup_emb',
shape=[self.g.nodes_num, emb_dim],
initializer=tf.glorot_normal_initializer(dtype=tf.float32))
layer['sup_emb'] = sup_emb
with tf.variable_scope('labels_emb'):
labels_emb = tf.get_variable(name='labels_emb',
shape=[self.g.labels_num, emb_dim],
initializer=tf.glorot_normal_initializer(dtype=tf.float32))
labels_l2_loss = label_l2_loss_wd * tf.nn.l2_loss(labels_emb)
tf.add_to_collection('label_loss_weight_decay', labels_l2_loss)
layer['labels_emb'] = labels_emb
with tf.variable_scope('lookup'):
walk_nodes_emb = tf.nn.embedding_lookup(emb, walk_nodes, name='context_nodes_emb')
walk_nodes_emb = layer_norm(walk_nodes_emb)
true_sup_emb = tf.nn.embedding_lookup(sup_emb, walk_nodes, name='true_sup_emb')
neg_sup_emb = tf.nn.embedding_lookup(sup_emb, neg_walk_nodes, name='neg_sup_emb')
fw_context_gru_cell_list = [tf.nn.rnn_cell.DropoutWrapper(tf.nn.rnn_cell.DeviceWrapper(
MRUCell(num_units=emb_dim,
forget_bias=0.0,
state_is_tuple=True,
activation=tf.nn.tanh,
reuse=tf.get_variable_scope().reuse,
kernel_initializer=tf.glorot_normal_initializer(dtype=tf.float32)),
"/gpu:%d" % (i % 2)),
input_keep_prob=keep_prob, variational_recurrent=True, input_size=emb_dim, dtype=tf.float32)
for i in range(gru_layer_num)]
fw_context_gru_cell = tf.nn.rnn_cell.MultiRNNCell(
fw_context_gru_cell_list, state_is_tuple=True)
fw_context_Residual_gru_cell = fw_context_gru_cell
bw_context_gru_cell_list = [tf.nn.rnn_cell.DropoutWrapper(tf.nn.rnn_cell.DeviceWrapper(
MRUCell(num_units=emb_dim,
forget_bias=0.0,
state_is_tuple=True,
activation=tf.nn.tanh,
reuse=tf.get_variable_scope().reuse,
kernel_initializer=tf.glorot_normal_initializer(dtype=tf.float32)),
"/gpu:%d" % (i % 2)),
input_keep_prob=keep_prob, variational_recurrent=True, input_size=emb_dim, dtype=tf.float32)
for i in range(gru_layer_num)]
bw_context_gru_cell = tf.nn.rnn_cell.MultiRNNCell(
bw_context_gru_cell_list, state_is_tuple=True)
bw_context_Residual_gru_cell = bw_context_gru_cell
with tf.variable_scope('context'):
(context_outputs, _) = tf.nn.bidirectional_dynamic_rnn(cell_fw=fw_context_Residual_gru_cell,
cell_bw=bw_context_Residual_gru_cell,
inputs=walk_nodes_emb,
sequence_length=[walk_len] * batch_num,
parallel_iterations=batch_num * 2,
dtype=tf.float32)
context_outputs = tf.concat(context_outputs, -1)
with tf.variable_scope('reduce_layer'):
emb_weight = tf.get_variable(name='emb_weight',
shape=[emb_dim * 2, emb_dim],
initializer=tf.glorot_normal_initializer(dtype=tf.float32))
emb_bias = tf.get_variable(name='emb_bias',
shape=[emb_dim],
initializer=tf.constant_initializer(0.0))
context_outputs = tf.reshape(context_outputs, [-1, emb_dim * 2])
context_outputs = tf.matmul(context_outputs, emb_weight) + emb_bias
context_outputs = tf.reshape(context_outputs, [batch_num, walk_len, emb_dim])
emb_l2_loss = emb_l2_loss_wd * tf.nn.l2_loss(emb_weight)
tf.add_to_collection('emb_loss_weight_decay', emb_l2_loss)
context_outputs = tf.nn.sigmoid(context_outputs)
with tf.variable_scope('output_gates'):
o_weight = tf.get_variable(name='o_weight',
shape=[emb_dim, emb_dim],
initializer=tf.glorot_normal_initializer(dtype=tf.float32))
o_diag = tf.get_variable(name='o_diag',
shape=[emb_dim])
o_bias = tf.get_variable(name='o_bias',
shape=[emb_dim],
initializer=tf.constant_initializer(0.0))
o_emb = tf.reshape(walk_nodes_emb, [-1, emb_dim])
o_outputs = tf.reshape(context_outputs, [-1, emb_dim])
o_gates = tf.matmul(o_emb, o_weight) + o_diag * o_outputs + o_bias
o_gates = tf.layers.batch_normalization(o_gates, axis=-1)
o_gates = tf.sigmoid(o_gates)
o_gates = tf.reshape(o_gates, [batch_num, walk_len, emb_dim])
label_context = o_gates * context_outputs
with tf.variable_scope('walk_context'):
walk_context = list()
for i in range(walk_len):
tmp_context = tf.concat([context_outputs[:, :i, :], context_outputs[:, i + 1:, :]], axis=1)
tmp_nodes_emb = tf.concat([walk_nodes_emb[:, :i, :], walk_nodes_emb[:, i + 1:, :]], axis=1)
tmp_context = tmp_context + tmp_nodes_emb
# [batch, walk_len - 1, emb]
walk_context.append(tmp_context)
context_vec = tf.stack(walk_context, axis=1)
# [batch, walk_len, walk_len - 1, emb]
context_mask = np.zeros([walk_len, walk_len - 1], dtype=np.float32)
for i in range(walk_len):
mask_min = np.max([0, i - context_size])
mask_max = np.min([walk_len - 1, i + context_size])
context_mask[i, mask_min: mask_max] = 1.0
context_mask = tf.constant(context_mask, dtype=tf.float32)
context_mask = tf.stack([context_mask] * batch_num, axis=0)
context_mask = tf.stack([context_mask] * emb_dim, axis=3)
# [batch, walk_len, walk_len - 1, emb]
context_vec = context_mask * context_vec
# [batch, walk_len, walk_len - 1, emb]
with tf.variable_scope('loss'):
label_context = tf.reshape(label_context, [-1, emb_dim])
label_score = tf.matmul(label_context, labels_emb, transpose_b=True)
label_bias = tf.get_variable(name='label_bias',
shape=[labels_num],
dtype=tf.float32,
initializer=tf.constant_initializer(0.0))
label_score = label_score + label_bias
label_score = tf.reshape(label_score, [batch_num, walk_len, labels_num])
label_loss = tf.nn.weighted_cross_entropy_with_logits(targets=walk_nodes_labels, logits=label_score,
pos_weight=pos_weight)
label_loss = label_loss / pos_weight
# [batch, walk_len, label]
label_loss = tf.reduce_sum(label_loss, 2)
# [batch, walk_len]
label_loss = tf.reduce_sum(label_loss, 1)
# [batch]
label_loss = tf.reduce_mean(label_loss, 0)
label_loss_wd = tf.add_n(tf.get_collection('label_loss_weight_decay'))
label_loss = label_loss + label_loss_wd
loss_summ = tf.summary.scalar('label_loss', label_loss)
tf.add_to_collection('loss_train_summary', loss_summ)
layer['label_loss'] = label_loss * 1.0
context_vec = tf.stack([context_vec] * (neg_sample_num + 1), axis=2)
# [batch, walk_len, node, walk_len - 1, emb]
true_emb = tf.expand_dims(true_sup_emb, axis=2)
# [batch, walk_len, 1, emb]
true_neg_emb = tf.concat([true_emb, tf.negative(neg_sup_emb)], 2)
# [batch, walk_len, node, emb]
true_neg_emb = tf.stack([true_neg_emb] * (walk_len - 1), axis=3)
# [batch, walk_len, node, walk_len - 1, emb]
sig = tf.sigmoid(tf.reduce_sum(tf.multiply(context_vec, true_neg_emb), 4))
# [batch, walk_len, node, walk_len - 1]
sig_log = tf.log(sig)
sig_log = tf.reduce_sum(sig_log, 2)
# [batch, walk_len, walk_len - 1]
sig_log = tf.reduce_sum(sig_log, 2)
# [batch, walk_len]
sig_log_batch = tf.reduce_sum(sig_log, 1)
# [batch]
emb_loss = tf.reduce_mean(sig_log_batch, 0, name='emb_loss')
emb_loss = tf.negative(emb_loss)
emb_loss_wd = tf.add_n(tf.get_collection('emb_loss_weight_decay'))
emb_loss = emb_loss + emb_loss_wd
layer['emb_loss'] = emb_loss
loss_summ = tf.summary.scalar('emb_loss', emb_loss)
tf.add_to_collection('loss_train_summary', loss_summ)
layer['loss'] = layer['label_loss'] + layer['emb_loss']
loss_summ = tf.summary.scalar('loss', layer['loss'])
tf.add_to_collection('loss_train_summary', loss_summ)
return layer
def setup_train_op(self):
with tf.variable_scope('train_op'):
loss_step = tf.Variable(0, trainable=False)
starter_learning_rate = 0.0005
decay_steps = 10000
learning_rate = tf.train.exponential_decay(starter_learning_rate, loss_step, decay_steps, 0.5,
staircase=True)
learning_rate = tf.maximum(0.0001, learning_rate)
loss_summ = tf.summary.scalar('learning_rate', learning_rate)
tf.add_to_collection('loss_train_summary', loss_summ)
optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate)
grads_pre, tvars = zip(*optimizer.compute_gradients(self.layer['loss']))
grads, _ = tf.clip_by_global_norm(grads_pre, self.config.max_grad_norm)
train_op = optimizer.apply_gradients(zip(grads, tvars), global_step=loss_step)
pre_optimizer = tf.train.AdamOptimizer(learning_rate=0.001)
pre_tvars = [var for var in tf.trainable_variables() if var.name != 'emb/emb:0']
pre_grads_pre, _ = zip(*pre_optimizer.compute_gradients(self.layer['loss'], var_list=pre_tvars))
pre_grads_pre, _ = tf.clip_by_global_norm(pre_grads_pre, self.config.max_grad_norm)
pre_train_op = pre_optimizer.apply_gradients(zip(pre_grads_pre, pre_tvars))
return [train_op, pre_train_op]
def train(self, passes, new_training=True):
sess_config = tf.ConfigProto()
sess_config.gpu_options.allow_growth = True
sess_config.allow_soft_placement = True
with tf.Session(config=sess_config) as sess:
if new_training:
saver, global_step = Model.start_new_session(sess)
else:
saver, global_step = Model.continue_previous_session(sess,
model_file='model/saver/{}'.format(self.saveFile),
ckpt_file='model/saver/{}/checkpoint'.format(
self.saveFile))
self.train_writer.add_graph(sess.graph, global_step=global_step)
walk_times = 1
for step in range(1 + global_step, 1 + passes + global_step):
with tf.variable_scope('Train'):
walk_nodes = self.reader.nodes_walk_reader()
neg_walk_nodes = [self.g.negative_sample(walk_nodes[i],
self.config.loss1_neg_sample_num,
self.g.nodes_degree_table)
for i in range(len(walk_nodes))]
neg_walk_nodes = np.array(neg_walk_nodes)
walk_nodes_labels = list()
for node_list in walk_nodes:
nodes_label_tmp = self.g.get_train_node_label(node_list)
walk_nodes_labels.append(nodes_label_tmp)
walk_nodes_labels = np.array(walk_nodes_labels)
# if (step - 1) % int(self.g.nodes_num / self.config.nodes_seq_batch_num) == 0:
# print(walk_times)
# walk_times += 1
if step < 200 and self.init_emb_file is not None:
train_op = self.train_op[1]
else:
train_op = self.train_op[0]
_, train_summary, loss = sess.run(
[train_op,
self.loss_train_merged,
self.layer['loss']],
feed_dict={self.layer['walk_nodes']: walk_nodes,
self.layer['walk_nodes_labels']: walk_nodes_labels,
self.layer['neg_walk_nodes']: neg_walk_nodes})
self.train_writer.add_summary(train_summary, step)
if step % 500 == 0 or step == 1:
[node_emb, sup_emb] = sess.run([self.layer['emb'],
self.layer['sup_emb']])
node_emb = np.concatenate((node_emb, sup_emb), axis=1)
print("gobal_step {},loss {}".format(step, loss))
if step % 1000 == 0 or step == 1:
micro_f1, macro_f1 = self.multi_label_node_classification(node_emb)
[test_summary] = sess.run([self.test_merged],
feed_dict={self.test_metrics['micro_f1']: micro_f1,
self.test_metrics['macro_f1']: macro_f1})
print("micro_f1 {},macro_f1 {}".format(micro_f1, macro_f1))
self.train_writer.add_summary(test_summary, step)
saver.save(sess, 'model/saver/{}/MPRSNE'.format(self.saveFile), global_step=step)
print('checkpoint saved')
def get_emb(self):
sess_config = tf.ConfigProto()
sess_config.gpu_options.allow_growth = True
with tf.Session(config=sess_config) as sess:
saver, global_step = Model.continue_previous_session(sess,
model_file='model/saver/{}'.format(self.saveFile),
ckpt_file='model/saver/{}/checkpoint'.format(self.saveFile))
ids_set = np.array(range(self.g.nodes_num))
emb_set = tf.nn.embedding_lookup(self.layer['emb'], ids_set)
sup_emb_set = tf.nn.embedding_lookup(self.layer['sup_emb'], ids_set)
[emb, sup_emb] = sess.run([emb_set, sup_emb_set])
emb = np.concatenate([emb, sup_emb], axis=1)
return emb
def multi_label_node_classification(self, emb):
g = self.g
emb = emb[:, :self.config.emb_dim]
classes = self.g.validate_labels_set
train_nodes_id = list()
train_y = list()
train_x = list()
for node in g.train_nodes_labels.keys():
train_nodes_id.append(node)
train_y.append(g.train_nodes_labels[node])
train_x.append(emb[g.nodes_ids[node], :])
train_x = np.array(train_x)
prepocess_y = sklearn.preprocessing.MultiLabelBinarizer(classes=classes)
train_y = prepocess_y.fit_transform(train_y)
test_nodes_id = list()
test_y = list()
test_x = list()
for node in g.test_nodes_labels.keys():
test_nodes_id.append(node)
test_y.append(g.test_nodes_labels[node])
test_x.append(emb[g.nodes_ids[node], :])
test_x = np.array(test_x)
test_y = prepocess_y.fit_transform(test_y)
multi_label_classifier = sklearn.multiclass.OneVsRestClassifier(sklearn.linear_model.LogisticRegression(),
n_jobs=1)
multi_label_classifier.fit(train_x, train_y)
pred_y = multi_label_classifier.predict(test_x)
mirco_f1 = sklearn.metrics.f1_score(test_y, pred_y, average='micro')
marco_f1 = sklearn.metrics.f1_score(test_y, pred_y, average='macro')
return mirco_f1, marco_f1
def get_test_metrics(self):
micro_f1 = tf.placeholder(dtype=tf.float32, shape=None, name='micro_f1')
macro_f1 = tf.placeholder(dtype=tf.float32, shape=None, name='macro_f1')
loss_summ = tf.summary.scalar('micro_f1', micro_f1)
tf.add_to_collection('multi_label_classification', loss_summ)
loss_summ = tf.summary.scalar('macro_f1', macro_f1)
tf.add_to_collection('multi_label_classification', loss_summ)
test_metrics = dict()
test_metrics['micro_f1'] = micro_f1
test_metrics['macro_f1'] = macro_f1
return test_metrics