def train(self, passes, new_training=True):
with tf.Session() as sess:
global_step=tf.Variable(0, trainable=False)
#learning_rate = tf.train.exponential_decay(0.001, global_step, 200, 0.8, staircase=True)
training = tf.train.AdamOptimizer(self.config.learning_rate).minimize(self.loss)
if new_training:
saver, global_step = Model.start_new_session(sess)
else:
saver, global_step = Model.continue_previous_session(sess,
model_file='cnn',
ckpt_file= self.config.root + '/event_detect/saver/cnn/checkpoint')
sess.run(tf.local_variables_initializer())
self.train_writer.add_graph(sess.graph, global_step=global_step)
test_restlt=[]
for step in range(1 + global_step, 1 + passes + global_step):
input, target = self.reader.get_cnn_batch_data('train')
#print(input.shape)
summary, _, acc = sess.run([self.merged, training, self.metrics['accuracy']],
feed_dict={self.layer['input']: input,
self.layer['target']: target})
self.train_writer.add_summary(summary, step)
if step % 10 == 0:
loss = sess.run(self.loss,
feed_dict={self.layer['input']: input,
self.layer['target']: target})
test_restlt.append(loss)
print("gobal_step {}, training_loss {}, accuracy {}".format(step, loss, acc))
if step % 100 == 0:
test_x, text_y = self.reader.get_cnn_batch_data('test')
acc, recall, precision = sess.run([self.metrics['accuracy'],
self.metrics['recall'],
self.metrics['precision']],
feed_dict={self.layer['input']: test_x,
self.layer['target']: text_y})
print("test: accuracy {}, recall {}, precision {}".format(acc, recall, precision))
saver.save(sess, self.config.root + '/event_detect/saver/cnn/cnn', global_step=step)
print('checkpoint saved')
#print(sess.run([self.layer['class_prob']], feed_dict={self.layer['input']: input}))
print(test_restlt)
def train(self, passes, new_training=True):
with tf.Session() as sess:
training = tf.train.AdamOptimizer(1e-3).minimize(self.loss)
if new_training:
saver, global_step = Model.start_new_session(sess)
else:
saver, global_step = Model.continue_previous_session(
sess, model_file='cnn', ckpt_file='saver/cnn/checkpoint')
sess.run(tf.local_variables_initializer())
self.train_writer.add_graph(sess.graph, global_step=global_step)
for step in range(1 + global_step, 1 + passes + global_step):
input, target = self.reader.get_cnn_batch_data('train')
summary, _, acc = sess.run(
[self.merged, training, self.metrics['accuracy']],
feed_dict={
self.layer['input']: input,
self.layer['target']: target
})
self.train_writer.add_summary(summary, step)
if step % 10 == 0:
loss = sess.run(self.loss,
feed_dict={
self.layer['input']: input,
self.layer['target']: target
})
print(
"gobal_step {}, training_loss {}, accuracy {}".format(
step, loss, acc))
if step % 100 == 0:
test_x, text_y = self.reader.get_cnn_batch_data('test')
acc, recall, precision = sess.run([
self.metrics['accuracy'], self.metrics['recall'],
self.metrics['precision']
],
feed_dict={
self.layer['input']:
test_x,
self.layer['target']:
text_y
})
print("test: accuracy {}, recall {}, precision {}".format(
acc, recall, precision))
saver.save(sess, 'saver/cnn/cnn', global_step=step)
print('checkpoint saved')
def get_emb(self):
sess_config = tf.ConfigProto()
sess_config.gpu_options.allow_growth = True
with tf.Session(config=sess_config) as sess:
saver, global_step = Model.continue_previous_session(sess,
model_file='model/saver/{}'.format(self.saveFile),
ckpt_file='model/saver/{}/checkpoint'.format(self.saveFile))
ids_set = np.array(range(self.g.nodes_num))
emb_set = tf.nn.embedding_lookup(self.layer['emb'], ids_set)
sup_emb_set = tf.nn.embedding_lookup(self.layer['sup_emb'], ids_set)
[emb, sup_emb] = sess.run([emb_set, sup_emb_set])
emb = np.concatenate([emb, sup_emb], axis=1)
return emb
def main(model_name, new_scan=False, preprocess=True):
config = Config()
plot = config.plot
cut = config.cut
bandpass = config.bandpass
resample = config.resample
# read data folders
file_list = os.listdir(config.root + '/data/after')
file_list.sort()
if new_scan == True:
print('start new scan!')
start_point = 0
event_num = 0
try:
os.system('rm -rf %s/event_detect/detect_result/cut/*' %
config.root)
os.system('rm -rf %s/event_detect/detect_result/png/*' %
config.root)
os.system('rm -rf %s/event_detect/detect_result/png2/*' %
config.root)
os.system('rm -rf %s/event_detect/detect_result/cnn/*.csv' %
config.root)
except:
pass
# file_list_len = len(file_list)
else:
with open(config.root + '/event_detect/detect_result/' + model_name +
'/checkpoint') as file:
start_point = int(file.readline())
event_num = int(file.readline())
file_list = file_list[start_point:]
# file_list_len = len(file_list)
print('restart from {}'.format(file_list[0]))
# load CNN model
if model_name == 'cnn':
from cnn import CNN
import tensorflow as tf
from tflib.models import Model
model = CNN()
# sess = tf.Session(config=tf.ConfigProto(device_count={"CPU":20},inter_op_parallelism_threads=0,intra_op_parallelism_threads=0))
sess = tf.Session()
saver, global_step = Model.continue_previous_session(
sess,
model_file='cnn',
ckpt_file=config.root + '/event_detect/saver/cnn/checkpoint')
# read group info
group = []
with open(config.root + '/config/group_info', 'r') as f:
for line in f.readlines():
if line != '\n':
if line[0] == '#':
group.append([])
else:
group[-1].append(line.split()[0])
# read data & detect eq
for file in file_list:
sac_file_name = [[], [], []]
all_group_sta_num = [0] * len(group)
path = os.path.join(config.root + '/data/after', file)
begin = datetime.datetime.now()
group_E = [[] for _ in range(len(group))]
group_N = [[] for _ in range(len(group))]
group_Z = [[] for _ in range(len(group))]
print('Start reading data: %s.' % file)
for i in range(len(group)):
for sta in group[i]:
if len(glob.glob(path + '/' + '*' + sta + '.*')) == 3:
all_group_sta_num[i] += 1
sacfile_E = glob.glob(path + '/' + '*' + sta + '.*' +
'E')[0]
sacfile_N = glob.glob(path + '/' + '*' + sta + '.*' +
'N')[0]
sacfile_Z = glob.glob(path + '/' + '*' + sta + '.*' +
'Z')[0]
sac_file_name[0].append(sacfile_E.split('/')[-1])
sac_file_name[1].append(sacfile_N.split('/')[-1])
sac_file_name[2].append(sacfile_Z.split('/')[-1])
group_E[i].append(obspy.read(sacfile_E))
group_N[i].append(obspy.read(sacfile_N))
group_Z[i].append(obspy.read(sacfile_Z))
flatten_group_E = [st for each_group in group_E for st in each_group]
flatten_group_N = [st for each_group in group_N for st in each_group]
flatten_group_Z = [st for each_group in group_Z for st in each_group]
st_E = reduce(lambda st1, st2: st1 + st2, flatten_group_E)
st_N = reduce(lambda st1, st2: st1 + st2, flatten_group_N)
st_Z = reduce(lambda st1, st2: st1 + st2, flatten_group_Z)
st_all = st_E + st_N + st_Z
all_sta_num = len(flatten_group_Z)
if resample:
st_all = st_all.resample(sampling_rate=resample)
if bandpass:
st_all = st_all.filter('bandpass',
freqmin=bandpass[0],
freqmax=bandpass[1],
corners=4,
zerophase=True)
endtime = st_all[0].stats.endtime
start_flag = -1
end_flag = -1
event_list = []
confidence_total = {}
start_total = []
end_total = []
pos_num_total = []
samples = 1.0 / st_all[0].stats.delta
# npts = st_all[0].stats.npts
print('Finish reading data.')
print('Start detection.')
for windowed_st in st_all.slide(window_length=(config.winsize - 1) /
samples,
step=config.winlag / samples):
cur_sta = 0
len_group_conf = 0
group_class, group_conf = [], []
# windowed_E = windowed_st[:all_sta_num]
# windowed_N = windowed_st[all_sta_num:2*all_sta_num]
# windowed_Z = windowed_st[2*all_sta_num:]
start = len(windowed_st) / 3 * 2
end = len(windowed_st)
group_max_conf = 0
for i in range(len(group)):
data_input = [[], [], []]
group_sta_num = all_group_sta_num[i]
if group_sta_num > 0:
for j in range(cur_sta, cur_sta + group_sta_num):
if len(windowed_st[j].data) < config.winsize:
windowed_st[j].data = np.concatenate([
windowed_st[j].data,
np.zeros(config.winsize -
len(windowed_st[j].data))
])
data_input[0].append(
windowed_st[j].data[:config.winsize])
# print(j, windowed_st[j])
for j in range(all_sta_num + cur_sta,
all_sta_num + cur_sta + group_sta_num):
if len(windowed_st[j].data) < config.winsize:
windowed_st[j].data = np.concatenate([
windowed_st[j].data,
np.zeros(config.winsize -
len(windowed_st[j].data))
])
data_input[1].append(
windowed_st[j].data[:config.winsize])
# print(j, windowed_st[j])
for j in range(2 * all_sta_num + cur_sta,
2 * all_sta_num + cur_sta + group_sta_num):
if len(windowed_st[j].data) < config.winsize:
windowed_st[j].data = np.concatenate([
windowed_st[j].data,
np.zeros(config.winsize -
len(windowed_st[j].data))
])
data_input[2].append(
windowed_st[j].data[:config.winsize])
# print(j, windowed_st[j])
plot_b = 2 * all_sta_num + cur_sta
plot_e = 2 * all_sta_num + cur_sta + group_sta_num
cur_sta += group_sta_num
if preprocess:
for i in range(3):
for j in range(group_sta_num):
data_input[i][j] = data_preprocess(
data_input[i][j])
data_input = np.array(data_input)
if len(data_input[0][0]) < config.winsize:
concat = np.zeros([
3, group_sta_num,
config.winsize - len(data_input[0][0])
])
data_input = np.concatenate([data_input, concat],
axis=2)
else:
data_input = data_input[:, :, :config.winsize]
data_input = data_input.transpose((1, 2, 0))
j = 0
while j < len(data_input):
if np.max(data_input[j]) == 0 or np.isnan(
np.max(data_input[j])):
data_input = np.delete(data_input, j, axis=0)
else:
j += 1
if len(data_input) >= 3:
len_group_conf += 1
class_pred, confidence = model.classify(
sess=sess, input_=[data_input])
group_class.append(class_pred)
group_conf.append(confidence[0])
if confidence[0] > group_max_conf:
start = plot_b
end = plot_e
group_max_conf = confidence[0]
else:
group_class.append(0)
group_conf.append(0)
else:
group_class.append(0)
group_conf.append(0)
# consider the result of multiple groups
pos_num = 0
for each in group_class:
if each == 1:
pos_num += 1
if pos_num >= config.group_num_thrd:
class_pred = 1
else:
class_pred = 0
confidence = sum(
group_conf) / len_group_conf if len_group_conf else 0
# calculate the window range
if class_pred == 1:
confidence_total[confidence] = [group_max_conf, start, end]
start_total.append(windowed_st[0].stats.starttime)
end_total.append(windowed_st[0].stats.endtime)
pos_num_total.append(pos_num)
if start_flag == -1:
start_flag = windowed_st[0].stats.starttime
end_flag = windowed_st[0].stats.endtime
else:
end_flag = windowed_st[0].stats.endtime
print("{} {} {} {} {:.8f} {:.8f}".format(class_pred,start_flag,end_flag, \
windowed_st[0].stats.starttime,confidence, group_max_conf))
if class_pred == 0 and start_flag != -1: #end_flag < windowed_st[0].stats.starttime:
confidence = max(list(confidence_total.keys()))
# for j in range(len(confidence_total)):
# if confidence == confidence_total[j]:
# break
# start_local = start_total[j]
# end_local = end_total[j]
# event = [file, start_flag, end_flag,
# confidence, start_local, end_local]
event_num += 1
group_max_conf = confidence_total[confidence][0]
start = confidence_total[confidence][1]
end = confidence_total[confidence][2]
event = [event_num, file, start_flag, end_flag, confidence, \
max(pos_num_total), start, end, group_max_conf]
confidence_total = {}
start_total = []
end_total = []
pos_num_total = []
event_list.append(event)
#print(event_list)
start_flag = -1
end_flag = -1
if class_pred == 1 and end_flag + config.winlag / samples >= endtime:
confidence = max(list(confidence_total.keys()))
# for j in range(len(confidence_total)):
# if confidence == confidence_total[j]:
# break
# start_local = start_total[j]
# end_local = end_total[j]
# event = [file.split('/')[-2], start_flag, endtime,
# confidence, start_total, end_total]
event_num += 1
group_max_conf = confidence_total[confidence][0]
start = confidence_total[confidence][1]
end = confidence_total[confidence][2]
event = [event_num, file, start_flag, endtime, confidence, \
max(pos_num_total), start, end, group_max_conf]
event_list.append(event)
start_flag = -1
end_flag = -1
if event_list:
new_event_list = [event_list[0]]
for i in range(1, len(event_list)):
if event_list[i][1] > new_event_list[-1][1] and \
event_list[i][1] < new_event_list[-1][1]+1000/(config.resample if config.resample else 200):
# if event_list[i][1] > new_event_list[-1][1] and event_list[i][1] < new_event_list[-1][2]:
new_event_list[-1][2] = event_list[i][2]
else:
new_event_list.append(event_list[i])
else:
new_event_list = []
# write event list
if len(event_list) != 0:
with open(config.root + '/event_detect/detect_result/' +
model_name + '/events_list.csv',
mode='a',
newline='') as f:
csvwriter = csv.writer(f)
for event in event_list:
csvwriter.writerow(event)
f.close()
if plot:
print('Plot detected events.')
for event in new_event_list:
plot_traces = st_Z
event_num, _, start_flag, end_flag, confidence, pos_num, start, end, group_max_conf = event
name = config.root + '/event_detect/detect_result/png/' \
+ str(int(event_num)) + '_' + str(confidence)[:4] + '.png'
plot_traces.plot(starttime=start_flag,
endtime=end_flag,
size=(800, 800),
automerge=False,
equal_scale=False,
linewidth=0.8,
outfile=name)
plot_traces2 = st_all[start:end]
name2 = config.root + '/event_detect/detect_result/png2/' \
+ str(int(event_num)) + '_' + str(group_max_conf)[:4] + '.png'
plot_traces2.plot(starttime=start_flag,
endtime=end_flag,
size=(800, 800),
automerge=False,
equal_scale=False,
linewidth=0.8,
outfile=name2)
## cut use Obspy, processed data
# if cut:
# print('Cut detected events.')
# for event in new_event_list:
# event_num, _, start_flag, end_flag, confidence, pos_num, start, end, group_max_conf = event
# slice_E = st_E.slice(start_flag, end_flag)
# slice_N = st_N.slice(start_flag, end_flag)
# slice_Z = st_Z.slice(start_flag, end_flag)
# save_path = config.root + '/event_detect/detect_result/cut/' \
# + str(int(event_num)) + '_' + str(confidence)[:4]
# os.system('mkdir %s'%save_path)
# for i in range(len(slice_E)):
# slice_E[i].write(save_path+'/'+sac_file_name[0][i], format='SAC')
# slice_N[i].write(save_path+'/'+sac_file_name[1][i], format='SAC')
# slice_Z[i].write(save_path+'/'+sac_file_name[2][i], format='SAC')
## cut use SAC, raw data
if cut:
print('Cut detected events.')
for event in new_event_list:
event_num, _, start_flag, end_flag, confidence, pos_num, start, end, group_max_conf = event
save_path = config.root + '/event_detect/detect_result/cut/' \
+ str(int(event_num)) + '_' + str(confidence)[:4] + '/'
os.system('mkdir %s' % save_path)
cut_b = 60 * 60 * int(start_flag.hour) + 60 * int(
start_flag.minute) + float(start_flag.second)
cut_e = 60 * 60 * int(end_flag.hour) + 60 * int(
end_flag.minute) + float(end_flag.second)
## SAC
os.putenv("SAC_DISPLAY_COPYRIGHT", "0")
p = subprocess.Popen(['sac'], stdin=subprocess.PIPE)
s = ''
s += "cut %s %s \n" % (cut_b, cut_e)
s += "r %s/* \n" % (config.root + '/data/after/' + file)
s += "w dir %s over \n" % (save_path)
s += "quit \n"
p.communicate(s.encode())
start_point += 1
with open(config.root + '/event_detect/detect_result/' + model_name +
'/checkpoint',
mode='w') as f:
f.write(str(start_point) + '\n')
f.write(str(event_num))
end = datetime.datetime.now()
print('{} completed, num {}, time {}.'.format(
file, start_point, end - begin))
print('Checkpoint saved.')
def main(model_name, new_scan=False, preprocess=True):
reader = Reader()
config = Config()
reader.aftername.sort()
if new_scan == True:
print('start new scan!')
file_list = reader.aftername
start_point = 0
else:
with open('detect_result/' + model_name + '/checkpoint') as file:
start_point = int(file.readline())
file_list = reader.aftername[start_point:]
print('restart from {}'.format(file_list[0]))
if model_name == 'cnn':
from event_detect.cnn import CNN
import tensorflow as tf
from tflib.models import Model
model = CNN()
sess = tf.Session()
saver, global_step = Model.continue_previous_session(
sess, model_file='cnn', ckpt_file='saver/cnn/checkpoint')
if model_name == 'cldnn':
from event_detect.cldnn import CLDNN
import tensorflow as tf
from tflib.models import Model
model = CLDNN()
sess = tf.Session()
saver, global_step = Model.continue_previous_session(
sess, model_file='cldnn', ckpt_file='saver/cldnn/checkpoint')
for file in file_list:
begin = datetime.datetime.now()
traces = obspy.read(file[0])
traces = traces + obspy.read(file[1])
traces = traces + obspy.read(file[2])
if not (traces[0].stats.starttime == traces[1].stats.starttime
and traces[0].stats.starttime == traces[2].stats.starttime):
starttime = max([
traces[0].stats.starttime, traces[1].stats.starttime,
traces[2].stats.starttime
])
for j in range(3):
traces[j] = traces[j].slice(starttime=starttime)
if not (traces[0].stats.endtime == traces[1].stats.endtime
and traces[0].stats.endtime == traces[2].stats.endtime):
endtime = min([
traces[0].stats.endtime, traces[1].stats.endtime,
traces[2].stats.endtime
])
for j in range(3):
traces[j] = traces[j].slice(endtime=endtime)
start_flag = -1
end_flag = -1
event_list = []
for windowed_st in traces.slide(window_length=(config.winsize - 1) /
100.0,
step=config.winlag / 100.0):
data_input = []
for j in range(3):
data_input.append(windowed_st[j].data)
if model_name == 'cnn':
# raw_data = [data_preprocess(d, 'bandpass', False) for d in data_input]
data_input = np.array(data_input).T
if preprocess:
# data_input = sklearn.preprocessing.minmax_scale(data_input)
data_mean = np.mean(data_input, axis=0)
data_input = np.absolute(data_input - data_mean)
data_input = data_input / (np.max(data_input, axis=0) +
np.array([1, 1, 1]))
data_input = np.array([np.array([data_input])])
elif model_name == 'cldnn':
# raw_data = [data_preprocess(d, 'bandpass', False) for d in data_input]
data_input = [data_preprocess(d) for d in data_input]
data_input = np.array(data_input).T
data_input = np.array([data_input])
class_pred, confidence = model.classify(sess=sess,
input_=data_input)
if class_pred == 1:
# plt.subplot(3, 1, 1)
# plt.plot(raw_data[0])
# plt.subplot(3, 1, 2)
# plt.plot(raw_data[1])
# plt.subplot(3, 1, 3)
# plt.plot(raw_data[2])
# plt.show()
if start_flag == -1:
start_flag = windowed_st[0].stats.starttime
end_flag = windowed_st[0].stats.endtime
else:
end_flag = windowed_st[0].stats.endtime
if class_pred == 0 and start_flag != -1 and end_flag < windowed_st[
0].stats.starttime:
event = [
file[0].split('\\')[-1][:-4], start_flag, end_flag,
confidence
]
# print(event)
event_list.append(event)
start_flag = -1
end_flag = -1
if len(event_list) != 0:
with open('detect_result/' + model_name + '/events_test.csv',
mode='a',
newline='') as f:
csvwriter = csv.writer(f)
for event in event_list:
csvwriter.writerow(event)
f.close()
start_point += 1
with open('detect_result/' + model_name + '/checkpoint',
mode='w') as f:
f.write(str(start_point))
end = datetime.datetime.now()
print('{} scanned, num {}, time {}.'.format(
file[0].split('\\')[-1][:-4], start_point, end - begin))
print('checkpoint saved.')
def start_sess(self, sess):
saver, global_step = Model.continue_previous_session(sess,
model_file='cblstm',
ckpt_file='eqpickup/saver/cblstm/checkpoint')
def train(self, passes, new_training=True):
sess_config = tf.ConfigProto()
sess_config.gpu_options.allow_growth = True
with tf.Session(config=sess_config) as sess:
if new_training:
saver, global_step = Model.start_new_session(sess)
else:
saver, global_step = Model.continue_previous_session(sess,
model_file='cblstm',
ckpt_file='eqpickup/saver/cblstm/checkpoint')
self.train_writer.add_graph(sess.graph, global_step=global_step)
for step in range(1 + global_step, 1 + passes + global_step):
with tf.variable_scope('Train'):
input_, targets = self.reader.get_cblstm_batch_data('train',
self.reader.pre_data_generator,
self.reader.pre_validation_data)
input_, seq_len = self.data_padding_preprocess(input_, 'input')
targets, _ = self.data_padding_preprocess(targets, 'targets')
_, train_summary, loss, pred_seq = sess.run(
[self.train_op, self.train_merged, self.loss, self.layer['pred_seq']],
feed_dict={self.layer['input']: input_,
self.layer['targets']: targets,
self.layer['seq_len']: seq_len,
self.layer['keep_prob']: self.config.dl_tradition_model_config.cblstm_keep_prob})
self.train_writer.add_summary(train_summary, step)
train_p_err, train_p_err_max, train_s_err, train_s_err_max = get_p_s_error(pred_seq, targets,
seq_len)
train_acc = get_acc(pred_seq, targets, seq_len)
[train_metrics_summary] = sess.run(
[self.train_metrics_merged],
feed_dict={self.train_metrics['acc']: train_acc,
self.train_metrics['p_error']: train_p_err,
self.train_metrics['p_error_max']: train_p_err_max,
self.train_metrics['s_error']: train_s_err,
self.train_metrics['s_error_max']: train_s_err_max})
self.train_writer.add_summary(train_metrics_summary, step)
print("gobal_step {},"
" training_loss {},"
" accuracy {},"
" p_error {},"
" p_err_max {},"
" s_error {},"
" s_err_max {}.".format(step, loss, train_acc, train_p_err, train_p_err_max, train_s_err,
train_s_err_max))
if step % 50 == 0:
with tf.variable_scope('Test', reuse=True):
test_input, test_targets = self.reader.get_cblstm_batch_data('test',
self.reader.pre_data_generator,
self.reader.pre_validation_data)
test_input, test_seq_len = self.data_padding_preprocess(test_input, 'input')
test_targets, _ = self.data_padding_preprocess(test_targets, 'targets')
[test_pred_seq] = sess.run([self.layer['pred_seq']],
feed_dict={self.layer['input']: test_input,
self.layer['seq_len']: test_seq_len,
self.layer['keep_prob']: 1.0})
test_p_err, test_p_err_max, test_s_err, test_s_err_max = get_p_s_error(test_pred_seq,
test_targets,
test_seq_len)
test_acc = get_acc(test_pred_seq,
test_targets,
test_seq_len)
[test_metrics_summary] = sess.run(
[self.test_metrics_merged],
feed_dict={self.test_metrics['acc']: test_acc,
self.test_metrics['p_error']: test_p_err,
self.test_metrics['p_error_max']: test_p_err_max,
self.test_metrics['s_error']: test_s_err,
self.test_metrics['s_error_max']: test_s_err_max})
self.train_writer.add_summary(test_metrics_summary, step)
print("test_acc {}, "
"test_p_err {},"
"test_p_err_max {},"
"test_s_err {},"
"test_s_err_max {}.".format(test_acc, test_p_err, test_p_err_max, test_s_err,
test_s_err_max))
if step % 50 == 0:
saver.save(sess, 'eqpickup/saver/cblstm/cblstm', global_step=step)
print('checkpoint saved')
if get_pred_seq:
return p_index, s_index, class_prob, pred_seq
else:
return p_index, s_index, class_prob
if __name__ == '__main__':
# cblstm = CBLSTM()
# cblstm.train(10000, False)
### test acc
cblstm = CBLSTM()
sess_config = tf.ConfigProto()
sess_config.gpu_options.allow_growth = True
sess = tf.Session(config=sess_config)
saver, global_step = Model.continue_previous_session(
sess, model_file='cblstm', ckpt_file='saver/cblstm/checkpoint')
acc_ave = 0
p_err_ave = 0
s_err_ave = 0
p_err_max = []
s_err_max = []
for i in range(20):
test_input, test_targets = cblstm.reader.get_birnn_batch_data('test')
test_targets, test_seq_len = cblstm.data_padding_preprocess(
test_targets, 'targets')
events_p_index, events_s_index, _, pred_seq = cblstm.pickup_p_s(
sess, test_input, get_pred_seq=True)
tmp_acc = get_acc(pred_seq, test_targets, test_seq_len)
acc_ave += tmp_acc
tmp_p_error, tmp_p_err_max, tmp_s_error, tmp_s_err_max = get_p_s_error(
def start_sess(self, sess):
saver, global_step = Model.continue_previous_session(sess,
model_file='cnn',
ckpt_file='eqdetector/saver/cnn/checkpoint')
def main(model_name, new_scan=False, preprocess=True):
reader = Reader()
config = Config()
plot = config.plot
bandpass = config.bandpass
resample = config.resample
confidence0=[]
plot_num = 0
reader.aftername.sort()
if new_scan == True:
print('start new scan!')
file_list = reader.aftername
start_point = 0
else:
with open(config.root + '/event_detect/detect_result/' + model_name + '/checkpoint') as file:
start_point = int(file.readline())
file_list = reader.aftername[start_point:]
print('restart from {}'.format(file_list[0]))
if model_name == 'cnn':
from cnn import CNN
import tensorflow as tf
from tflib.models import Model
model = CNN()
sess = tf.Session(config=tf.ConfigProto(device_count={"CPU":20},inter_op_parallelism_threads=0,intra_op_parallelism_threads=0))
saver, global_step = Model.continue_previous_session(sess,
model_file='cnn',
ckpt_file=config.root + '/event_detect/saver/cnn/checkpoint')
file_list_len = len(file_list)
# print(file_list_len)
try:
os.system('rm -rf %s/event_detect/detect_result/png/*'%config.root)
os.system('rm -rf %s/event_detect/detect_result/cnn/*.csv'%config.root)
except:
pass
for file in file_list:
file=np.array(file)
#print(file)
#file=file.T
#np.random.shuffle(file) #random
#file=file.T
#print(file,'\n')
begin = datetime.datetime.now()
if plot:
plot_traces = obspy.read(file[2][0]) #Z component
sta_num = len(file[0])
trace_len = []
for i in range(3):
for j in range(sta_num):
trace_len.append(obspy.read(file[i][j])[0].stats.npts)
max_len = max(trace_len)
for i in range(3):
for j in range(sta_num): # station number
each_tr = obspy.read(file[i][j])
if each_tr[0].stats.npts < max_len:
zero = np.zeros(max_len-each_tr[0].stats.npts)
each_tr[0].data = np.concatenate([each_tr[0].data,zero])
if i==j==0:
traces = each_tr
else:
traces=traces + each_tr
if i == 2:
if j == 0:
pass
else:
plot_traces = plot_traces + each_tr
if plot:
if resample:
plot_traces = plot_traces.resample(sampling_rate=resample)
plot_traces = plot_traces.filter('bandpass',freqmin=bandpass[0],freqmax=bandpass[1],corners=4,zerophase=True)
if resample:
traces = traces.resample(sampling_rate=resample)
traces = traces.filter('bandpass',freqmin=bandpass[0],freqmax=bandpass[1],corners=4,zerophase=True)
starttime = traces[0].stats.starttime;
endtime = traces[0].stats.endtime;
#print(traces)
start_flag = -1
end_flag = -1
event_list = []
confidence_total=[]
start_total=[]
end_total=[]
samples_trace= 1.0/traces[0].stats.delta;
npts = traces[0].stats.npts
for windowed_st in traces.slide(window_length=(config.winsize-1)/samples_trace,
step=config.winlag / samples_trace):
data_input = [[],[],[]]
for j in range(sta_num):
if len(windowed_st[j].data) < config.winsize:
windowed_st[j].data = np.concatenate([windowed_st[j].data,np.zeros(config.winsize-len(windowed_st[j].data))])
data_input[0].append(windowed_st[j].data[:config.winsize])
for j in range(sta_num,2*sta_num):
if len(windowed_st[j].data) < config.winsize:
windowed_st[j].data = np.concatenate([windowed_st[j].data,np.zeros(config.winsize-len(windowed_st[j].data))])
data_input[1].append(windowed_st[j].data[:config.winsize])
for j in range(2*sta_num,3*sta_num):
if len(windowed_st[j].data) < config.winsize:
windowed_st[j].data = np.concatenate([windowed_st[j].data,np.zeros(config.winsize-len(windowed_st[j].data))])
data_input[2].append(windowed_st[j].data[:config.winsize])
if model_name == 'cnn':
if preprocess:
for i in range(3):
for j in range(sta_num):
data_input[i][j] = data_preprocess(data_input[i][j])
data_input=np.array(data_input)
if len(data_input[0][0])<config.winsize:
concat = np.zeros([3, sta_num, config.winsize - len(data_input[0][0])])
data_input=np.concatenate([data_input,concat],axis=2)
if len(data_input[0][0])>config.winsize:
data_input=data_input[:, :, :config.winsize]
data_input=data_input.transpose((1,2,0))
data_input = np.array([data_input])
#print(event_list)
class_pred, confidence = model.classify(sess=sess, input_=data_input)
confidence0.append(confidence)
print(class_pred,confidence)
if class_pred == 1:
confidence_total.append(confidence)
start_total.append(windowed_st[0].stats.starttime)
end_total.append(windowed_st[0].stats.endtime)
if start_flag == -1:
start_flag = windowed_st[0].stats.starttime
end_flag = windowed_st[0].stats.endtime
else:
end_flag = windowed_st[0].stats.endtime
print(class_pred,start_flag,end_flag,windowed_st[0].stats.starttime)
if class_pred == 0 and start_flag != -1: #end_flag < windowed_st[0].stats.starttime:
confidence = np.max(confidence_total)
for j in range(len(confidence_total)):
if confidence == confidence_total[j]:
break
start_local = start_total[j]
end_local = end_total[j]
a=True
# event = [file[0][0].split('/')[-2], start_flag, end_flag,
# confidence, start_local, end_local]
event = [file[0][0].split('/')[-2], start_flag, end_flag, confidence]
confidence_total=[]
start_total = []
end_total = []
if plot:
plot_num = int(plot_num + 1)
name = config.root + '/event_detect/detect_result/png/' \
+ str(plot_num) + '_' + str(confidence) + '.png'
plot_traces.plot(starttime=start_flag, endtime=end_flag, size=(800, 800),
automerge=False, equal_scale=False, linewidth=0.8, outfile=name)
# print(event)
event_list.append(event)
#print(event_list)
start_flag = -1
end_flag = -1
if class_pred == 1 and end_flag+config.winlag / samples_trace>=endtime:
confidence = np.max(confidence_total)
for j in range(len(confidence_total)):
if confidence == confidence_total[j]:
break
start_local = start_total[j]
end_local = end_total[j]
if plot:
plot_num = int(plot_num + 1)
name = config.root + '/event_detect/detect_result/png/' \
+ str(plot_num) + '_' + str(confidence) + '.png'
plot_traces.plot(starttime=start_flag, endtime=endtime, size=(800, 800),
automerge=False, equal_scale=False, linewidth=0.8, outfile=name)
# event = [file[0][0].split('/')[-2], start_flag, endtime,
# confidence, start_total, end_total]
event = [file[0][0].split('/')[-2], start_flag, endtime, confidence]
event_list.append(event)
start_flag = -1
end_flag = -1
if len(event_list) != 0:
with open(config.root + '/event_detect/detect_result/' + model_name + '/events_list.csv', mode='a', newline='') as f:
csvwriter = csv.writer(f)
for event in event_list:
csvwriter.writerow(event)
f.close()
start_point += 1
with open(config.root + '/event_detect/detect_result/' + model_name + '/checkpoint', mode='w') as f:
f.write(str(start_point))
end = datetime.datetime.now()
print('{} scanned, num {}, time {}.'.format(file[0][0].split('/')[-2], start_point, end - begin))
print('checkpoint saved.')
def train(self, passes, new_training=True):
sess_config = tf.ConfigProto()
sess_config.gpu_options.allow_growth = True
sess_config.allow_soft_placement = True
with tf.Session(config=sess_config) as sess:
if new_training:
saver, global_step = Model.start_new_session(sess)
else:
saver, global_step = Model.continue_previous_session(sess,
model_file='model/saver/{}'.format(self.saveFile),
ckpt_file='model/saver/{}/checkpoint'.format(
self.saveFile))
self.train_writer.add_graph(sess.graph, global_step=global_step)
walk_times = 1
for step in range(1 + global_step, 1 + passes + global_step):
with tf.variable_scope('Train'):
walk_nodes = self.reader.nodes_walk_reader()
neg_walk_nodes = [self.g.negative_sample(walk_nodes[i],
self.config.loss1_neg_sample_num,
self.g.nodes_degree_table)
for i in range(len(walk_nodes))]
neg_walk_nodes = np.array(neg_walk_nodes)
walk_nodes_labels = list()
for node_list in walk_nodes:
nodes_label_tmp = self.g.get_train_node_label(node_list)
walk_nodes_labels.append(nodes_label_tmp)
walk_nodes_labels = np.array(walk_nodes_labels)
# if (step - 1) % int(self.g.nodes_num / self.config.nodes_seq_batch_num) == 0:
# print(walk_times)
# walk_times += 1
if step < 200 and self.init_emb_file is not None:
train_op = self.train_op[1]
else:
train_op = self.train_op[0]
_, train_summary, loss = sess.run(
[train_op,
self.loss_train_merged,
self.layer['loss']],
feed_dict={self.layer['walk_nodes']: walk_nodes,
self.layer['walk_nodes_labels']: walk_nodes_labels,
self.layer['neg_walk_nodes']: neg_walk_nodes})
self.train_writer.add_summary(train_summary, step)
if step % 500 == 0 or step == 1:
[node_emb, sup_emb] = sess.run([self.layer['emb'],
self.layer['sup_emb']])
node_emb = np.concatenate((node_emb, sup_emb), axis=1)
print("gobal_step {},loss {}".format(step, loss))
if step % 1000 == 0 or step == 1:
micro_f1, macro_f1 = self.multi_label_node_classification(node_emb)
[test_summary] = sess.run([self.test_merged],
feed_dict={self.test_metrics['micro_f1']: micro_f1,
self.test_metrics['macro_f1']: macro_f1})
print("micro_f1 {},macro_f1 {}".format(micro_f1, macro_f1))
self.train_writer.add_summary(test_summary, step)
saver.save(sess, 'model/saver/{}/MPRSNE'.format(self.saveFile), global_step=step)
print('checkpoint saved')