def main(_):
cfg = config.Config()
cfg.batch_size = 1
cfg.n_epochs = 1
data_pipeline = dpp.DataPipeline(FLAGS.data_path,
config=cfg,
is_training=False)
samples = data_pipeline.samples
labels = data_pipeline.labels
start_time = data_pipeline.start_time
end_time = data_pipeline.end_time
with tf.Session() as sess:
coord = tf.train.Coordinator()
tf.initialize_local_variables().run()
threads = tf.train.start_queue_runners(coord=coord)
try:
for i in (range(FLAGS.windows)):
to_fetch = [samples, labels, start_time, end_time]
sample, label, starttime, endtime = sess.run(to_fetch)
# assert starttime < endtime
print('starttime {}, endtime {}'.format(
UTCDateTime(starttime), UTCDateTime(endtime)))
print("label", label[0])
sample = np.squeeze(sample, axis=(0, ))
target = np.squeeze(label, axis=(0, ))
except tf.errors.OutOfRangeError:
print 'Evaluation completed ({} epochs).'.format(cfg.n_epochs)
print "{} windows seen".format(i + 1)
coord.request_stop()
coord.join(threads)
def fetch_window_data(stream):
"""fetch data from a stream window and dump in np array"""
cfg = config.Config()
data = np.empty((cfg.win_size, 3))
for i in range(3):
data[:, i] = stream[i].data.astype(np.float32)
data = np.expand_dims(data, 0)
return data
def data_is_complete(stream):
"""Returns True if there is 1001*3 points in win"""
cfg = config.Config()
try:
data_size = len(stream[0].data) + len(stream[1].data) + len(
stream[2].data)
except:
data_size = 0
data_lenth = int(cfg.win_size) * 3
if data_size == data_lenth:
return True
else:
return False
def main(args):
setproctitle.setproctitle('quakenet')
tf.set_random_seed(1234)
if args.n_clusters == None:
raise ValueError('Define the number of clusters with --n_clusters')
cfg = config.Config()
cfg.batch_size = args.batch_size
cfg.add = 1
cfg.n_clusters = args.n_clusters
cfg.n_clusters += 1
pos_path = os.path.join(args.dataset, "positive")
neg_path = os.path.join(args.dataset, "negative")
# data pipeline for positive and negative examples
pos_pipeline = dp.DataPipeline(pos_path, cfg, True)
neg_pipeline = dp.DataPipeline(neg_path, cfg, True)
pos_samples = {
'data': pos_pipeline.samples,
'cluster_id': pos_pipeline.labels
}
neg_samples = {
'data': neg_pipeline.samples,
'cluster_id': neg_pipeline.labels
}
samples = {
"data":
tf.concat(0, [pos_samples["data"], neg_samples["data"]]),
"cluster_id":
tf.concat(0, [pos_samples["cluster_id"], neg_samples["cluster_id"]])
}
# model
model = models.get(args.model,
samples,
cfg,
args.checkpoint_dir,
is_training=True)
# train loop
model.train(args.learning_rate,
resume=args.resume,
profiling=args.profiling,
summary_step=10)
def main(args):
#setproctitle.setproctitle('quakenet')
tf.set_random_seed(1234)
tf.reset_default_graph()
cfg = config.Config()
cfg.batch_size = args.batch_size
cfg.add = 1
cfg.n_clusters = args.n_clusters
cfg.n_clusters += 1
pos_path = os.path.join(args.dataset,"signal")
neg_path = os.path.join(args.dataset,"noise")
# data pipeline for positive and negative examples
pos_pipeline = dp.DataPipeline(pos_path, cfg, True)
neg_pipeline = dp.DataPipeline(neg_path, cfg, True)
pos_samples = {
'data': pos_pipeline.samples,
'cluster_id': pos_pipeline.labels
}
neg_samples = {
'data': neg_pipeline.samples,
'cluster_id': neg_pipeline.labels
}
samples = {
"data": tf.concat([pos_samples["data"],neg_samples["data"]],0),
"cluster_id" : tf.concat([pos_samples["cluster_id"],neg_samples["cluster_id"]],0)
}
# model
model = models.get(args.model, samples,cfg, args.checkpoint_dir, is_training=True)
# train loop
model.train(
args.learning_rate,
resume=args.resume,
profiling=args.profiling,
summary_step=10)
def main(_):
cfg = config.Config()
cfg.batch_size = 1
# Make dirs
if not os.path.exists(FLAGS.output_path):
os.makedirs(FLAGS.output_path)
data_pipeline = dpp.DataPipeline(FLAGS.data_path,
config=cfg,
is_training=False)
samples = data_pipeline.samples
labels = data_pipeline.labels
with tf.Session() as sess:
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(coord=coord)
print '+ Plotting {} windows'.format(FLAGS.windows)
for i in tqdm(range(FLAGS.windows)):
sample, label = sess.run([samples, labels])
sample = np.squeeze(sample, axis=(0, ))
target = np.squeeze(label, axis=(0, ))
plt.clf()
fig, ax = plt.subplots(3, 1)
for t in range(sample.shape[1]):
ax[t].plot(sample[:, t])
ax[t].set_xlabel('time (samples)')
ax[t].set_ylabel('amplitude')
ax[0].set_title('window {:04d}, cluster_id: {}'.format(i, target))
plt.savefig(
os.path.join(FLAGS.output_path, 'window_{:04d}.pdf'.format(i)))
plt.close()
coord.request_stop()
coord.join(threads)
def train():
"""
Train unet using specified args:
"""
data_files, data_size = load_datafiles(FLAGS.tfrecords_prefix)
print data_files, data_size
# images, labels, filenames = dataset_loader.inputs(
# data_files = data_files,
# image_size = FLAGS.image_size,
# batch_size = FLAGS.batch_size,
# num_epochs = FLAGS.num_epochs,
# train = True)
setproctitle.setproctitle('quakenet')
tf.set_random_seed(1234)
cfg = config.Config()
cfg.batch_size = FLAGS.batch_size
cfg.add = 1
cfg.n_clusters = FLAGS.num_classes
cfg.n_clusters += 1
# data pipeline for positive and negative examples
pos_pipeline = dp.DataPipeline(FLAGS.tfrecords_dir, cfg, True)
# images:[batch_size, n_channels, n_points]
images = pos_pipeline.samples
labels = pos_pipeline.labels
logits = unet.build_30s(images, FLAGS.num_classes, True)
accuarcy = unet.accuracy(logits, labels)
print "accuarcy,recall,f1", accuarcy
#load class weights if available
if FLAGS.class_weights is not None:
weights = np.load(FLAGS.class_weights)
class_weight_tensor = tf.constant(weights,
dtype=tf.float32,
shape=[FLAGS.num_classes, 1])
else:
class_weight_tensor = None
loss = unet.loss(logits, labels, FLAGS.weight_decay_rate)
global_step = tf.Variable(0, name='global_step', trainable=False)
train_op = unet.train(loss, FLAGS.learning_rate,
FLAGS.learning_rate_decay_steps,
FLAGS.learning_rate_decay_rate, global_step)
#print "train_op",train_op
init_op = tf.group(tf.global_variables_initializer(),
tf.local_variables_initializer())
saver = tf.train.Saver()
session_manager = tf.train.SessionManager(
local_init_op=tf.local_variables_initializer())
sess = session_manager.prepare_session("",
init_op=init_op,
saver=saver,
checkpoint_dir=FLAGS.checkpoint_dir)
writer = tf.summary.FileWriter(FLAGS.checkpoint_dir + "/train_logs",
sess.graph)
merged = tf.summary.merge_all()
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(sess=sess, coord=coord)
start_time = time.time()
try:
while not coord.should_stop():
step = tf.train.global_step(sess, global_step)
_, loss_value, summary = sess.run([train_op, loss, merged])
#print loss_value
writer.add_summary(summary, step)
if step % 1000 == 0:
acc_seg_value = sess.run([accuarcy])
#print "acc_seg_value:",acc_seg_value,acc_seg_value[0],acc_seg_value[0][1],acc_seg_value[0][1][0]
epoch = step * FLAGS.batch_size / data_size
#print epoch
duration = time.time() - start_time
#print step,duration
start_time = time.time()
#print('[PROGRESS]\tEpoch %d | Step %d | loss = %.2f | total. acc. = %.2f | P. acc. = %.3f \
# | S. acc. = %.3f | N. acc. = %.3f | dur. = (%.3f sec)'\
# % (epoch, step, loss_value, acc_seg_value[0][1][0],acc_seg_value[0][1][1], acc_seg_value[0][1][2],\
# acc_seg_value[0][3],duration))
print('[PROGRESS]\tEpoch %d | Step %d | loss = %.2f | P. acc. = %.3f \
| S. acc. = %.3f | N. acc. = %.3f | dur. = (%.3f sec)'\
% (epoch, step, loss_value, acc_seg_value[0][1][1],acc_seg_value[0][1][2], acc_seg_value[0][1][0],\
duration))
if step % 5000 == 0:
print('[PROGRESS]\tSaving checkpoint')
checkpoint_path = os.path.join(FLAGS.checkpoint_dir,
'unet.ckpt')
saver.save(sess, checkpoint_path, global_step=step)
except tf.errors.OutOfRangeError:
print('[INFO ]\tDone training for %d epochs, %d steps.' %
(FLAGS.num_epochs, step))
finally:
# When done, ask the threads to stop.
coord.request_stop()
# Wait for threads to finish.
coord.join(threads)
writer.close()
sess.close()
def main(_):
setproctitle.setproctitle('quakenet_viz')
ckpt = tf.train.get_checkpoint_state(args.checkpoint_dir)
cfg = config.Config()
cfg.batch_size = 1
cfg.n_clusters = args.n_clusters
cfg.add = 1
cfg.n_clusters += 1
# Remove previous output directory
if os.path.exists(args.output):
shutil.rmtree(args.output)
os.makedirs(args.output)
windows_list = fetch_streams_list(args.dataset)
# stream data with a placeholder
samples = {
'data': tf.placeholder(tf.float32,
shape=(cfg.batch_size, 1001, 3),
name='input_data'),
'cluster_id': tf.placeholder(tf.int64,
shape=(cfg.batch_size,),
name='input_label')
}
# set up model and validation metrics
model = models.get(args.model, samples, cfg,
args.checkpoint_dir,
is_training=False)
metrics = model.validation_metrics()
with tf.Session() as sess:
model.load(sess, args.step)
print 'Evaluating at step {}'.format(sess.run(model.global_step))
step = tf.train.global_step(sess, model.global_step)
mean_metrics = {}
for key in metrics:
mean_metrics[key] = 0
for n in range(len(windows_list)):
# Get One stream and label from the list
stream, cluster_id = fetch_window_and_label(windows_list[n])
# Get coordinates of the event
lat_event, lon_event = fetch_lat_and_lon(windows_list[n])
# Fetch class_proba and label
to_fetch = [samples['data'],
metrics,
model.layers['class_prob']]
feed_dict = {samples['data']: stream,
samples['cluster_id']: cluster_id}
sample, metrics_, class_prob_= sess.run(to_fetch,
feed_dict)
# Keep only clusters proba, remove noise proba
clusters_prob = class_prob_[0,1::]
# Print Misclassified window
if metrics_['localization_accuracy'] >= 1.0:
map_file ='cluster_ids_{}_comp.npy'.format(args.n_clusters)
clusters_map = np.load(map_file)
lat = np.load("cluster_ids_{}_comp_lat.npy".format(args.n_clusters))
lon = np.load("cluster_ids_{}_comp_lon.npy".format(args.n_clusters))
plot_proba_map(n, lat, lon, clusters_map, clusters_prob,
cluster_id, lat_event, lon_event)
for key in metrics:
mean_metrics[key] += cfg.batch_size * metrics_[key]
mess = model.validation_metrics_message(metrics_)
print '{:03d} | '.format(n) + mess
for key in metrics:
mean_metrics[key] /= len(windows_list)
mess = model.validation_metrics_message(mean_metrics)
print 'Average | ' + mess
def evaluate():
"""
Eval unet using specified args:
"""
if FLAGS.events:
summary_dir = os.path.join(FLAGS.checkpoint_path, "events")
while True:
ckpt = tf.train.get_checkpoint_state(FLAGS.checkpoint_path)
if FLAGS.eval_interval < 0 or ckpt:
print('Evaluating model')
break
print('Waiting for training job to save a checkpoint')
time.sleep(FLAGS.eval_interval)
#data_files, data_size = load_datafiles(FLAGS.tfrecords_prefix)
setproctitle.setproctitle('quakenet')
tf.set_random_seed(1234)
cfg = config.Config()
cfg.batch_size = FLAGS.batch_size
cfg.add = 1
cfg.n_clusters = FLAGS.num_classes
cfg.n_clusters += 1
cfg.n_epochs = 1
model_files = [
file for file in os.listdir(FLAGS.checkpoint_path)
if fnmatch.fnmatch(file, '*.meta')
]
for model_file in sorted(model_files):
step = model_file.split(".meta")[0].split("-")[1]
print(step)
try:
model_file = os.path.join(FLAGS.checkpoint_path, model_file)
# data pipeline for positive and negative examples
pos_pipeline = dp.DataPipeline(FLAGS.tfrecords_dir, cfg, True)
# images:[batch_size, n_channels, n_points]
images = pos_pipeline.samples
labels = pos_pipeline.labels
logits = unet.build_30s(images, FLAGS.num_classes, False)
predicted_images = unet.predict(logits, FLAGS.batch_size,
FLAGS.image_size)
accuracy = unet.accuracy(logits, labels)
loss = unet.loss(logits, labels, FLAGS.weight_decay_rate)
summary_writer = tf.summary.FileWriter(summary_dir, None)
init_op = tf.group(tf.global_variables_initializer(),
tf.local_variables_initializer())
sess = tf.Session()
sess.run(init_op)
saver = tf.train.Saver()
#if not tf.gfile.Exists(FLAGS.checkpoint_path + '.meta'):
if not tf.gfile.Exists(model_file):
raise ValueError("Can't find checkpoint file")
else:
print('[INFO ]\tFound checkpoint file, restoring model.')
saver.restore(sess, model_file.split(".meta")[0])
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(sess=sess, coord=coord)
#metrics = validation_metrics()
global_accuracy = 0.0
global_p_accuracy = 0.0
global_s_accuracy = 0.0
global_n_accuracy = 0.0
global_loss = 0.0
n = 0
#mean_metrics = {}
#for key in metrics:
# mean_metrics[key] = 0
#pred_labels = np.empty(1)
#true_labels = np.empty(1)
try:
while not coord.should_stop():
acc_seg_value, loss_value, predicted_images_value, images_value = sess.run(
[accuracy, loss, predicted_images, images])
accuracy_p_value = acc_seg_value[1][1]
accuracy_s_value = acc_seg_value[1][2]
accuracy_n_value = acc_seg_value[1][0]
#pred_labels = np.append(pred_labels, predicted_images_value)
#true_labels = np.append(true_labels, images_value)
global_accuracy += acc_seg_value
global_p_accuracy += accuracy_p_value
global_s_accuracy += accuracy_s_value
global_n_accuracy += accuracy_n_value
global_loss += loss_value
# print true_labels
#for key in metrics:
# mean_metrics[key] += cfg.batch_size * metrics_[key]
filenames_value = []
# for i in range(FLAGS.batch_size):
# filenames_value.append(str(step)+"_"+str(i)+".png")
#print (predicted_images_value[:,100:200])
if (FLAGS.plot):
maybe_save_images(predicted_images_value, images_value,
filenames_value)
#s='loss = {:.5f} | det. acc. = {:.1f}% | loc. acc. = {:.1f}%'.format(metrics['loss']
print(
'[PROGRESS]\tAccuracy for current batch: | P. acc. =%.5f| S. acc. =%.5f| '
'noise. acc. =%.5f.' %
(accuracy_p_value, accuracy_s_value, accuracy_n_value))
n += cfg.batch_size
# step += 1
print(n)
except KeyboardInterrupt:
print('stopping evaluation')
except tf.errors.OutOfRangeError:
print('Evaluation completed ({} epochs).'.format(cfg.n_epochs))
print("{} windows seen".format(n))
#print('[INFO ]\tDone evaluating in %d steps.' % step)
if n > 0:
loss_value /= n
summary = tf.Summary(value=[
tf.Summary.Value(tag='loss/val',
simple_value=loss_value)
])
if FLAGS.save_summary:
summary_writer.add_summary(summary, global_step=step)
global_accuracy /= n
global_p_accuracy /= n
global_s_accuracy /= n
global_n_accuracy /= n
summary = tf.Summary(value=[
tf.Summary.Value(tag='accuracy/val',
simple_value=global_accuracy)
])
if FLAGS.save_summary:
summary_writer.add_summary(summary, global_step=step)
summary = tf.Summary(value=[
tf.Summary.Value(tag='accuracy/val_p',
simple_value=global_p_accuracy)
])
if FLAGS.save_summary:
summary_writer.add_summary(summary, global_step=step)
summary = tf.Summary(value=[
tf.Summary.Value(tag='accuracy/val_s',
simple_value=global_s_accuracy)
])
if FLAGS.save_summary:
summary_writer.add_summary(summary, global_step=step)
summary = tf.Summary(value=[
tf.Summary.Value(tag='accuracy/val_noise',
simple_value=global_n_accuracy)
])
if FLAGS.save_summary:
summary_writer.add_summary(summary, global_step=step)
print(
'[End of evaluation for current epoch]\n\nAccuracy for current epoch:%s | total. acc. =%.5f| P. acc. =%.5f| S. acc. =%.5f| '
'noise. acc. =%.5f.' %
(step, global_accuracy, global_p_accuracy,
global_s_accuracy, global_n_accuracy))
print('Sleeping for {}s'.format(FLAGS.eval_interval))
time.sleep(FLAGS.eval_interval)
summary_writer.flush()
finally:
# When done, ask the threads to stop.
coord.request_stop()
tf.reset_default_graph()
#print('Sleeping for {}s'.format(FLAGS.eval_interval))
#time.sleep(FLAGS.eval_interval)
finally:
print('joining data threads')
coord = tf.train.Coordinator()
coord.request_stop()
#pred_labels = pred_labels[1::]
#true_labels = true_labels[1::]
#print ("---Confusion Matrix----")
#print (confusion_matrix(true_labels, pred_labels))
# Wait for threads to finish.
coord.join(threads)
sess.close()
def main(args):
setproctitle.setproctitle('quakenet_eval')
if args.n_clusters == None:
raise ValueError('Define the number of clusters with --n_clusters')
ckpt = tf.train.get_checkpoint_state(args.checkpoint_dir)
cfg = config.Config()
cfg.batch_size = 1
cfg.n_clusters = args.n_clusters
cfg.add = 1
cfg.n_clusters += 1
cfg.n_epochs = 1
# Remove previous output directory
if os.path.exists(args.output):
shutil.rmtree(args.output)
os.makedirs(args.output)
if args.plot:
os.makedirs(os.path.join(args.output, "viz"))
# data pipeline
data_pipeline = DataPipeline(args.dataset, config=cfg, is_training=False)
samples = {
'data': data_pipeline.samples,
'cluster_id': data_pipeline.labels,
'start_time': data_pipeline.start_time,
'end_time': data_pipeline.end_time
}
# set up model and validation metrics
model = models.get(args.model,
samples,
cfg,
args.checkpoint_dir,
is_training=False)
if args.max_windows is None:
max_windows = 2**31
else:
max_windows = args.max_windows
# Dictonary to store info on detected events
events_dic = {
"start_time": [],
"end_time": [],
"utc_timestamp": [],
"cluster_id": [],
"clusters_prob": []
}
# Create catalog name in which the events are stored
output_catalog = os.path.join(args.output, 'catalog_detection.csv')
print 'Catalog created to store events', output_catalog
# Run ConvNetQuake
with tf.Session(config=tf_config) as sess:
coord = tf.train.Coordinator()
tf.initialize_local_variables().run()
threads = tf.train.start_queue_runners(sess=sess, coord=coord)
model.load(sess, args.step)
print 'Predicting using model at step {}'.format(
sess.run(model.global_step))
step = tf.train.global_step(sess, model.global_step)
n_events = 0
idx = 0
time_start = time.time()
while True:
try:
# Fetch class_proba and label
to_fetch = [
samples['data'], model.layers['class_prob'],
model.layers['class_prediction'], samples['start_time'],
samples['end_time']
]
sample, class_prob_, cluster_id, start_time, end_time = sess.run(
to_fetch)
# # Keep only clusters proba, remove noise proba
clusters_prob = class_prob_[0, 1::]
cluster_id -= 1
# label for noise = -1, label for cluster \in {0:n_clusters}
is_event = cluster_id[0] > -1
if is_event:
n_events += 1
idx += 1
if idx % 1000 == 0:
print "processed {} windows".format(idx)
if is_event:
events_dic["start_time"].append(UTCDateTime(start_time))
events_dic["end_time"].append(UTCDateTime(end_time))
events_dic["utc_timestamp"].append(
(start_time + end_time) / 2.0)
events_dic["cluster_id"].append(cluster_id[0])
events_dic["clusters_prob"].append(list(clusters_prob))
if idx >= max_windows:
print "stopped after {} windows".format(max_windows)
print "found {} events".format(n_events)
break
except KeyboardInterrupt:
print "processed {} windows, found {} events".format(
idx + 1, n_events)
print "Run time: ", time.time() - time_start
except tf.errors.OutOfRangeError:
print 'Evaluation completed ({} epochs).'.format(cfg.n_epochs)
break
print 'joining data threads'
m, s = divmod(time.time() - time_start, 60)
print "Prediction took {} min {} seconds".format(m, s)
coord.request_stop()
coord.join(threads)
# Dump dictionary into csv file
df = pd.DataFrame.from_dict(events_dic)
df.to_csv(output_catalog)
def main(args):
setproctitle.setproctitle('quakenet_eval')
if args.n_clusters == None:
raise ValueError('Define the number of clusters with --n_clusters')
ckpt = tf.train.get_checkpoint_state(args.checkpoint_dir)
cfg = config.Config()
#cfg.batch_size = 64
cfg.n_clusters = args.n_clusters
cfg.add = 1
cfg.n_clusters += 1
cfg.n_epochs = 1
# Remove previous output directory
if os.path.exists(args.output):
shutil.rmtree(args.output)
os.makedirs(args.output)
if args.plot:
os.makedirs(os.path.join(args.output, "viz"))
# data pipeline
data_generator = dg.DataGenerator(args.dataset, cfg, is_training=False)
dataset = data_generator.read()
#samples = {
# 'data': data_pipeline.samples,
# 'cluster_id': data_pipeline.labels,
# 'start_time': data_pipeline.start_time,
# 'end_time': data_pipeline.end_time}
# set up model and validation metrics
#model = models.get(args.model, samples, cfg,
# args.checkpoint_dir,
# is_training=False)
# load json and create model
json_file = open("models/model.json", 'r')
loaded_model_json = json_file.read()
json_file.close()
loaded_model = model_from_json(loaded_model_json)
# load weights into new model
loaded_model.load_weights("models/model.h5")
print("Loaded model from disk")
loaded_model.compile(optimizer='adam',
loss='sparse_categorical_crossentropy',
metrics=['accuracy'])
if args.max_windows is None:
max_windows = 2**31
else:
max_windows = args.max_windows
# Dictonary to store info on detected events
events_dic = {
"start_time": [],
"end_time": [],
"utc_timestamp": [],
"cluster_id": [],
"clusters_prob": []
}
# Create catalog name in which the events are stored
output_catalog = os.path.join(args.output, 'catalog_detection.csv')
print('Catalog created to store events', output_catalog)
n_events = 0
idx = 0
time_start = time.time()
clusters_prob = loaded_model.predict(dataset, steps=max_windows)
print(clusters_prob.shape)
cluster_id = np.argmax(clusters_prob, axis=1)
print(cluster_id.shape)
features = []
with tf.Session() as sess:
data_generator = dg.DataGenerator(args.dataset, cfg, is_training=False)
dataset = data_generator.getFeatures()
dataset = dataset.batch(1)
iterator = dataset.make_one_shot_iterator()
n = iterator.get_next()
for idx in range(cluster_id.shape[0]):
values = sess.run(n)
if (cluster_id[idx] > 0):
features.append(
dict({
"start_time": values['start_time'],
"end_time": values['end_time'],
"cluster_id": cluster_id[idx]
}))
events_dic["start_time"].append(
UTCDateTime(values['start_time']))
events_dic["end_time"].append(UTCDateTime(values['end_time']))
events_dic["utc_timestamp"].append(
(values['start_time'] + values['end_time']) / 2.0)
events_dic["cluster_id"].append(cluster_id[idx])
events_dic["clusters_prob"].append(list(clusters_prob[idx]))
for i in range(len(features)):
print(i, features[i]['start_time'], features[i]['end_time'],
features[i]['cluster_id'])
# label for noise = -1, label for cluster \in {0:n_clusters}
m, s = divmod(time.time() - time_start, 60)
print("Prediction took {} min {} seconds".format(m, s))
# Dump dictionary into csv file
df = pd.DataFrame.from_dict(events_dic)
df.to_csv(output_catalog)
def main(_):
if FLAGS.stretch_data:
print "ADD NOISE AND STRETCH DATA"
if FLAGS.compress_data:
print "ADD NOISE AND COMPRESS DATA"
if FLAGS.shift_data:
print "ADD NOISE AND SHIFT DATA"
# Make dirs
output_dir = os.path.split(FLAGS.output)[0]
if not os.path.exists(output_dir):
os.makedirs(output_dir)
if FLAGS.plot:
if not os.path.exists(os.path.join(output_dir, "true_data")):
os.makedirs(os.path.join(output_dir, "true_data"))
if not os.path.exists(os.path.join(output_dir, "augmented_data")):
os.makedirs(os.path.join(output_dir, "augmented_data"))
cfg = config.Config()
cfg.batch_size = 1
cfg.n_epochs = 1
data_pipeline = DataPipeline(FLAGS.tfrecords,
config=cfg,
is_training=False)
samples = data_pipeline.samples
labels = data_pipeline.labels
with tf.Session() as sess:
coord = tf.train.Coordinator()
tf.initialize_local_variables().run()
threads = tf.train.start_queue_runners(coord=coord)
output_tfrecords = FLAGS.output
writer = DataWriter(output_tfrecords)
n_examples = 0
while True:
try:
sample, label = sess.run([samples, labels])
sample = np.squeeze(sample, axis=0)
label = label[0]
noised_sample = add_noise_to_signal(np.copy(sample))
if FLAGS.compress_data:
noised_sample = compress_signal(noised_sample)
if FLAGS.stretch_data:
noised_sample = stretch_signal(noised_sample)
if FLAGS.shift_data:
noised_sample = shift_signal(noised_sample)
if FLAGS.plot:
plot_true_and_augmented_data(sample, noised_sample, label,
n_examples)
stream = convert_np_to_stream(noised_sample)
writer.write(stream, label)
n_examples += 1
except KeyboardInterrupt:
print 'stopping data augmentation'
break
except tf.errors.OutOfRangeError:
print 'Augmentation completed ({} epochs, {} examples seen).'\
.format(cfg.n_epochs,n_examples-1)
break
writer.close()
coord.request_stop()
coord.join(threads)
def main(args):
setproctitle.setproctitle('quakenet_predict')
ckpt = tf.train.get_checkpoint_state(args.checkpoint_dir)
cfg = config.Config()
cfg.batch_size = 1
cfg.n_clusters = args.n_clusters
cfg.add = 1
cfg.n_clusters += 1
# Remove previous output directory
if os.path.exists(args.output):
shutil.rmtree(args.output)
os.makedirs(args.output)
if args.plot:
os.makedirs(os.path.join(args.output, "viz"))
if args.save_sac:
os.makedirs(os.path.join(args.output, "sac"))
# Load stream
stream_path = args.stream_path
stream_file = os.path.split(stream_path)[-1]
print("+ Loading Stream {}".format(stream_file))
stream = read(stream_path)
print('+ Preprocessing stream')
stream = preprocess_stream(stream)
# # TODO: change and look at all streams
# stream_path = args.stream_path
# stream_file = os.path.split(stream_path)[-1]
# print " + Loading stream {}".format(stream_file)
# stream = load_stream(stream_path)
# print " + Preprocess stream"
# stream = preprocess_stream(stream)
# print " -- Stream is ready, starting detection"
# Create catalog name in which the events are stored
catalog_name = os.path.split(stream_file)[-1].split(".mseed")[0] + ".csv"
output_catalog = os.path.join(args.output, catalog_name)
print('Catalog created to store events', output_catalog)
# Dictonary to store info on detected events
events_dic = {
"start_time": [],
"end_time": [],
"cluster_id": [],
"clusters_prob": []
}
# Windows generator
win_gen = stream.slide(window_length=args.window_size,
step=args.window_step,
include_partial_windows=False)
if args.max_windows is None:
total_time_in_sec = stream[0].stats.endtime - stream[0].stats.starttime
max_windows = (total_time_in_sec - args.window_size) / args.window_step
else:
max_windows = args.max_windows
# stream data with a placeholder
samples = {
'data':
tf.placeholder(tf.float32,
shape=(cfg.batch_size, 1001, 3),
name='input_data'),
'cluster_id':
tf.placeholder(tf.int64, shape=(cfg.batch_size, ), name='input_label')
}
# set up model and validation metrics
model = models.get(args.model,
samples,
cfg,
args.checkpoint_dir,
is_training=False)
with tf.Session() as sess:
model.load(sess, args.step)
print('Predicting using model at step {}'.format(
sess.run(model.global_step)))
step = tf.train.global_step(sess, model.global_step)
n_events = 0
time_start = time.time()
try:
for idx, win in enumerate(win_gen):
# Fetch class_proba and label
to_fetch = [
samples['data'], model.layers['class_prob'],
model.layers['class_prediction']
]
# Feed window and fake cluster_id (needed by the net) but
# will be predicted
if data_is_complete(win):
feed_dict = {
samples['data']: fetch_window_data(win),
samples['cluster_id']: np.array([0])
}
sample, class_prob_, cluster_id = sess.run(
to_fetch, feed_dict)
else:
continue
# # Keep only clusters proba, remove noise proba
clusters_prob = class_prob_[0, 1::]
cluster_id -= 1
# label for noise = -1, label for cluster \in {0:n_clusters}
is_event = cluster_id[0] > -1
if is_event:
n_events += 1
# print "event {} ,cluster id {}".format(is_event,class_prob_)
if is_event:
events_dic["start_time"].append(win[0].stats.starttime)
events_dic["end_time"].append(win[0].stats.endtime)
events_dic["cluster_id"].append(cluster_id[0])
events_dic["clusters_prob"].append(list(clusters_prob))
if idx % 1000 == 0:
print("Analyzing {} records".format(
win[0].stats.starttime))
if args.plot and is_event:
# if args.plot:
win_filtered = win.copy()
# win_filtered.filter("bandpass",freqmin=4.0, freqmax=16.0)
win_filtered.plot(outfile=os.path.join(
args.output, "viz", "event_{}_cluster_{}.png".format(
idx, cluster_id)))
if args.save_sac and is_event:
win_filtered = win.copy()
win_filtered.write(os.path.join(
args.output, "sac",
"event_{}_cluster_{}.sac".format(idx, cluster_id)),
format="SAC")
if idx >= max_windows:
print("stopped after {} windows".format(max_windows))
print("found {} events".format(n_events))
break
except KeyboardInterrupt:
print('Interrupted at time {}.'.format(win[0].stats.starttime))
print("processed {} windows, found {} events".format(
idx + 1, n_events))
print("Run time: ", time.time() - time_start)
# Dump dictionary into csv file
#TODO
df = pd.DataFrame.from_dict(events_dic)
df.to_csv(output_catalog)
print("Run time: ", time.time() - time_start)
def evaluate():
"""
Eval unet using specified args:
"""
setproctitle.setproctitle('quakenet')
tf.set_random_seed(1234)
cfg = config.Config()
cfg.batch_size = FLAGS.batch_size
cfg.add = 1
cfg.n_clusters = FLAGS.num_classes
cfg.n_clusters += 1
# data pipeline for positive and negative examples
pos_pipeline = dp.DataPipeline(FLAGS.tfrecords_dir, cfg, True)
# images:[batch_size, n_channels, n_points]
images = pos_pipeline.samples
labels = pos_pipeline.labels
print("images", images,labels)
logits = unet.build_30s(images, FLAGS.num_classes, False)
predicted_images = unet.predict(logits, FLAGS.batch_size, FLAGS.image_size)
init_op = tf.group(tf.global_variables_initializer(),
tf.local_variables_initializer())
sess = tf.Session()
sess.run(init_op)
saver = tf.train.Saver()
if not tf.gfile.Exists(FLAGS.checkpoint_path + '.meta'):
raise ValueError("Can't find checkpoint file")
else:
print('[INFO ]\tFound checkpoint file, restoring model.')
saver.restore(sess, FLAGS.checkpoint_path)
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(sess=sess, coord=coord)
global_accuracy = 0.0
step = 0
try:
while not coord.should_stop():
print(predicted_images, images)
predicted_images_value, images_value = sess.run([predicted_images, images])
print (predicted_images_value, images_value)
filenames_value = []
for i in range(FLAGS.batch_size):
filenames_value.append(str(step) + "_" + str(i) + ".png")
#print (predicted_images_value[:,100:200])
maybe_save_images(predicted_images_value, images_value, filenames_value)
step += 1
except tf.errors.OutOfRangeError:
print('[INFO ]\tDone evaluating in %d steps.' % step)
finally:
# When done, ask the threads to stop.
coord.request_stop()
# Wait for threads to finish.
coord.join(threads)
sess.close()
def main(args):
setproctitle.setproctitle('quakenet_debug')
if not os.path.exists(args.output):
os.makedirs(args.output)
if args.n_clusters == None:
raise ValueError('Define the number of clusters with --n_clusters')
cfg = config.Config()
cfg.batch_size = 1
cfg.n_epochs = 1
cfg.add = 2
cfg.n_clusters = args.n_clusters
cfg.n_clusters +=1
# data pipeline
data_pipeline = dp.DataPipeline(args.dataset, cfg, False)
samples = {
'data': data_pipeline.samples,
'cluster_id': data_pipeline.labels
}
# model
model_name = args.model
model = models.get(model_name, samples,
cfg, args.checkpoint_dir, is_training=False)
with tf.Session() as sess:
coord = tf.train.Coordinator()
tf.initialize_local_variables().run()
threads = tf.train.start_queue_runners(sess=sess, coord=coord)
model.load(sess)
step = sess.run(model.global_step)
print 'Debugging at step {}'.format(step)
# summary_writer = tf.train.SummaryWriter(model.checkpoint_dir, None)
activations = tf.get_collection(tf.GraphKeys.ACTIVATIONS)
weights = tf.get_collection(tf.GraphKeys.WEIGHTS)
biases = tf.get_collection(tf.GraphKeys.BIASES)
toget = {}
toget['0_input'] = model.inputs['data']
for i, a in enumerate(activations):
name = a.name.replace('/', '_').replace(':', '_')
toget['{}_{}'.format(i+1, name)] = a
for it in range(10):
print 'running session'
fetched = sess.run(toget)
print fetched
print it
for f in fetched:
d = fetched[f]
d = np.squeeze(d, axis=0)
plt.figure()
if len(d.shape) == 2:
for i in range(d.shape[1]):
plt.plot(d[:, i])
# tot_mean = np.mean(np.mean(d,axis=1),axis=0)
# plt.plot(np.mean(d,axis=1) / tot_mean)
plt.savefig(os.path.join(args.output, '{}_{}.pdf'.format(it, f)))
plt.clf()
coord.request_stop()
coord.join(threads)
def main(args):
if args.n_clusters == None:
raise ValueError('Define the number of clusters with --n_clusters')
if not args.noise and not args.events:
raise ValueError("Define if evaluating accuracy on noise or events")
# Directory in which the evaluation summaries are written
if args.noise:
summary_dir = os.path.join(args.checkpoint_dir, "noise")
if args.events:
summary_dir = os.path.join(args.checkpoint_dir, "events")
if args.save_false:
false_start = []
false_end = []
false_origintime = []
false_dir = os.path.join("output", "false_predictions")
if not os.path.exists(false_dir):
os.makedirs(false_dir)
while True:
ckpt = tf.train.get_checkpoint_state(args.checkpoint_dir)
if args.eval_interval < 0 or ckpt:
print 'Evaluating model'
break
print 'Waiting for training job to save a checkpoint'
time.sleep(args.eval_interval)
cfg = config.Config()
if args.noise:
cfg.batch_size = 256
if args.events:
cfg.batch_size = 1
if args.save_false:
cfg.batch_size = 1
cfg.n_epochs = 1
cfg.add = 1
cfg.n_clusters = args.n_clusters
cfg.n_clusters += 1
while True:
try:
# data pipeline
data_pipeline = DataPipeline(args.dataset,
config=cfg,
is_training=False)
samples = {
'data': data_pipeline.samples,
'cluster_id': data_pipeline.labels,
"start_time": data_pipeline.start_time,
"end_time": data_pipeline.end_time
}
# set up model and validation metrics
model = models.get(args.model,
samples,
cfg,
args.checkpoint_dir,
is_training=False)
metrics = model.validation_metrics()
# Validation summary writer
summary_writer = tf.train.SummaryWriter(summary_dir, None)
with tf.Session() as sess:
coord = tf.train.Coordinator()
tf.initialize_local_variables().run()
threads = tf.train.start_queue_runners(sess=sess, coord=coord)
model.load(sess, args.step)
print 'Evaluating at step {}'.format(
sess.run(model.global_step))
step = tf.train.global_step(sess, model.global_step)
mean_metrics = {}
for key in metrics:
mean_metrics[key] = 0
n = 0
pred_labels = np.empty(1)
true_labels = np.empty(1)
while True:
try:
to_fetch = [
metrics, model.layers["class_prediction"],
samples["cluster_id"], samples["start_time"],
samples["end_time"]
]
metrics_, batch_pred_label, batch_true_label, starttime, endtime = sess.run(
to_fetch)
batch_pred_label -= 1
pred_labels = np.append(pred_labels, batch_pred_label)
true_labels = np.append(true_labels, batch_true_label)
# Save times of false preds
if args.save_false and \
batch_pred_label != batch_true_label:
print "---False prediction---"
print UTCDateTime(starttime), UTCDateTime(endtime)
false_origintime.append(
(starttime[0] + endtime[0]) / 2)
false_end.append(UTCDateTime(endtime))
false_start.append(UTCDateTime(starttime))
# print true_labels
for key in metrics:
mean_metrics[key] += cfg.batch_size * metrics_[key]
n += cfg.batch_size
mess = model.validation_metrics_message(metrics_)
print '{:03d} | '.format(n) + mess
except KeyboardInterrupt:
print 'stopping evaluation'
break
except tf.errors.OutOfRangeError:
print 'Evaluation completed ({} epochs).'.format(
cfg.n_epochs)
print "{} windows seen".format(n)
break
if n > 0:
for key in metrics:
mean_metrics[key] /= n
summary = tf.Summary(value=[
tf.Summary.Value(tag='{}/val'.format(key),
simple_value=mean_metrics[key])
])
if args.save_summary:
summary_writer.add_summary(summary,
global_step=step)
summary_writer.flush()
mess = model.validation_metrics_message(mean_metrics)
print 'Average | ' + mess
if args.eval_interval < 0:
print 'End of evaluation'
break
tf.reset_default_graph()
print 'Sleeping for {}s'.format(args.eval_interval)
time.sleep(args.eval_interval)
finally:
print 'joining data threads'
coord.request_stop()
if args.save_false:
false_preds = {}
false_preds["start_time"] = false_start
false_preds["end_time"] = false_end
false_preds["origintime"] = false_origintime
# false_preds = np.array((false_start, false_end)).transpose()[0]
# print 'shape', false_preds.shape
df = pd.DataFrame(false_preds)
df.to_csv(os.path.join(false_dir, "false_preds.csv"))
pred_labels = pred_labels[1::]
true_labels = true_labels[1::]
# np.save("output/pred_labels_noise.npy",pred_labels)
# np.save("output/true_labels_noise.npy",true_labels)
print "---Confusion Matrix----"
print confusion_matrix(true_labels, pred_labels)
coord.join(threads)
def main(args):
setproctitle.setproctitle('quakenet_predict')
ckpt = tf.train.get_checkpoint_state(args.checkpoint_dir)
cfg = config.Config()
cfg.batch_size = 1
cfg.n_clusters = args.n_clusters
cfg.add = 1
cfg.n_clusters += 1
# Remove previous output directory
if os.path.exists(args.output):
shutil.rmtree(args.output)
os.makedirs(args.output)
if args.plot:
os.makedirs(os.path.join(args.output, "viz"))
os.makedirs(os.path.join(args.output, "viz_not"))
if args.save_sac:
os.makedirs(os.path.join(args.output, "sac"))
# Load stream
#stream_path = args.stream_path
#stream_file = os.path.split(stream_path)[-1]
#print "+ Loading Stream {}".format(stream_file)
#stream = read(stream_path)
#print '+ Preprocessing stream'
#stream = preprocess_stream(stream)
# # change to use a dir list,2017/12/07
stream_path = args.stream_path
try:
stream_files = [
file for file in os.listdir(stream_path)
if fnmatch.fnmatch(file, '*.mseed')
]
except:
stream_files = os.path.split(stream_path)[-1]
# stream data with a placeholder
samples = {
'data':
tf.placeholder(tf.float32,
shape=(cfg.batch_size, cfg.win_size, 3),
name='input_data'),
'cluster_id':
tf.placeholder(tf.int64, shape=(cfg.batch_size, ), name='input_label')
}
# set up model and validation metrics
model = models.get(args.model,
samples,
cfg,
args.checkpoint_dir,
is_training=False)
with tf.Session() as sess:
model.load(sess, args.step)
print 'Predicting using model at step {}'.format(
sess.run(model.global_step))
step = tf.train.global_step(sess, model.global_step)
n_events = 0
time_start = time.time()
for stream_file in stream_files:
stream_path1 = os.path.join(stream_path, stream_file)
print " + Loading stream {}".format(stream_file)
stream = load_stream(stream_path1)
#print stream[0],stream[1].stats
print " + Preprocess stream"
stream = preprocess_stream(stream)
print " -- Stream is ready, starting detection"
# Create catalog name in which the events are stored
catalog_name = os.path.split(stream_file)[-1].split(
".mseed")[0] + ".csv"
output_catalog = os.path.join(args.output, catalog_name)
print 'Catalog created to store events', output_catalog
# Dictonary to store info on detected events
events_dic = {
"start_time": [],
"end_time": [],
"cluster_id": [],
"clusters_prob": []
}
# Windows generator
# win_gen = stream.slide(window_length=args.window_size,
# step=args.window_step,
# include_partial_windows=False)
if args.save_sac:
first_slice = stream.slice(
stream[0].stats.starttime,
stream[0].stats.starttime + args.window_size)
first_slice.write(os.path.join(
args.output, "sac", "{}_{}_{}.sac".format(
stream[0].stats.station, '0',
str(stream[0].stats.starttime).replace(':', '_'))),
format="SAC")
if args.max_windows is None:
total_time_in_sec = stream[0].stats.endtime - stream[
0].stats.starttime
max_windows = (total_time_in_sec -
args.window_size) / args.window_step
else:
max_windows = args.max_windows
try:
lists = [0]
#lists = np.arange(0,30,5)
for i in lists:
win_gen = stream.slide(window_length=args.window_size,
step=args.window_step,
offset=i,
include_partial_windows=False)
for idx, win in enumerate(win_gen):
if data_is_complete(win):
ampl_e, ampl_n, ampl_z = filter_small_ampitude(win)
if ampl_e > 0.3 or ampl_n > 0.3 or ampl_z > 0.3:
continue
ampm_e = max(abs(win[0].data))
ampm_n = max(abs(win[1].data))
ampm_z = max(abs(win[2].data))
if ampm_e < 600 and ampm_n < 600 and ampm_z < 600:
continue
# Fetch class_proba and label
to_fetch = [
samples['data'], model.layers['class_prob'],
model.layers['class_prediction']
]
# Feed window and fake cluster_id (needed by the net) but
# will be predicted
feed_dict = {
samples['data']:
fetch_window_data(win.copy().normalize()),
samples['cluster_id']:
np.array([0])
}
sample, class_prob_, cluster_id = sess.run(
to_fetch, feed_dict)
else:
continue
# # Keep only clusters proba, remove noise proba
clusters_prob = class_prob_[0, 1::]
cluster_id -= 1
# label for noise = -1, label for cluster \in {0:n_clusters}
is_event = cluster_id[0] > -1
#print cluster_id[0],is_event
probs = '{:.5f}'.format(max(list(clusters_prob)))
save_event = (float(probs) - 0.1) >= 0
#print probs,save_event
if is_event:
n_events += 1
print "event {} ,cluster id {}".format(
is_event, class_prob_)
events_dic["start_time"].append(
win[0].stats.starttime)
events_dic["end_time"].append(win[0].stats.endtime)
events_dic["cluster_id"].append(cluster_id[0])
events_dic["clusters_prob"].append(
list(clusters_prob))
if idx % 1000 == 0:
print "Analyzing {} records".format(
win[0].stats.starttime)
if args.plot:
import matplotlib
matplotlib.use('Agg')
win_filtered = win.copy()
if is_event:
# if args.plot:
# win_filtered.filter("bandpass",freqmin=4.0, freqmax=16.0)
win_filtered.plot(outfile=os.path.join(
args.output,
"viz",
####changed at 2017/11/25,use max cluster_prob instead of cluster_id
# "event_{}_cluster_{}.png".format(idx,cluster_id)))
"{}_{}_{}.png".format(
win[0].stats.station, str(probs),
str(win[0].stats.starttime).replace(
':', '_'))))
else:
win_filtered.plot(outfile=os.path.join(
args.output, "viz_not",
"{}_{}_{}.png".format(
win[0].stats.station, str(probs),
str(win[0].stats.starttime).replace(
':', '_'))))
if args.save_sac and save_event:
# win_filtered = win.copy()
save_start = win[0].stats.starttime - 12
save_end = win[0].stats.endtime + 10
win_filtered = stream.slice(save_start, save_end)
win_filtered.write(os.path.join(
args.output, "sac", "{}_{}_{}.sac".format(
win[0].stats.station, str(probs),
str(win[0].stats.starttime).replace(
':', '_'))),
format="SAC")
if idx >= max_windows:
print "stopped after {} windows".format(
max_windows)
print "found {} events".format(n_events)
break
except KeyboardInterrupt:
print 'Interrupted at time {}.'.format(win[0].stats.starttime)
print "processed {} windows, found {} events".format(
idx + 1, n_events)
print "Run time: ", time.time() - time_start
# Dump dictionary into csv file
#TODO
df = pd.DataFrame.from_dict(events_dic)
df.to_csv(output_catalog)
print "Run time: ", time.time() - time_start
def evaluate():
"""
Eval unet using specified args:
"""
#data_files, data_size = load_datafiles(FLAGS.tfrecords_prefix)
setproctitle.setproctitle('quakenet')
tf.set_random_seed(1234)
cfg = config.Config()
cfg.batch_size = 1
cfg.add = 1
cfg.n_clusters = FLAGS.num_classes
cfg.n_clusters += 1
# stream data with a placeholder
samples = {
'data':
tf.placeholder(tf.float32,
shape=(cfg.batch_size, cfg.win_size, 3),
name='input_data')
}
stream_path = FLAGS.stream_path
try:
#stream_files = [file for file in os.listdir(stream_path) if
# fnmatch.fnmatch(file, '*.mseed')]
stream_files = [
file for file in tree(stream_path)
if fnmatch.fnmatch(file, '*.mseed')
]
except:
stream_files = os.path.split(stream_path)[-1]
print("stream_files", stream_files)
#data_files, data_size = load_datafiles(stream_path)
n_events = 0
time_start = time.time()
print(" + Loading stream files {}".format(stream_files))
events_dic = {
"slice_start_time": [],
"P_pick": [],
"stname": [],
"utc_timestamp_p": [],
"utc_timestamp_s": [],
"S_pick": []
}
with tf.Session() as sess:
logits = unet.build_30s(samples['data'], FLAGS.num_classes, False)
time_start = time.time()
catalog_name = "PS_pick_blocks.csv"
output_catalog = os.path.join(FLAGS.output_dir, catalog_name)
print('Catalog created to store events', output_catalog)
init_op = tf.group(tf.global_variables_initializer(),
tf.local_variables_initializer())
sess.run(init_op)
saver = tf.train.Saver()
if not tf.gfile.Exists(FLAGS.checkpoint_path + '.meta'):
raise ValueError("Can't find checkpoint file")
else:
print('[INFO ]\tFound checkpoint file, restoring model.')
saver.restore(sess, FLAGS.checkpoint_path)
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(sess=sess, coord=coord)
for stream_file in stream_files:
#stream_path1 = os.path.join(stream_path, stream_file)
print(" + Loading stream {}".format(stream_file))
#stream = load_stream(stream_path1)
stream = load_stream(stream_file)
stream = stream.normalize()
#print stream[0],stream[1].stats
print(" + Preprocess stream")
# stream = preprocess_stream(stream)
print(" -- Stream is ready, starting detection")
try:
#lists = [0]
lists = np.arange(0, 30, 10)
for i in lists:
win_gen = stream.slide(window_length=FLAGS.window_size,
step=FLAGS.window_step,
offset=i,
include_partial_windows=False)
#print(win_gen)
for idx, win in enumerate(win_gen):
#win.resample(10)
if data_is_complete(win):
predicted_images = unet.predict(
logits, cfg.batch_size, FLAGS.image_size)
to_fetch = [predicted_images, samples['data']]
# Feed window and fake cluster_id (needed by the net) but
# will be predicted
feed_dict = {
samples['data']:
fetch_window_data(win.copy().normalize(), 3)
}
#samples_data=fetch_window_data(win.copy().normalize()
predicted_images_value, images_value = sess.run(
to_fetch, feed_dict)
clusters_p = np.where(
predicted_images_value[0, :] == 1)
clusters_s = np.where(
predicted_images_value[0, :] == 2)
p_boxes = group_consecutives(clusters_p[0])
s_boxes = group_consecutives(clusters_s[0])
tp = []
ts = []
tpstamp = []
tsstamp = []
if len(p_boxes) > 1:
for ip in range(len(p_boxes)):
#print (len(p_boxes),p_boxes,p_boxes[ip])
tpmean = float(
min(p_boxes[ip]) / 200.00 +
max(p_boxes[ip]) / 200.00)
tp.append(tpmean)
tpstamp = UTCDateTime(
win[0].stats.starttime +
tpmean).timestamp
if len(s_boxes) > 1:
for iss in range(len(s_boxes)):
tsmean = float(
min(s_boxes[iss]) / 200.00 +
max(s_boxes[iss]) / 200.00)
ts.append(tsmean)
tsstamp = UTCDateTime(
win[0].stats.starttime +
tsmean).timestamp
if len(p_boxes) > 1 or len(s_boxes) > 1:
events_dic["slice_start_time"].append(
win[0].stats.starttime)
events_dic["stname"].append(
win[0].stats.station)
events_dic["P_pick"].append(tp)
events_dic["S_pick"].append(ts)
events_dic["utc_timestamp_p"].append(tpstamp)
events_dic["utc_timestamp_s"].append(tsstamp)
#print (p_boxes,s_boxes)
win_filtered = win.copy()
lab = win_filtered[2].copy()
lab.stats.channel = "LAB"
# lab =win[0].copy()
print("predicted_images_value",
predicted_images_value.shape)
lab.data[...] = predicted_images_value[0, :]
win_filtered += lab
if FLAGS.save_sac:
output_sac = os.path.join(
FLAGS.output_dir, "sac",
"{}_{}.sac".format(
win_filtered[0].stats.station,
str(win_filtered[0].stats.starttime).
replace(':', '_')))
print(output_sac, win_filtered)
win_filtered.write(output_sac, format="SAC")
if FLAGS.plot:
win_filtered.plot(outfile=os.path.join(
FLAGS.output_dir, "viz",
"{}_{}.png".format(
win_filtered[0].stats.station,
str(win_filtered[0].stats.starttime).
replace(':', '_'))))
# Wait for threads to finish.
coord.join(threads)
except KeyboardInterrupt:
print('Interrupted at time {}.'.format(win[0].stats.starttime))
print("processed {} windows, found {} events".format(
idx + 1, n_events))
print("Run time: ", time.time() - time_start)
df = pd.DataFrame.from_dict(events_dic)
df.to_csv(output_catalog)
print("Run time: ", time.time() - time_start)
