def evaluate(segments, triggers, metrics):
"""Applies metrics to given segments and triggers.
Parameters
----------
segments : `~gwpy.segments.DataQualityFlag`
segments to be analyzed.
triggers : `~gwpy.segments.SnglInspiralTable`
triggers to be analyzed.
metrics : `Metric`, `list`
metric or list of metrics with which to evaluate segments and triggers
Returns
-------
results : `float`, `list`
result(s) produced by the metric(s).
"""
# needs type checking here
if isinstance(metrics, Metric):
results = metrics(segments, triggers)
elif isinstance(metrics, list):
results = []
for metric in metrics: results.append(metric(segments, triggers))
else:
raise Exception('Third argument must be a metric or list of metrics.')
return results
def evaluate(segments, triggers, metrics):
"""Applies metrics to given segments and triggers.
Parameters
----------
segments : `~gwpy.segments.DataQualityFlag`
segments to be analyzed.
triggers : `~gwpy.segments.SnglInspiralTable`
triggers to be analyzed.
metrics : `Metric`, `list`
metric or list of metrics with which to evaluate segments and triggers
Returns
-------
results : `float`, `list`
result(s) produced by the metric(s).
"""
# needs type checking here
if isinstance(metrics, Metric):
results = metrics(segments, triggers)
elif isinstance(metrics, list):
results = []
for metric in metrics:
results.append(metric(segments, triggers))
else:
raise Exception('Third argument must be a metric or list of metrics.')
return results
def train(model, loader, optim, loss_func, metrics, device, **kwargs):
model.train()
avg = RunningAverageMultiVar(loss=RunningAverage(), auc=RunningAverage())
progbar = tqdm(loader)
disp_step = kwargs['display_step']
viz = kwargs['viz']
epoch = kwargs['epoch']
for idx, sample in enumerate(progbar):
ref_img_batch = sample['ref'].to(device)
srch_img_batch = sample['srch'].to(device)
label_batch = sample['label'].to(device)
score_map = model(ref_img_batch, srch_img_batch)
loss = loss_func(score_map=score_map, labels=label_batch)
optim.zero_grad()
loss.backward()
optim.step()
loss_val = loss.to('cpu').item()
if idx % disp_step == 0 and viz is not None:
display_images(sample['ref'][0],
sample['srch'][0],
score_map.to('cpu')[0],
sample['label'][0],
viz=viz)
auc_val = metrics(score_map.detach().cpu().numpy(),
label_batch.detach().cpu().numpy())
avg.update(loss=loss_val, auc=auc_val)
plot_2d_line(loss_val, idx, epoch, type='train', kinds='loss', viz=viz)
plot_2d_line(auc_val, idx, epoch, type='train', kinds='auc', viz=viz)
progbar.set_postfix({
"Loss": "%0.4f" % avg['loss'](),
"AUC Score": "%0.3f" % avg['auc']()
})
torch.cuda.empty_cache()
return avg['loss'](), avg['auc']()
def __init__(self):
''' 1. setup connection to cassandra 2. intialise logger object '''
try:
self.db = dblayer()
self.metricsObj = metrics()
except Exception,e:
logging.error("chunker:__init__ failed with error %s", e)
sys.exit(1)
def main():
# data frame with our dataset
labels_A, labels_C, df_features_class_A, df_features_class_C = get_BAS_dataset(
"data/HUM/", "data/FAK/")
# preprocessing of the data to capture the features we want
#df_features_class_A = get_class_A_features(df)
#df_features_class_C = get_class_C_features(df)
# classifiers (training + prediction)
class_A_classifiers_predictions = classify(df_features_class_A, labels_A)
class_C_classifiers_predictions = classify(df_features_class_C, labels_C)
# Metrics on the result predictions from the classifiers
results_class_A_classifiers = metrics(labels_A,
class_A_classifiers_predictions)
results_class_C_classifiers = metrics(labels_C,
class_C_classifiers_predictions)
# publish and save results
print("Metrics of the Class A classifiers(profile)")
for key, value in results_class_A_classifiers.items():
print("Algorithm: " + key)
print("Accuracy: " + str(round(value[0], 3)) + " Precision: " +
str(round(value[1], 3)))
print("Recall: " + str(round(value[2], 3)) + " F-M: " +
str(round(value[3], 3)))
print("MCC: " + str(round(value[4], 3)) + " AUC: " +
str(round(value[5], 3)))
print()
#publish(results_class_A_classifiers)
print("===================== oOo ========================")
print("Metrics of the Class C classifiers(all features)")
for key, value in results_class_C_classifiers.items():
print("Algorithm: " + key)
print("Accuracy: " + str(round(value[0], 3)) + " Precision: " +
str(round(value[1], 3)))
print("Recall: " + str(round(value[2], 3)) + " F-M: " +
str(round(value[3], 3)))
print("MCC: " + str(round(value[4], 3)) + " AUC: " +
str(round(value[5], 3)))
print()
def evaluate(model, loader, loss_func, metrics, device, **kwargs):
epoch = kwargs['epoch']
model.eval()
avg = RunningAverageMultiVar(loss=RunningAverage(), auc=RunningAverage())
for idx, sample in enumerate(loader):
ref_image = sample['ref'].to(device)
srch_image = sample['srch'].to(device)
label = sample['label'].to(device)
score_map = model(ref_image, srch_image)
loss = loss_func(score_map, label)
loss_val = loss.to('cpu').item()
avg.update(loss=loss_val,
auc=metrics(score_map.detach().cpu().numpy(),
label.detach().cpu().numpy()))
plot_2d_line(avg['loss'](), idx, epoch, 'eval', 'loss')
plot_2d_line(avg['auc'](), idx, epoch, 'eval', 'auc')
torch.cuda.empty_cache()
return avg['loss'](), avg['auc']()
#!/usr/bin/python
import sys
from metrics import *
from analyzor import *
from common import *
lines = readCfg(sys.argv[1])
consoleCfg = parseConsole(lines)
ManagementPlaneAudit = metrics()
console = ManagementPlaneAudit.addMetric('consolePort')
print analyzorConsole(consoleCfg, console, lines)
def model_train(model,
input_placeholder,
output_placeholder,
train,
val,
batch_size,
num_epochs=10,
save_path="/tensorflow/dr",
summary_dir=None):
"""
input:
input_placeholder: A placeholder to feed the network with images
output_placeholder: A placeholder to feed the true labels of the input images
keep_prob: A placeholder to feed the network with probability value . useful for generalization and avoiding overfitting.
drop_out: dropout need to be added
input_image_shape: the input shape of the image, A tuple (h,w)
actual_train: the actual training data.
train: the oversampled train data. Incase of No oversampling it is the same as actual train
val: list of val images locations
labels_frame: the truth labels of the input images, index as image location.
train_step: the optimizer to back prop the error
accuracy: the accuracy of the model
batch_size: the size of the batch
num_epochs: total number of epochs
save_path: path to save the model at different checkpoints
summary_dir: path to write the various summaries written.
output:
save model at defined checkpoints, prints the val_accuracy after every epoch
process:
1) calculate the total number of iterations
2) ensemble both input_pipelines defined in reader.py
3) save all the variables
4) merge all the summaries
5) initiate the network
6) in session , pass a batch of images to the network and train the model
- calculate train_summaries and add to the summaries_dir
- After every-epoch, run the eval pipeline for one epoch and calculate the accuracy and print it.
- save the model
"""
num_epochs_per_decay = 2.0
iterations = (int(50000 / batch_size)) * num_epochs
decay_steps = int(50000 / batch_size * num_epochs_per_decay)
if tf.gfile.Exists(summary_dir):
tf.gfile.DeleteRecursively(summary_dir)
#decay_steps = tf.constant(decay_steps, name="decay_steps")
with tf.name_scope("cross_entropy"):
out = tf.nn.softmax_cross_entropy_with_logits(
labels=output_placeholder, logits=model)
with tf.name_scope("total"):
cross_entropy = tf.reduce_mean(out)
tf.summary.scalar("cross_entropy", cross_entropy)
with tf.name_scope("train"):
global_step = tf.Variable(0, name='global_step', trainable=False)
optimizer = tf.train.AdamOptimizer(0.001)
train_step = optimizer.minimize(cross_entropy, global_step=global_step)
tf.add_to_collection("train_step", train_step)
tf.summary.scalar("global_step", global_step)
#tf.summary.scalar("learning_rate", learning_rate)
#tf.summary.scalar("optimizer", optimizer)
## setting up the three pipelines
train_images, train_labels = train_input_pipeline(train,
batch_size=batch_size,
num_epochs=num_epochs)
#Pipeline for calculating training accuracy TA (training accuracy)
TA_images, TA_labels = eval_input_pipeline(train,
batch_size=batch_size,
num_epochs=num_epochs + 1)
#Pipeline for calculating Validation accuracy VA (validation accuracy)
VA_images, VA_labels = eval_input_pipeline(val, batch_size, num_epochs + 1)
saver = tf.train.Saver(tf.global_variables(), max_to_keep=20)
merged = tf.summary.merge_all()
init_op = tf.group(tf.global_variables_initializer(),
tf.local_variables_initializer())
with tf.Session() as sess:
sess.run(init_op)
training_writer = tf.summary.FileWriter(summary_dir + '/training',
sess.graph)
train_writer = tf.summary.FileWriter(summary_dir + '/train')
valid_writer = tf.summary.FileWriter(summary_dir + '/valid')
if tf.gfile.Exists(save_path):
filename = tf.train.latest_checkpoint(save_path)
print("[Restoring the last checkpoint with filename]", filename)
saver.restore(sess, filename)
#train_step = tf.get_collection("train_step")[0]
print("model restored")
g = sess.run(global_step)
print("[global step] : ", g)
print("starting from epoch:", int(batch_size * g / 50000))
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(coord=coord)
print("[queue runners started]")
try:
print("[Training has initiated]")
i, step = 0, 0
step_tb = 0
while not coord.should_stop():
images, labels = sess.run([train_images, train_labels])
summary, _ = sess.run([merged, train_step],
feed_dict={
input_placeholder: images,
output_placeholder: dummy(labels)
})
training_writer.add_summary(summary, i)
i += 1
if i == 1 or i == step * int(50000 / batch_size):
#### calculating training and testing accuracy
tb, t_p, t_t, v_p, v_t = metrics(
step_tb, merged, sess, train_writer, valid_writer,
input_placeholder, output_placeholder, val, batch_size,
model, TA_images, TA_labels, VA_images, VA_labels)
step_tb += tb
train_kappa = quadratic_weighted_kappa(t_t,
t_p,
min_rating=0,
max_rating=99)
test_kappa = quadratic_weighted_kappa(v_t,
v_p,
min_rating=0,
max_rating=99)
print("step:", i, "train_kappa:", train_kappa,
"valid_kappa:", test_kappa, "train_acc:",
accuracy_score(t_t, t_p), "valid_acc:",
accuracy_score(v_t, v_p))
if not tf.gfile.IsDirectory(save_path):
tf.gfile.MkDir(save_path)
checkpoint_path = os.path.join(save_path, 'model.ckpt')
saver.save(sess, checkpoint_path, global_step=global_step)
tf.train.write_graph(sess.graph_def, '/tmp/fmp_model',
'train_{}.pb'.format(step))
tf.train.write_graph(sess.graph_def,
'/tmp/fmp_model',
'train_{}.pbtxt'.format(step),
as_text=True)
step += 1
except tf.errors.OutOfRangeError:
print("Done training -- Maximum epoch limit has reached")
finally:
coord.request_stop()
coord.request_stop()
coord.join(threads)
L_, E_, A_, n_, nodes_array_ = load_LON(lons_foldername + "/" +
filename)
results1 = sorted([path_length(s, A, n) for s in L]) # ascending
results2 = sorted([path_length(s, A_, n_) for s in L_]) # ascending
g_min = min(results1[0], results2[0])
generate_image(L_, E_, A_, n_, images_foldername + "/" + out_file,
g_min)
generate_projection_image(
L, E, L_, E_, A, n, projection_images_foldername + "/" + out_file,
g_min)
def main_part7__projection_metrics(projections_foldername,
projection_metrics_foldername):
for filename in os.listdir(projections_foldername):
L, E, A, n, nodes_array = load_LON(projections_foldername + "/" +
filename)
out_file = filename.split('.')[0]
generate_projection_metrics(
L, E, A, n, projection_metrics_foldername + "/" + out_file)
if __name__ == '__main__':
metrics("checking_data", "checking/1_lons",
"checking/3_performance_metrics", "checking/4_network_metrics",
1000)
# metrics("data/data_bays29", "new_results/1_lons", "new_results/3_performance_metrics", "new_results/4_network_metrics", 1000)
else:
__builtin__.ipv4_mgmt_inbound = None
print stdout_header()
# configuration file reading
lines = read_cfg(args.iosconfig)
__builtin__.wholeconfig = lines
# Cisco IOS configuration file type checking
check_cfg(lines)
# Basic configuration info
__builtin__.genericCfg = addBasicInfo(lines)
# Add metrics for the Management Plane.
MgmtPlane = metrics()
# Add metrics for the Control Plane.
CtrlPlane = CPmetrics()
# Add metrics for the Data Plane.
DataPlane = DPmetrics()
# Add metric for the interfaces.
interfaces = IFSmetrics()
# Add metric for the IPv4 ACLs.
AclsV4 = ACLV4metrics()
# Add metric for the IPv6 ACLs.
AclsV6 = ACLV6metrics()
# Find interfaces (ifaceCfg).
ifaceCfg = populate_ifaces(lines,interfaces)
for i in range(0, len(ifaceCfg)):
ifaceCfg[i].get_metrics_from_config()
import sys
sys.path.append('../.')
from common.config import *
from common.Database import *
from OneSingleGaussian import *
from common.extractPerformance import *
from GaussianColor import *
from metrics import *
if __name__ == "__main__":
gt_db = Database(abs_dir_gt, start_frame=start_frame, end_frame=end_frame)
input_db = Database(abs_dir_input, start_frame=start_frame, end_frame=end_frame)
# results_db = Database(abs_dir_result, start_frame=0)
gt = gt_db.loadDB(im_color=False)
input = input_db.loadDB(im_color=True, color_space="HSV", im_show=False)
params = np.arange(0, 10, 0.2)
# matrix_mean, matrix_std, gt_test = GaussianColorRGB(input, array_alpha, im_show=False)
matrix_mean, matrix_std, gt_test = GaussianColorHSV(input, params)
gt2 = gt[(len(gt)//2):]
TP_list, FP_list, TN_list, FN_list = extractPerformance(gt2, gt_test, array_params=params)
precision_list, recall_list, fscore_list, accuracy_list = metrics(TP_list, FP_list, TN_list, FN_list, gt_test, array_params=params)
plotF1Score2D(np.linspace(0, params[-1], len(params)), fscore_list)
plotPrecisionRecall(recall_list, precision_list, label=DATABASE)
# Generate model
print("Generating model.")
print("-" * 10)
weights, loss = model(y, tX)
M, N = tX.shape
# Replace invalid values with mean of valid values and standardize
tX = modifyCSV(tX)
# Remove features due to correlation
tX = featureRemoval(tX)
tX = np.concatenate((np.ones((M, 1)), tX, np.sin(tX), np.cos(tX)), axis=1)
# Metrics
F1, accuracy = metrics(y, predict_labels(weights, tX))
print("F1-score: {}, accuracy: {}.".format(F1, accuracy))
print("-" * 10)
# Generating predictions
print("Generating predictions for {}.".format(test_path))
print("-" * 10)
_, tX_test, ids_test = load_csv_data(test_path)
M_test, N_test = tX_test.shape
# Replace invalid values with mean of valid values and standardize
tX_test = modifyCSV(tX_test)
# Remove features due to correlation
tX_test = featureRemoval(tX_test)
tX_test = np.concatenate((np.ones(
from skimage.io import imread
from skimage.segmentation import slic
from groundtruth import *
from metrics import *
# ---------------------------------
# when executed, run the main script
# ---------------------------------
if __name__ == '__main__':
img_path = 'data/Berkeley/train/'
names = os.listdir(img_path)
for name in names[:5]:
# Load the input image
img = imread(img_path + name)
print("Processing image " + name[:-4])
# Performs the segmentation
labels = slic(img, n_segments=300, compactness=10.0)
# Load the ground truth
segments = get_segment_from_filename(name[:-4])
# Evaluate metrics
m = metrics(img, labels, segments)
m.set_metrics()
m.display_metrics()
else:
__builtin__.ipv4_mgmt_inbound = None
print stdout_header()
# configuration file reading
lines = read_cfg(args.iosconfig)
__builtin__.wholeconfig = lines
# Cisco IOS configuration file type checking
check_cfg(lines)
# Basic configuration info
__builtin__.genericCfg = addBasicInfo(lines)
# Add metrics for the Management Plane.
MgmtPlane = metrics()
# Add metrics for the Control Plane.
CtrlPlane = CPmetrics()
# Add metrics for the Data Plane.
DataPlane = DPmetrics()
# Add metric for the interfaces.
interfaces = IFSmetrics()
# Add metric for the IPv4 ACLs.
AclsV4 = ACLV4metrics()
# Add metric for the IPv6 ACLs.
AclsV6 = ACLV6metrics()
# Find interfaces (ifaceCfg).
ifaceCfg = populate_ifaces(lines, interfaces)
for i in range(0, len(ifaceCfg)):
ifaceCfg[i].get_metrics_from_config()
) #BackgroundSubtractorMOG()
for id, p in enumerate(params):
bg_substracttor = cv2.bgsegm.createBackgroundSubtractorMOG() #TODO
x_mask.append(
extractBackgroundSubtrasctor_CV(bg_substractor,
data=input,
im_show=False))
sys.stdout.write("\r> Computing ... {:.2f}%".format(
(id + 1) * 100 / len(params)))
sys.stdout.flush()
print("\n")
TP_list, FP_list, TN_list, FN_list = extractPerformance(
gt, x_mask, array_params=params)
precision_list, recall_list, fscore_list, accuracy_list = metrics(
TP_list, FP_list, TN_list, FN_list, x_mask, array_params=params)
if SHOW_PLOT:
plotF1Score2D(np.linspace(0, params[-1], len(params)), fscore_list)
plotPrecisionRecall(recall_list, precision_list, label=DATABASE)
except AttributeError:
warnings.warn(
"BackgroundSubtractorMOG does not exist in your OpenCV {}".format(
cv2.__version__))
"""
BGSubstractorMOG2
"""
params = np.linspace(1, 100, 20)
x_mask = []
print("\n\nBackgroundSubtractorMOG2 Method (from OpenCV {})".format(
