def action_compare(args):
metrics = {}
projects = { \
args.projects_paths[0]: None,
args.projects_paths[1]: None,
}
ref = None
inp_shape = None
out_shape = None
for path in projects:
prj = Project(path)
err = prj.load()
if err is not None:
log.error("error while loading project %s: %s", path, err)
quit()
if inp_shape is None:
inp_shape = prj.model.input_shape
elif inp_shape != prj.model.input_shape:
log.error("model %s input shape is %s, expected %s", path,
prj.model.input_shape, inp_shape)
quit()
if out_shape is None:
out_shape = prj.model.output_shape
elif out_shape != prj.model.output_shape:
log.error("model %s output shape is %s, expected %s", path,
prj.model.output_shape, out_shape)
quit()
if ref is None:
ref = prj
projects[path] = prj
metrics[path] = None
ref.prepare(args.dataset, 0.0, 0.0)
log.info("evaluating %d models on %d samples ...", len(projects),
len(ref.dataset.X))
for path, prj in projects.items():
# TODO: Run in parallel?
log.debug("running %s ...", path)
metrics[path] = prj.accuracy_for(ref.dataset.X,
ref.dataset.Y,
repo_as_dict=True)
prev = None
for path, m in metrics.items():
if prev is None:
prev = m
continue
ref_repo, ref_cm = prev
new_repo, new_cm = m
diffs = {'report': [], 'cm': [], 'cm_stats': {}}
table = [["Name", "Ref", "New", "Delta"]]
for label, ref_run in ref_repo.items():
for name, ref_value in ref_run.items():
new_value = new_repo[label][name]
if new_value != ref_value:
delta = new_value - ref_value
sign, fn = ('+', green) if delta >= 0 else ('', red)
diffs['report'].append({
'name': '%s / %s' % (label, name),
'delta': delta,
})
table.append( [\
"%s / %s" % (label, name),
"%.2f" % ref_value,
"%.2f" % new_value,
fn("%s%.2f" % (sign, delta))] )
print("")
print(AsciiTable(table).table)
heads = [""]
for i in range(0, ref_cm.shape[0]):
heads.append("class %d" % i)
table = [heads]
total = 0
impr = 0
regr = 0
for i in range(0, ref_cm.shape[0]):
row = ["class %d" % i]
row_diffs = []
for j in range(0, ref_cm.shape[1]):
ref_v = ref_cm[i][j]
new_v = new_cm[i][j]
total = total + new_v
delta = new_v - ref_v
if ref_v != new_v:
sign = '+' if delta >= 0 else ''
if i == j:
fn = green if delta >= 0 else red
else:
fn = red if delta >= 0 else green
if fn == green:
impr += abs(delta)
else:
regr += abs(delta)
cell = fn("%d (%s%d)" % (new_v, sign, delta))
else:
cell = "%d" % ref_v
row.append(cell)
row_diffs.append(delta)
diffs['cm'].append(row_diffs)
table.append(row)
print("")
print(AsciiTable(table).table)
diffs['cm_stats'] = {
'improvements': {
'total': impr,
'perc': impr / float(total) * 100.0
},
'regressions': {
'total': regr,
'perc': regr / float(total) * 100.0
}
}
print("")
print("Improvements: %d ( %.2f %% )" %
(impr, impr / float(total) * 100.0))
print("Regressions : %d ( %.2f %% )" %
(regr, regr / float(total) * 100.0))
if args.to_json is not None:
print("")
log.info("creating %s ...", args.to_json)
with open(args.to_json, 'w+') as fp:
json.dump(diffs, fp, default=default)
def action_relevance(argc, argv):
args = parse_args(argv)
prj = Project(args.path)
err = prj.load()
if err is not None:
log.error("error while loading project: %s", err)
quit()
elif not prj.is_trained():
log.error("no trained Keras model found for this project")
quit()
prj.prepare(args.dataset, 0.0, 0.0)
X, y = prj.dataset.subsample(args.ratio)
nrows, ncols = X.shape if prj.dataset.is_flat else (X[0].shape[0], len(X))
attributes = get_attributes(args.attributes, ncols)
log.info("computing relevance of %d attributes on %d samples ...", ncols,
nrows)
start = time.time()
ref_accu, ref_cm = prj.accuracy_for(X, y, repo_as_dict=True)
deltas = []
tot = 0
speed = (1.0 / (time.time() - start)) * nrows
for col in range(0, ncols):
log.info(
"[%.2f evals/s] computing relevance for attribute [%d/%d] %s ...",
speed, col + 1, ncols, attributes[col])
backup = zeroize_feature(X, col, prj.dataset.is_flat)
start = time.time()
accu, cm = prj.accuracy_for(X, y, repo_as_dict=True)
speed = (1.0 / (time.time() - start)) * nrows
delta = ref_accu['weighted avg']['precision'] - accu['weighted avg'][
'precision']
tot += delta
deltas.append((col, delta))
restore_feature(X, col, backup, prj.dataset.is_flat)
deltas = sorted(deltas, key=lambda x: abs(x[1]), reverse=True)
rels = []
num_zero = 0
table = [("Column", "Feature", "Relevance")]
for delta in deltas:
col, d = delta
colname = attributes[col]
rel = {"attribute": colname, "index": col, "relevance": 0.0}
if d != 0.0:
relevance = (d / tot) * 100.0
row = ("%d" % col, attributes[col], "%.2f%%" % relevance)
row = ["\033[31m%s\033[0m" % e
for e in row] if relevance < 0.0 else row
table.append(row)
rel['relevance'] = relevance
else:
num_zero += 1
rels.append(rel)
print("")
print(AsciiTable(table).table)
print("")
if num_zero > 0:
log.info("%d features have 0 relevance.", num_zero)
if args.to_json is not None:
print("")
log.info("creating %s ...", args.to_json)
with open(args.to_json, 'w+') as fp:
json.dump(rels, fp, default=default)
def action_relevance(argc, argv):
global prj, deltas, tot, start, speed, nrows, ncols, attributes
args = parse_args(argv)
prj = Project(args.path)
err = prj.load()
if err is not None:
log.error("error while loading project: %s", err)
quit()
elif not prj.is_trained():
log.error("no trained Keras model found for this project")
quit()
prj.prepare(args.dataset, 0.0, 0.0)
# one single worker in blocking mode = serial
if args.workers == 0:
args.workers = 1
X, y = prj.dataset.subsample(args.ratio)
nrows, ncols = X.shape if prj.dataset.is_flat else (X[0].shape[0], len(X))
attributes = get_attributes(args.attributes, ncols)
queue = TaskQueue('relevance', num_workers=args.workers, blocking=True)
if args.workers == 1:
log.info("computing relevance of %d attributes on %d samples using '%s' metric (slow mode) ...", ncols, nrows, args.metric)
else:
log.info("computing relevance of %d attributes on %d samples using '%s' metric (parallel with %d workers) ...", ncols, nrows, args.metric, queue.num_workers)
start = time.time()
ref_accu, ref_cm = prj.accuracy_for(X, y, repo_as_dict = True)
speed = (1.0 / (time.time() - start)) * nrows
for col in range(0, ncols):
queue.add_task( run_inference_without,
X, y, col, prj.dataset.is_flat,
ref_accu['weighted avg'][args.metric],
args.metric)
# wait for all inferences to finish
queue.join()
# sort relevances by absolute value
deltas = sorted(deltas, key = lambda x: abs(x[1]), reverse = True)
rels = []
num_zero = 0
table = [("Column", "Feature", "Relevance")]
for delta in deltas:
col, d = delta
colname = attributes[col]
rel = {
"attribute": colname,
"index": col,
"relevance": 0.0
}
if d != 0.0:
relevance = (d / tot) * 100.0
row = ("%d" % col, attributes[col], "%.2f%%" % relevance)
row = ["\033[31m%s\033[0m" % e for e in row] if relevance < 0.0 else row
table.append(row)
rel['relevance'] = relevance
else:
num_zero += 1
rels.append(rel)
print("")
print(AsciiTable(table).table)
print("")
if num_zero > 0:
log.info("%d features have 0 relevance.", num_zero)
if args.to_json is not None:
print("")
log.info("creating %s ...", args.to_json)
with open(args.to_json, 'w+') as fp:
json.dump(rels, fp, default=default)
def action_compare(argc, argv):
args = parse_args(argv)
metrics = {}
projects = { \
args.path_1: None,
args.path_2: None,
}
ref = None
inp_shape = None
out_shape = None
is_prepared = None
prjs = []
for path in projects:
prj = Project(path)
err = prj.load()
if err is not None:
log.error("error while loading project %s: %s", path, err)
quit()
prjs.append(prj)
if not is_prepared:
is_prepared = True
else:
small_dataset = generate_reduced_dataset(args.dataset)
are_equal = are_preparation_equal(prjs, small_dataset)
log.info("deleting temporal file %s", small_dataset)
os.remove(small_dataset)
if out_shape is None:
out_shape = prj.model.output_shape
elif out_shape != prj.model.output_shape:
log.error("model %s output shape is %s, expected %s", path,
prj.model.output_shape, out_shape)
quit()
projects[path] = prj
for prj, path in zip(prjs, projects):
prj = Project(path)
err = prj.load()
if err is not None:
log.error("error while loading project %s: %s", path, err)
quit()
if ref is None:
prj.prepare(args.dataset, 0, 0, False)
ref = prj
is_prepared = True
else:
if are_equal:
log.info("Projects use same prepare.py file ...")
prj.dataset.X, prj.dataset.Y, prj.dataset.n_labels = ref.dataset.X.copy(
), ref.dataset.Y.copy(), ref.dataset.n_labels
else:
log.info(
"Projects use different prepare.py files, reloading dataset ..."
)
prj.prepare(args.dataset, 0., 0., False)
# TODO: Run in parallel?
log.debug("running %s ...", path)
metrics[path] = prj.accuracy_for(prj.dataset.X,
prj.dataset.Y,
repo_as_dict=True)
prev = None
for path, m in metrics.items():
if prev is None:
prev = m
continue
ref_repo, ref_cm = prev
new_repo, new_cm = m
diffs = {'report': [], 'cm': [], 'cm_stats': {}}
table = [["Name", "Ref", "New", "Delta"]]
for label, ref_run in ref_repo.items():
for name, ref_value in ref_run.items():
new_value = new_repo[label][name]
if new_value != ref_value:
delta = new_value - ref_value
sign, fn = ('+', green) if delta >= 0 else ('', red)
diffs['report'].append({
'name': '%s / %s' % (label, name),
'delta': delta,
})
table.append([ \
"%s / %s" % (label, name),
"%.2f" % ref_value,
"%.2f" % new_value,
fn("%s%.2f" % (sign, delta))])
print("")
print(AsciiTable(table).table)
heads = [""]
for i in range(0, ref_cm.shape[0]):
heads.append("class %d" % i)
table = [heads]
total = 0
impr = 0
regr = 0
for i in range(0, ref_cm.shape[0]):
row = ["class %d" % i]
row_diffs = []
for j in range(0, ref_cm.shape[1]):
ref_v = ref_cm[i][j]
new_v = new_cm[i][j]
total = total + new_v
delta = new_v - ref_v
if ref_v != new_v:
sign = '+' if delta >= 0 else ''
if i == j:
fn = green if delta >= 0 else red
else:
fn = red if delta >= 0 else green
if fn == green:
impr += abs(delta)
else:
regr += abs(delta)
cell = fn("%d (%s%d)" % (new_v, sign, delta))
else:
cell = "%d" % ref_v
row.append(cell)
row_diffs.append(delta)
diffs['cm'].append(row_diffs)
table.append(row)
print("")
print(AsciiTable(table).table)
diffs['cm_stats'] = {
'improvements': {
'total': impr,
'perc': impr / float(total) * 100.0
},
'regressions': {
'total': regr,
'perc': regr / float(total) * 100.0
}
}
print("")
print("Improvements: %d ( %.2f %% )" %
(impr, impr / float(total) * 100.0))
print("Regressions : %d ( %.2f %% )" %
(regr, regr / float(total) * 100.0))
if args.to_json is not None:
print("")
log.info("creating %s ...", args.to_json)
with open(args.to_json, 'w+') as fp:
json.dump(diffs, fp, default=default)
def action_compare(argc, argv):
if argc < 4:
usage()
args = parse_args(argv[2:])
metrics = {}
projects = { \
argv[0]: None,
argv[1]: None,
}
ref = None
inp_shape = None
out_shape = None
for path in projects:
prj = Project(path)
err = prj.load()
if err is not None:
log.error("error while loading project %s: %s", path, err)
quit()
if inp_shape is None:
inp_shape = prj.model.input_shape
elif inp_shape != prj.model.input_shape:
log.error("model %s input shape is %s, expected %s", path,
prj.model.input_shape, inp_shape)
quit()
if out_shape is None:
out_shape = prj.model.output_shape
elif out_shape != prj.model.output_shape:
log.error("model %s output shape is %s, expected %s", path,
prj.model.output_shape, out_shape)
quit()
if ref is None:
ref = prj
projects[path] = prj
metrics[path] = None
ref.prepare(args.dataset, 0.0, 0.0)
log.info("evaluating %d models on %d samples ...", len(projects),
len(ref.dataset.X))
for path, prj in projects.items():
# TODO: Run in parallel?
log.debug("running %s ...", path)
metrics[path] = prj.accuracy_for(ref.dataset.X,
ref.dataset.Y,
repo_as_dict=True)
prev = None
for path, m in metrics.items():
if prev is None:
prev = m
continue
ref_repo, ref_cm = prev
new_repo, new_cm = m
table = [["Name", "Ref", "New", "Delta"]]
for label, ref_run in ref_repo.items():
for name, ref_value in ref_run.items():
new_value = new_repo[label][name]
if new_value != ref_value:
delta = new_value - ref_value
sign = '+' if delta >= 0 else ''
table.append( [\
"%s / %s" % (label, name),
"%.2f" % ref_value,
"%.2f" % new_value,
"%s%.2f" % (sign, delta)] )
print("")
print("Report:")
print(AsciiTable(table).table)
heads = [""]
for i in range(0, ref_cm.shape[0]):
heads.append("class %d" % i)
table = [heads]
for i in range(0, ref_cm.shape[0]):
row = ["class %d" % i]
for j in range(0, ref_cm.shape[1]):
ref_v = ref_cm[i][j]
new_v = new_cm[i][j]
if ref_v != new_v:
delta = new_v - ref_v
sign = '+' if delta >= 0 else ''
cell = "%d (%s%d)" % (new_v, sign, delta)
else:
cell = "%d" % ref_v
row.append(cell)
table.append(row)
print("")
print("Confusion matrix:")
print(AsciiTable(table).table)