def action_compare(args): metrics = {} projects = { \ args.projects_paths[0]: None, args.projects_paths[1]: None, } ref = None inp_shape = None out_shape = None for path in projects: prj = Project(path) err = prj.load() if err is not None: log.error("error while loading project %s: %s", path, err) quit() if inp_shape is None: inp_shape = prj.model.input_shape elif inp_shape != prj.model.input_shape: log.error("model %s input shape is %s, expected %s", path, prj.model.input_shape, inp_shape) quit() if out_shape is None: out_shape = prj.model.output_shape elif out_shape != prj.model.output_shape: log.error("model %s output shape is %s, expected %s", path, prj.model.output_shape, out_shape) quit() if ref is None: ref = prj projects[path] = prj metrics[path] = None ref.prepare(args.dataset, 0.0, 0.0) log.info("evaluating %d models on %d samples ...", len(projects), len(ref.dataset.X)) for path, prj in projects.items(): # TODO: Run in parallel? log.debug("running %s ...", path) metrics[path] = prj.accuracy_for(ref.dataset.X, ref.dataset.Y, repo_as_dict=True) prev = None for path, m in metrics.items(): if prev is None: prev = m continue ref_repo, ref_cm = prev new_repo, new_cm = m diffs = {'report': [], 'cm': [], 'cm_stats': {}} table = [["Name", "Ref", "New", "Delta"]] for label, ref_run in ref_repo.items(): for name, ref_value in ref_run.items(): new_value = new_repo[label][name] if new_value != ref_value: delta = new_value - ref_value sign, fn = ('+', green) if delta >= 0 else ('', red) diffs['report'].append({ 'name': '%s / %s' % (label, name), 'delta': delta, }) table.append( [\ "%s / %s" % (label, name), "%.2f" % ref_value, "%.2f" % new_value, fn("%s%.2f" % (sign, delta))] ) print("") print(AsciiTable(table).table) heads = [""] for i in range(0, ref_cm.shape[0]): heads.append("class %d" % i) table = [heads] total = 0 impr = 0 regr = 0 for i in range(0, ref_cm.shape[0]): row = ["class %d" % i] row_diffs = [] for j in range(0, ref_cm.shape[1]): ref_v = ref_cm[i][j] new_v = new_cm[i][j] total = total + new_v delta = new_v - ref_v if ref_v != new_v: sign = '+' if delta >= 0 else '' if i == j: fn = green if delta >= 0 else red else: fn = red if delta >= 0 else green if fn == green: impr += abs(delta) else: regr += abs(delta) cell = fn("%d (%s%d)" % (new_v, sign, delta)) else: cell = "%d" % ref_v row.append(cell) row_diffs.append(delta) diffs['cm'].append(row_diffs) table.append(row) print("") print(AsciiTable(table).table) diffs['cm_stats'] = { 'improvements': { 'total': impr, 'perc': impr / float(total) * 100.0 }, 'regressions': { 'total': regr, 'perc': regr / float(total) * 100.0 } } print("") print("Improvements: %d ( %.2f %% )" % (impr, impr / float(total) * 100.0)) print("Regressions : %d ( %.2f %% )" % (regr, regr / float(total) * 100.0)) if args.to_json is not None: print("") log.info("creating %s ...", args.to_json) with open(args.to_json, 'w+') as fp: json.dump(diffs, fp, default=default)
def action_relevance(argc, argv): args = parse_args(argv) prj = Project(args.path) err = prj.load() if err is not None: log.error("error while loading project: %s", err) quit() elif not prj.is_trained(): log.error("no trained Keras model found for this project") quit() prj.prepare(args.dataset, 0.0, 0.0) X, y = prj.dataset.subsample(args.ratio) nrows, ncols = X.shape if prj.dataset.is_flat else (X[0].shape[0], len(X)) attributes = get_attributes(args.attributes, ncols) log.info("computing relevance of %d attributes on %d samples ...", ncols, nrows) start = time.time() ref_accu, ref_cm = prj.accuracy_for(X, y, repo_as_dict=True) deltas = [] tot = 0 speed = (1.0 / (time.time() - start)) * nrows for col in range(0, ncols): log.info( "[%.2f evals/s] computing relevance for attribute [%d/%d] %s ...", speed, col + 1, ncols, attributes[col]) backup = zeroize_feature(X, col, prj.dataset.is_flat) start = time.time() accu, cm = prj.accuracy_for(X, y, repo_as_dict=True) speed = (1.0 / (time.time() - start)) * nrows delta = ref_accu['weighted avg']['precision'] - accu['weighted avg'][ 'precision'] tot += delta deltas.append((col, delta)) restore_feature(X, col, backup, prj.dataset.is_flat) deltas = sorted(deltas, key=lambda x: abs(x[1]), reverse=True) rels = [] num_zero = 0 table = [("Column", "Feature", "Relevance")] for delta in deltas: col, d = delta colname = attributes[col] rel = {"attribute": colname, "index": col, "relevance": 0.0} if d != 0.0: relevance = (d / tot) * 100.0 row = ("%d" % col, attributes[col], "%.2f%%" % relevance) row = ["\033[31m%s\033[0m" % e for e in row] if relevance < 0.0 else row table.append(row) rel['relevance'] = relevance else: num_zero += 1 rels.append(rel) print("") print(AsciiTable(table).table) print("") if num_zero > 0: log.info("%d features have 0 relevance.", num_zero) if args.to_json is not None: print("") log.info("creating %s ...", args.to_json) with open(args.to_json, 'w+') as fp: json.dump(rels, fp, default=default)
def action_relevance(argc, argv): global prj, deltas, tot, start, speed, nrows, ncols, attributes args = parse_args(argv) prj = Project(args.path) err = prj.load() if err is not None: log.error("error while loading project: %s", err) quit() elif not prj.is_trained(): log.error("no trained Keras model found for this project") quit() prj.prepare(args.dataset, 0.0, 0.0) # one single worker in blocking mode = serial if args.workers == 0: args.workers = 1 X, y = prj.dataset.subsample(args.ratio) nrows, ncols = X.shape if prj.dataset.is_flat else (X[0].shape[0], len(X)) attributes = get_attributes(args.attributes, ncols) queue = TaskQueue('relevance', num_workers=args.workers, blocking=True) if args.workers == 1: log.info("computing relevance of %d attributes on %d samples using '%s' metric (slow mode) ...", ncols, nrows, args.metric) else: log.info("computing relevance of %d attributes on %d samples using '%s' metric (parallel with %d workers) ...", ncols, nrows, args.metric, queue.num_workers) start = time.time() ref_accu, ref_cm = prj.accuracy_for(X, y, repo_as_dict = True) speed = (1.0 / (time.time() - start)) * nrows for col in range(0, ncols): queue.add_task( run_inference_without, X, y, col, prj.dataset.is_flat, ref_accu['weighted avg'][args.metric], args.metric) # wait for all inferences to finish queue.join() # sort relevances by absolute value deltas = sorted(deltas, key = lambda x: abs(x[1]), reverse = True) rels = [] num_zero = 0 table = [("Column", "Feature", "Relevance")] for delta in deltas: col, d = delta colname = attributes[col] rel = { "attribute": colname, "index": col, "relevance": 0.0 } if d != 0.0: relevance = (d / tot) * 100.0 row = ("%d" % col, attributes[col], "%.2f%%" % relevance) row = ["\033[31m%s\033[0m" % e for e in row] if relevance < 0.0 else row table.append(row) rel['relevance'] = relevance else: num_zero += 1 rels.append(rel) print("") print(AsciiTable(table).table) print("") if num_zero > 0: log.info("%d features have 0 relevance.", num_zero) if args.to_json is not None: print("") log.info("creating %s ...", args.to_json) with open(args.to_json, 'w+') as fp: json.dump(rels, fp, default=default)
def action_compare(argc, argv): args = parse_args(argv) metrics = {} projects = { \ args.path_1: None, args.path_2: None, } ref = None inp_shape = None out_shape = None is_prepared = None prjs = [] for path in projects: prj = Project(path) err = prj.load() if err is not None: log.error("error while loading project %s: %s", path, err) quit() prjs.append(prj) if not is_prepared: is_prepared = True else: small_dataset = generate_reduced_dataset(args.dataset) are_equal = are_preparation_equal(prjs, small_dataset) log.info("deleting temporal file %s", small_dataset) os.remove(small_dataset) if out_shape is None: out_shape = prj.model.output_shape elif out_shape != prj.model.output_shape: log.error("model %s output shape is %s, expected %s", path, prj.model.output_shape, out_shape) quit() projects[path] = prj for prj, path in zip(prjs, projects): prj = Project(path) err = prj.load() if err is not None: log.error("error while loading project %s: %s", path, err) quit() if ref is None: prj.prepare(args.dataset, 0, 0, False) ref = prj is_prepared = True else: if are_equal: log.info("Projects use same prepare.py file ...") prj.dataset.X, prj.dataset.Y, prj.dataset.n_labels = ref.dataset.X.copy( ), ref.dataset.Y.copy(), ref.dataset.n_labels else: log.info( "Projects use different prepare.py files, reloading dataset ..." ) prj.prepare(args.dataset, 0., 0., False) # TODO: Run in parallel? log.debug("running %s ...", path) metrics[path] = prj.accuracy_for(prj.dataset.X, prj.dataset.Y, repo_as_dict=True) prev = None for path, m in metrics.items(): if prev is None: prev = m continue ref_repo, ref_cm = prev new_repo, new_cm = m diffs = {'report': [], 'cm': [], 'cm_stats': {}} table = [["Name", "Ref", "New", "Delta"]] for label, ref_run in ref_repo.items(): for name, ref_value in ref_run.items(): new_value = new_repo[label][name] if new_value != ref_value: delta = new_value - ref_value sign, fn = ('+', green) if delta >= 0 else ('', red) diffs['report'].append({ 'name': '%s / %s' % (label, name), 'delta': delta, }) table.append([ \ "%s / %s" % (label, name), "%.2f" % ref_value, "%.2f" % new_value, fn("%s%.2f" % (sign, delta))]) print("") print(AsciiTable(table).table) heads = [""] for i in range(0, ref_cm.shape[0]): heads.append("class %d" % i) table = [heads] total = 0 impr = 0 regr = 0 for i in range(0, ref_cm.shape[0]): row = ["class %d" % i] row_diffs = [] for j in range(0, ref_cm.shape[1]): ref_v = ref_cm[i][j] new_v = new_cm[i][j] total = total + new_v delta = new_v - ref_v if ref_v != new_v: sign = '+' if delta >= 0 else '' if i == j: fn = green if delta >= 0 else red else: fn = red if delta >= 0 else green if fn == green: impr += abs(delta) else: regr += abs(delta) cell = fn("%d (%s%d)" % (new_v, sign, delta)) else: cell = "%d" % ref_v row.append(cell) row_diffs.append(delta) diffs['cm'].append(row_diffs) table.append(row) print("") print(AsciiTable(table).table) diffs['cm_stats'] = { 'improvements': { 'total': impr, 'perc': impr / float(total) * 100.0 }, 'regressions': { 'total': regr, 'perc': regr / float(total) * 100.0 } } print("") print("Improvements: %d ( %.2f %% )" % (impr, impr / float(total) * 100.0)) print("Regressions : %d ( %.2f %% )" % (regr, regr / float(total) * 100.0)) if args.to_json is not None: print("") log.info("creating %s ...", args.to_json) with open(args.to_json, 'w+') as fp: json.dump(diffs, fp, default=default)
def action_compare(argc, argv): if argc < 4: usage() args = parse_args(argv[2:]) metrics = {} projects = { \ argv[0]: None, argv[1]: None, } ref = None inp_shape = None out_shape = None for path in projects: prj = Project(path) err = prj.load() if err is not None: log.error("error while loading project %s: %s", path, err) quit() if inp_shape is None: inp_shape = prj.model.input_shape elif inp_shape != prj.model.input_shape: log.error("model %s input shape is %s, expected %s", path, prj.model.input_shape, inp_shape) quit() if out_shape is None: out_shape = prj.model.output_shape elif out_shape != prj.model.output_shape: log.error("model %s output shape is %s, expected %s", path, prj.model.output_shape, out_shape) quit() if ref is None: ref = prj projects[path] = prj metrics[path] = None ref.prepare(args.dataset, 0.0, 0.0) log.info("evaluating %d models on %d samples ...", len(projects), len(ref.dataset.X)) for path, prj in projects.items(): # TODO: Run in parallel? log.debug("running %s ...", path) metrics[path] = prj.accuracy_for(ref.dataset.X, ref.dataset.Y, repo_as_dict=True) prev = None for path, m in metrics.items(): if prev is None: prev = m continue ref_repo, ref_cm = prev new_repo, new_cm = m table = [["Name", "Ref", "New", "Delta"]] for label, ref_run in ref_repo.items(): for name, ref_value in ref_run.items(): new_value = new_repo[label][name] if new_value != ref_value: delta = new_value - ref_value sign = '+' if delta >= 0 else '' table.append( [\ "%s / %s" % (label, name), "%.2f" % ref_value, "%.2f" % new_value, "%s%.2f" % (sign, delta)] ) print("") print("Report:") print(AsciiTable(table).table) heads = [""] for i in range(0, ref_cm.shape[0]): heads.append("class %d" % i) table = [heads] for i in range(0, ref_cm.shape[0]): row = ["class %d" % i] for j in range(0, ref_cm.shape[1]): ref_v = ref_cm[i][j] new_v = new_cm[i][j] if ref_v != new_v: delta = new_v - ref_v sign = '+' if delta >= 0 else '' cell = "%d (%s%d)" % (new_v, sign, delta) else: cell = "%d" % ref_v row.append(cell) table.append(row) print("") print("Confusion matrix:") print(AsciiTable(table).table)