def test_completeness(self):
golds = [1, 1, 1, 0, 0, 0, 0, 0, 1, 1]
scores = dict([(i, Score(i, g, i / 10)) for i, g in enumerate(golds)])
se = ScoreExport(scores, False)
c = se.completeness(0.5)
assert c == 0.4
def test_find_purity_2(self):
golds = [0, 1, 0, 1, 0, 1, 0, 1, 1]
scores = dict([(i, Score(i, g, i / 10)) for i, g in enumerate(golds)])
se = ScoreExport(scores, False)
p = se.find_purity(0.99)
print(p)
assert p == 0.6
def test_roc_notgold(self):
golds = {0: -1, 1: 0, 2: 1}
scores = dict([(i, Score(i, g, 0)) for i, g in golds.items()])
se = ScoreExport(scores, False)
print(list(se.roc()))
assert len(list(se.roc())) == 2
def test_counts(self):
golds = {1: 0, 2: 0, 3: 1, 4: 1, 5: -1, 6: 1}
scores = dict([(i, Score(i, g, 0)) for i, g in golds.items()])
se = ScoreExport(scores, False)
print(se.scores)
assert se.counts(0) == {-1: 1, 0: 2, 1: 3}
def test_purity(self):
golds = {1: 0, 2: 0, 3: 1, 4: 1, 5: -1, 6: 1}
scores = dict([(i, Score(i, g, 0)) for i, g in golds.items()])
se = ScoreExport(scores, False)
print(se.scores)
assert se.purity(0) == 3 / 5
def test_composition(self):
golds = {1: 0, 2: 0, 3: 1, 4: 1, 5: -1, 6: 1}
scores = dict([(i, Score(i, g, 0)) for i, g in golds.items()])
se = ScoreExport(scores, False)
print(se.scores)
assert se.composition(0) == {-1: 1 / 5, 0: 2 / 5, 1: 3 / 5}
assert False
def test_completeness_2(self):
scores = [(1, .1), (1, .1), (1, .1), (0, .4), (0, .5), (0, .5),
(0, .5), (0, .6), (1, .9), (1, .9)]
scores = dict([(i, Score(i, gp[0], gp[1]))
for i, gp in enumerate(scores)])
se = ScoreExport(scores, False)
c = se.completeness(0.5)
assert c == 0.4
def test_roc(self):
golds = {1: 0, 2: 0, 3: 1, 4: 1, 5: 0, 6: 1}
scores = dict([(i, Score(i, g, 0)) for i, g in golds.items()])
se = ScoreExport(scores, False)
print(list(se.roc()))
for i, item in enumerate(se.roc()):
g, p = item
assert g == golds[i + 1]
assert p == 0
def load(self, fname):
_, extension = os.path.splitext(fname)
if extension == '.pkl':
return super().load(fname)
elif extension == '.csv':
return ScoreExport.from_csv(fname)
def score_export(self, history=None):
"""
Generate object containing subject score data
Used in most of our plotting functions and other analysis tools
Returns
-------
swap.utils.scores.ScoreExport
ScoreExport
"""
if history is None:
history = self.history_export()
logger.info('Generating score export')
scores = {}
for subject in self.subjects:
if len(subject.ledger) == 0:
continue
id_ = subject.id
score = subject.score
scores[id_] = Score(id_, None, score)
logger.debug('done')
return ScoreExport(scores, history=history)
def test_builtins(self):
golds = {1: 0, 2: 0, 3: 1, 4: 1, 5: -1, 6: 1}
scores = dict([(i, Score(i, g, 0)) for i, g in golds.items()])
se = ScoreExport(scores, False)
print(se.scores)
assert len(se) == 6
iter(se)
def test_assign_golds(self, mock):
golds = [1, 1, 0, 0, 1, 1]
rg = dict([(i, g) for i, g in enumerate(golds)])
mock.return_value = rg
scores = dict([(i, Score(i, None, i / 10)) for i in range(len(golds))])
se = ScoreExport(scores, True)
for score in se.scores.values():
assert golds[score.id] == score.gold
def score_export(self):
"""
Generate object containing subject score data
Used in most of our plotting functions and other analysis tools
Returns
-------
swap.utils.scores.ScoreExport
ScoreExport
"""
return ScoreExport(self.data, new_golds=False)
def test_init(self):
golds = {1: 0, 2: 0, 3: 1, 4: 1}
scores = {}
for i, g in golds.items():
scores[i] = Score(i, g, i / 10)
se = ScoreExport(scores, False)
print(se.scores)
assert len(se.scores) == 4
for score in se.scores.values():
assert score.gold == golds[score.id]
assert score.p == score.id / 10
def next(self):
self.__bounds()
if self.i > len(self.items):
raise StopIteration()
label, fname, load = self.items[self.i]
_, extension = os.path.splitext(fname)
logger.info(label, fname, load)
self.i += 1
if extension == '.csv':
return (label, ScoreExport.from_csv(fname).roc())
obj = load(fname)
return (label, self._get_export(obj))
def gen_scores(self, scores):
out = {}
for i, score in enumerate(scores):
g, p = score
score = Score(i, g, p)
if p < .2 or p > .8:
score.retired = p
out[i] = score
scores = out
print(scores)
print(scores[1])
scores = ScoreExport(scores, False)
return scores
def score_export(self, thresholds=None, all_golds=False):
scores = {}
if thresholds is None:
retire = lambda h: h.last()
else:
retire = lambda h: h.retire(thresholds)
for history in self.history.values():
id_ = history.id
n, p = retire(history)
score = Score(id_, history.gold, p, ncl=n)
scores[id_] = score
return ScoreExport(scores,
gold_getter=self.gold_getter,
thresholds=thresholds,
new_golds=all_golds)
def generate_scores(cls):
user = config.online_swap._auth_username
key = CaesarConfig.registered_key()
r = Requests.generate_scores(user, key)
data = json.loads(r.text)
logger.debug('Parsed scores for %d subjects', len(data))
scores = {}
for item in data['scores'].values():
item['id_'] = item.pop('id')
score = Score(**item)
scores[score.id] = score
thresholds = tuple(data['thresholds'])
return ScoreExport(scores, new_golds=False, thresholds=thresholds)
def difference(self, args):
base = load_pickle(args.diff[0])
p_args = []
args_ = args.diff[1:-1]
args_ = [tuple(args_[i:i + 2]) for i in range(0, len(args_), 2)]
for label, fname in args_:
_, extension = os.path.splitext(fname)
if extension == '.csv':
scores = ScoreExport.from_csv(fname)
elif extension == '.pkl':
scores = load_pickle(fname)
p_args.append((label, scores))
fname = self.f(args.diff[-1])
print(p_args)
# other = [load_pickle(x) for x in args.diff[1:]]
plots.performance.p_diff(base, p_args, fname)
def scores_from_db(scores):
scores = [(id_, Score(id_, gold, p)) for id_, gold, p in scores]
scores = ScoreExport(dict(scores), new_golds=False)
return scores
def call(self, args):
swap = None
score_export = None
if args.load:
obj = self.load(args.load[0])
if isinstance(obj, SWAP):
swap = obj
score_export = swap.score_export()
elif isinstance(obj, ScoreExport):
score_export = obj
if args.scores_from_csv:
fname = args.scores_from_csv[0]
score_export = ScoreExport.from_csv(fname)
if args.run:
swap = self.run_swap(args)
score_export = swap.score_export()
if swap is not None:
if args.save:
manifest = self.manifest(swap, args)
self.save(swap, self.f(args.save[0]), manifest)
if args.subject:
fname = self.f(args.subject[0])
plots.traces.plot_subjects(swap.history_export(), fname)
if args.user:
fname = self.f(args.user[0])
plots.plot_user_cm(swap, fname)
# if args.utraces:
# fname = self.f(args.user[0])
# plots.traces.plot_user(swap, fname)
if args.log:
fname = self.f(args.log[0])
write_log(swap, fname)
if args.stats:
s = swap.stats_str()
print(s)
logger.debug(s)
if args.test:
from swap.utils.golds import GoldGetter
gg = GoldGetter()
logger.debug('applying new gold labels')
swap.set_gold_labels(gg.golds)
swap.process_changes()
logger.debug('done')
if args.test_reorder:
self.reorder_classifications(swap)
if args.export_user_scores:
fname = self.f(args.export_user_scores[0])
self.export_user_scores(swap, fname)
if score_export is not None:
if args.save_scores:
fname = self.f(args.save_scores[0])
self.save(score_export, fname)
if args.hist:
fname = self.f(args.hist[0])
plots.plot_class_histogram(fname, score_export)
if args.dist:
data = [s.getScore() for s in swap.subjects]
plots.plot_pdf(data,
self.f(args.dist[0]),
swap,
cutoff=float(args.dist[1]))
if args.presrec:
fname = self.f(args.presrec[0])
plots.distributions.sklearn_purity_completeness(
fname, score_export)
if args.scores_to_csv:
self.scores_to_csv(score_export, args.scores_to_csv[0])
if args.diff:
self.difference(args)
if args.shell:
import code
code.interact(local=locals())
return swap
def call(self, args):
swap = None
scores = None
if args.load:
obj = self.load(args.load[0])
if isinstance(obj, SWAP):
swap = obj
scores = swap.score_export()
elif isinstance(obj, ScoreExport):
scores = obj
if args.scores_from_csv:
fname = args.scores_from_csv[0]
scores = ScoreExport.from_csv(fname)
if args.run:
swap = self.run_swap(args)
scores = swap.score_export()
if swap is not None:
if args.save:
manifest = self.manifest(swap, args)
self.save(swap, self.f(args.save[0]), manifest)
if args.log:
fname = self.f(args.log[0])
write_log(swap, fname)
if args.stats:
s = swap.stats_str()
print(s)
logger.debug(s)
if args.test:
from swap.utils.golds import GoldGetter
gg = GoldGetter()
logger.debug('applying new gold labels')
swap.set_gold_labels(gg.golds)
swap.process_changes()
logger.debug('done')
if args.test_reorder:
self.reorder_classifications(swap)
if args.export_user_scores:
fname = self.f(args.export_user_scores[0])
self.export_user_scores(swap, fname)
if scores is not None:
if args.save_scores:
DB().subjects.save_scores(scores)
if args.scores_to_csv:
self.scores_to_csv(scores, args.scores_to_csv[0])
self.plot(args, swap, scores)
if args.shell:
import code
code.interact(local=locals())
return swap
