def score_export(self, history=None):
"""
Generate object containing subject score data
Used in most of our plotting functions and other analysis tools
Returns
-------
swap.utils.scores.ScoreExport
ScoreExport
"""
if history is None:
history = self.history_export()
logger.info('Generating score export')
scores = {}
for subject in self.subjects:
if len(subject.ledger) == 0:
continue
id_ = subject.id
score = subject.score
scores[id_] = Score(id_, None, score)
logger.debug('done')
return ScoreExport(scores, history=history)
def test_completeness(self):
golds = [1, 1, 1, 0, 0, 0, 0, 0, 1, 1]
scores = dict([(i, Score(i, g, i / 10)) for i, g in enumerate(golds)])
se = ScoreExport(scores, False)
c = se.completeness(0.5)
assert c == 0.4
def test_counts(self):
golds = {1: 0, 2: 0, 3: 1, 4: 1, 5: -1, 6: 1}
scores = dict([(i, Score(i, g, 0)) for i, g in golds.items()])
se = ScoreExport(scores, False)
print(se.scores)
assert se.counts(0) == {-1: 1, 0: 2, 1: 3}
def test_purity(self):
golds = {1: 0, 2: 0, 3: 1, 4: 1, 5: -1, 6: 1}
scores = dict([(i, Score(i, g, 0)) for i, g in golds.items()])
se = ScoreExport(scores, False)
print(se.scores)
assert se.purity(0) == 3 / 5
def gen_scores(self, scores):
out = {}
for i, score in enumerate(scores):
g, p = score
score = Score(i, g, p)
if p < .2 or p > .8:
score.retired = p
out[i] = score
scores = out
print(scores)
print(scores[1])
scores = ScoreExport(scores, False)
return scores
def test_find_purity_2(self):
golds = [0, 1, 0, 1, 0, 1, 0, 1, 1]
scores = dict([(i, Score(i, g, i / 10)) for i, g in enumerate(golds)])
se = ScoreExport(scores, False)
p = se.find_purity(0.99)
print(p)
assert p == 0.6
def test_completeness_2(self):
scores = [(1, .1), (1, .1), (1, .1), (0, .4), (0, .5), (0, .5),
(0, .5), (0, .6), (1, .9), (1, .9)]
scores = dict([(i, Score(i, gp[0], gp[1]))
for i, gp in enumerate(scores)])
se = ScoreExport(scores, False)
assert se.stats.completeness == 0.4
def test_roc_notgold(self):
golds = {0: -1, 1: 0, 2: 1}
scores = dict([(i, Score(i, g, 0)) for i, g in golds.items()])
se = ScoreExport(scores, False)
print(list(se.roc()))
assert len(list(se.roc())) == 2
def test_builtins(self):
golds = {1: 0, 2: 0, 3: 1, 4: 1, 5: -1, 6: 1}
scores = dict([(i, Score(i, g, 0)) for i, g in golds.items()])
se = ScoreExport(scores, False)
print(se.scores)
assert len(se) == 6
iter(se)
def test_composition(self):
golds = {1: 0, 2: 0, 3: 1, 4: 1, 5: -1, 6: 1}
scores = dict([(i, Score(i, g, 0)) for i, g in golds.items()])
se = ScoreExport(scores, False)
print(se.scores)
assert se.composition(0) == {-1: 1 / 5, 0: 2 / 5, 1: 3 / 5}
assert False
def test_assign_golds(self, mock):
golds = [1, 1, 0, 0, 1, 1]
rg = dict([(i, g) for i, g in enumerate(golds)])
mock.return_value = rg
scores = dict([(i, Score(i, None, i / 10)) for i in range(len(golds))])
se = ScoreExport(scores, True)
for score in se.scores.values():
assert golds[score.id] == score.gold
def process(self):
"""
Process all subjects
"""
cursor = self.get_cursor()
for item in cursor:
score = item['votes'] / item['total']
gold = item['gold']
id_ = item['_id']
self.data[id_] = Score(id_, gold, score)
def test_roc(self):
golds = {1: 0, 2: 0, 3: 1, 4: 1, 5: 0, 6: 1}
scores = dict([(i, Score(i, g, 0)) for i, g in golds.items()])
se = ScoreExport(scores, False)
print(list(se.roc()))
for i, item in enumerate(se.roc()):
g, p = item
assert g == golds[i + 1]
assert p == 0
def test_init(self):
golds = {1: 0, 2: 0, 3: 1, 4: 1}
scores = {}
for i, g in golds.items():
scores[i] = Score(i, g, i / 10)
se = ScoreExport(scores, False)
print(se.scores)
assert len(se.scores) == 4
for score in se.scores.values():
assert score.gold == golds[score.id]
assert score.p == score.id / 10
def test_(self):
golds = {1: 0, 2: 0, 3: 1, 4: 1, 5: -1, 6: 1}
scores = dict([(i, Score(i, g, 0)) for i, g in golds.items()])
si = ScoreIterator(scores, lambda score: score)
for score in si:
i = score.id
id_ = scores[i].id
gold = scores[i].gold
p = scores[i].p
assert score.id == id_
assert score.gold == gold
assert score.p == p
def generate_scores(cls):
user = config.online_swap._auth_username
key = CaesarConfig.registered_key()
r = Requests.generate_scores(user, key)
data = json.loads(r.text)
logger.debug('Parsed scores for %d subjects', len(data))
scores = {}
for item in data['scores'].values():
item['id_'] = item.pop('id')
score = Score(**item)
scores[score.id] = score
thresholds = tuple(data['thresholds'])
return ScoreExport(scores, new_golds=False, thresholds=thresholds)
def score_export(self, thresholds=None, all_golds=False):
scores = {}
if thresholds is None:
retire = lambda h: h.last()
else:
retire = lambda h: h.retire(thresholds)
for history in self.history.values():
id_ = history.id
n, p = retire(history)
score = Score(id_, history.gold, p, ncl=n)
scores[id_] = score
return ScoreExport(scores,
gold_getter=self.gold_getter,
thresholds=thresholds,
new_golds=all_golds)
def scores_from_db(scores):
scores = [(id_, Score(id_, gold, p)) for id_, gold, p in scores]
scores = ScoreExport(dict(scores), new_golds=False)
return scores
