def isect_info(self, other):
set1 = set(self.rel_fpath_list)
set2 = set(other.rel_fpath_list)
set_comparisons = ut.odict([
('s1', set1),
('s2', set2),
('union', set1.union(set2)),
('isect', set1.intersection(set2)),
('s1 - s2', set1.difference(set2)),
('s2 - s1', set1.difference(set1)),
])
stat_stats = ut.map_vals(len, set_comparisons)
print(ut.repr4(stat_stats))
return set_comparisons
if False:
idx_lookup1 = ut.make_index_lookup(self.rel_fpath_list)
idx_lookup2 = ut.make_index_lookup(other.rel_fpath_list)
uuids1 = ut.take(self.uuids,
ut.take(idx_lookup1, set_comparisons['union']))
uuids2 = ut.take(other.uuids,
ut.take(idx_lookup2, set_comparisons['union']))
uuids1 == uuids2
def get_annot_viewpoint_stats(ibs, aid_list):
annots = ibs.annots(aid_list)
viewcode2_nAnnots = ut.order_dict_by(
ut.map_vals(len, annots.group_items(annots.viewpoint_code)),
list(ibs.const.VIEW.CODE_TO_INT.keys()) + [None],
)
return viewcode2_nAnnots
def get_annot_qual_stats(ibs, aid_list):
annots = ibs.annots(aid_list)
qualtext2_nAnnots = ut.order_dict_by(
ut.map_vals(len, annots.group_items(annots.quality_texts)),
list(ibs.const.QUALITY_TEXT_TO_INT.keys()),
)
return qualtext2_nAnnots
def oracle_review(sim):
queue_params = {
'pos_diameter': None,
'neg_diameter': None,
}
infr = sim.infr
prev = infr.verbose
infr.verbose = 0
# rng = np.random.RandomState(0)
infr = sim.infr
primary_truth = sim.primary_truth
review_edges = infr.generate_reviews(**queue_params)
max_reviews = 1000
for count, (aid1, aid2) in enumerate(ut.ProgIter(review_edges)):
state = primary_truth.loc[(aid1, aid2)].idxmax()
tags = []
infr.add_feedback(aid1,
aid2,
state,
tags,
apply=True,
rectify=False,
user_id='oracle',
confidence='absolutely_sure')
if count > max_reviews:
break
infr.verbose = prev
sim.results['max_reviews'] = max_reviews
n_clusters, n_inconsistent = infr.relabel_using_reviews(rectify=False)
assert n_inconsistent == 0, 'should not create any inconsistencies'
sim.results['n_user_clusters'] = n_clusters
# infr.apply_review_inference()
curr_decisions = infr.edge_attr_df('decision')
curr_truth = primary_truth.loc[curr_decisions.index].idxmax(axis=1)
n_user_mistakes = curr_decisions != curr_truth
sim.results['n_user_mistakes'] = sum(n_user_mistakes)
gt_clusters = ut.group_pairs(infr.gen_node_attrs('orig_name_label'))
curr_clusters = ut.group_pairs(infr.gen_node_attrs('name_label'))
compare_results = compare_groups(list(gt_clusters.values()),
list(curr_clusters.values()))
sim.results.update(ut.map_vals(len, compare_results))
common_per_num = ut.group_items(compare_results['common'],
map(len, compare_results['common']))
sumafter = 3
greater = [i for i in common_per_num.keys() if i > sumafter]
common_per_num['>%s' % sumafter] = ut.flatten(
ut.take(common_per_num, greater))
ut.delete_keys(common_per_num, greater)
for k, v in common_per_num.items():
sim.results['common@' + str(k)] = len(v)
sim.results['n_names_common'] = len(compare_results['common'])
def check_baseline_results(sim):
import networkx as nx
infr = sim.infr
n_names_possible = 0
real_groups = ut.group_pairs(infr.gen_node_attrs('orig_name_label'))
possible_clusters = []
for nid, nodes in real_groups.items():
if len(nodes) == 1:
possible_clusters.append(nodes)
n_names_possible += 1
continue
cc_cand_edges = list(ut.nx_edges_between(infr.graph, nodes))
cc = ut.nx_from_node_edge(nodes, cc_cand_edges)
mst = nx.minimum_spanning_tree(cc)
ccs = list(nx.connected_components(mst))
possible_clusters.extend(ccs)
n_names_possible += (len(ccs))
sumafter = 3
best_possible_compare_results = compare_groups(
list(real_groups.values()), list(possible_clusters))
possible_per_num = ut.map_vals(
len,
ut.group_items(best_possible_compare_results['common'],
map(len, best_possible_compare_results['common'])))
greater = [i for i in possible_per_num.keys() if i > sumafter]
possible_per_num['>%s' % sumafter] = sum(
ut.take(possible_per_num, greater))
ut.delete_keys(possible_per_num, greater)
for k, v in possible_per_num.items():
sim.results['possible@' + str(k)] = v
sim.results['possible'] = len(best_possible_compare_results['common'])
# Measure the number of real names in the test (per number of annots)
real_per_num = ut.dict_hist(map(len, real_groups.values()))
greater = [i for i in real_per_num.keys() if i > sumafter]
real_per_num['>%s' % sumafter] = sum(ut.take(real_per_num, greater))
ut.delete_keys(real_per_num, greater)
for k, v in real_per_num.items():
sim.results['real@' + str(k)] = v
sim.results['n_names_possible'] = n_names_possible
sim.results['n_names_real'] = len(real_groups)
sim.results['real'] = len(real_groups)
def __init__(verif, infr):
verif.rng = np.random.RandomState(4033913)
verif.dummy_params = {
NEGTV: {
'mean': 0.2,
'std': 0.25
},
POSTV: {
'mean': 0.85,
'std': 0.2
},
INCMP: {
'mean': 0.15,
'std': 0.1
},
}
verif.score_dist = randn
verif.infr = infr
verif.orig_nodes = set(infr.aids)
verif.orig_labels = infr.get_node_attrs('orig_name_label')
verif.orig_groups = ut.invert_dict(verif.orig_labels, False)
verif.orig_groups = ut.map_vals(set, verif.orig_groups)
def isect_info(self, other):
set1 = set(self.rel_fpath_list)
set2 = set(other.rel_fpath_list)
set_comparisons = ut.odict([
('s1', set1),
('s2', set2),
('union', set1.union(set2)),
('isect', set1.intersection(set2)),
('s1 - s2', set1.difference(set2)),
('s2 - s1', set1.difference(set1)),
])
stat_stats = ut.map_vals(len, set_comparisons)
print(ut.repr4(stat_stats))
return set_comparisons
if False:
idx_lookup1 = ut.make_index_lookup(self.rel_fpath_list)
idx_lookup2 = ut.make_index_lookup(other.rel_fpath_list)
uuids1 = ut.take(self.uuids, ut.take(idx_lookup1, set_comparisons['union']))
uuids2 = ut.take(other.uuids, ut.take(idx_lookup2, set_comparisons['union']))
uuids1 == uuids2
def nbytes_info(X):
size_info = ut.map_vals(ut.get_object_nbytes, X.__dict__)
return size_info
def clean_tags():
zotero = get_libzotero()
# dict of all zotero items
# items = zotero.index
# get sql cursor
cur = zotero.cur
if False:
sorted(ut.util_sqlite.get_tablenames(cur))
ut.print_database_structure(cur)
# Debug info about tags table in sql
# The `tags` table stores all tags
# The itemTags table stores the association between items and tags
ut.get_table_columninfo_list(cur, 'fields')
# ut.get_table_columninfo_list(cur, 'relations')
ut.get_table_columninfo_list(cur, 'fieldsCombined')
ut.get_table_columninfo_list(cur, 'itemData')
ut.get_table_columninfo_list(cur, 'itemDataValues')
ut.get_table_columninfo_list(cur, 'tags')
ut.get_table_columninfo_list(cur, 'itemTags')
import pandas as pd
pd.options.display.max_colwidth = 40
pd.options.display.max_rows = 20
def pandas_sql(table, columns):
return pd.DataFrame(ut.get_table_rows(cur, table, columns),
columns=columns)
item_df = pandas_sql('items', ('itemID', 'itemTypeID', 'libraryID', 'key')).set_index('itemID', drop=False)
tags_df = pandas_sql('tags', ('tagID', 'name', 'type', 'libraryID', 'key')).set_index('tagID', drop=False)
itemData_df = pandas_sql('itemData', ('itemID', 'fieldID', 'valueID'))
itemTag_df = pandas_sql('itemTags', ('itemID', 'tagID'))
itemDataValues_df = pandas_sql('itemDataValues', ('valueID', 'value')).set_index('valueID')
field_df = pandas_sql('fields', ('fieldID', 'fieldName', 'fieldFormatID')).set_index('fieldID')
itemData_df['value'] = itemDataValues_df['value'].loc[itemData_df['valueID'].values].values
itemData_df['fieldName'] = field_df['fieldName'].loc[itemData_df['fieldID'].values].values
titles = itemData_df[itemData_df['fieldName'] == 'title']
assert len(ut.unique(ut.map_vals(len, titles.groupby('itemID').indices).values())) == 1
# itemTag_df.groupby('itemID').count()
# Find how often each tag is used
tagid_to_count = itemTag_df.groupby('tagID').count()
tagid_to_count = tagid_to_count.rename(columns={'itemID': 'nItems'})
tagid_to_count['name'] = tags_df.loc[tagid_to_count.index]['name']
tagid_to_count = tagid_to_count.sort_values('nItems')
bad_tags = tagid_to_count[tagid_to_count['nItems'] == 1]
tagid_to_count['tag_ncharsize'] = tagid_to_count['name'].apply(len)
tagid_to_count = tagid_to_count.sort_values('tag_ncharsize')
bad_tags = tagid_to_count[tagid_to_count['tag_ncharsize'] > 25]['name'].values.tolist()
def clean_tags2():
api_key = 'fBDBqRPwW9O3mYyNLiksBKZy'
base_url = 'https://api.zotero.org'
library_id = '1279414'
library_type = 'user'
from pyzotero import zotero
zot = zotero.Zotero(library_id, library_type, api_key)
for chunk in ut.ProgChunks(bad_tags, 50):
zot.delete_tags(*chunk)
if False:
api_key = 'fBDBqRPwW9O3mYyNLiksBKZy'
base_url = 'https://api.zotero.org'
user_id = '1279414'
userOrGroupPrefix = '/users/' + user_id
params = {'v': 3, 'key': api_key}
items_resp = requests.get(base_url + userOrGroupPrefix + '/items', params=params)
print(items_resp.content)
print(items_resp)
json_tags = []
get_url = base_url + userOrGroupPrefix + '/tags'
while True:
print('get_url = %r' % (get_url,))
tag_resp = requests.get(get_url, params=params)
if tag_resp.status_code != 200:
break
json_tags.extend(tag_resp.json())
if 'next' in tag_resp.links:
get_url = tag_resp.links['next']['url']
else:
break
version_to_tags = ut.ddict(list)
bad_tags = []
for tag in ut.ProgIter(json_tags, label='parsing tags'):
# x = requests.get(tag['links']['self']['href'], params=params)
if tag['meta']['numItems'] == 1:
import urllib2
try:
bad_tags.append(urllib2.quote(tag['tag']))
except Exception as ex:
print('cant encode tag=%r' % (tag,))
pass
for chunk in ut.ProgIter(ut.ichunks(bad_tags, 50), length=len(bad_tags) / 50):
search_url = base_url + userOrGroupPrefix + '/items?tag=' + ' || '.join(chunk)
r = requests.get(search_url, params=params)
matching_items = r.json()
# assert len(matching_items) == 1
for item in matching_items:
version = item['version']
version_to_tags[item['version']].append(tag['tag'])
# DELETE MULTIPLE TAGS
import requests
for chunk in ut.ichunks(bad_tags['name'], 50):
import urllib2
encoded_chunk = []
for t in chunk:
try:
encoded_chunk.append(urllib2.quote(t))
except Exception:
print(t)
suffix = ' || '.join(encoded_chunk)
delete_url = base_url + userOrGroupPrefix + '/tags?' + suffix
print('delete_url = %r' % (delete_url,))
resp = requests.delete(delete_url, params=params)
bad_tags = tagid_to_count[tagid_to_count['nItems'] == 1]
bad_tags['tagID'] = bad_tags.index
for tagid in bad_tags:
delete from itemTags where tagID in (select tagID from tags where type=1);
pass
for name in k['name'].values.tolist()
item_df['title'] = titles.set_index('itemID')['value']
for idx, item in zotero.index.items():
sql_title = item_df.loc[item.id]['title']
if item.title != sql_title:
if pd.isnull(sql_title) and item.title is not None:
print(item.__dict__)
print(item_df.loc[item.id])
print('item.title = %r' % (item.title,))
print('sql_title = %r' % (sql_title,))
assert False
duplicate_tags = [
(name, idxs) for name, idxs in tags_df.groupby('name', sort=True).indices.items() if len(idxs) > 2
]
tagname_to_tagid = tags_df.groupby('name', sort=True).first()
new_to_oldtags = {}
# Determine which tagi to use for each name
for tagname, idxs in duplicate_tags:
tags_subdf = tags_df.iloc[idxs]
mapping = itemTag_df[itemTag_df['tagID'].isin(tags_subdf['tagID'])]
tag_hist = mapping.groupby('tagID').count()
best_tagid = tag_hist['itemID'].idxmax()
new_to_oldtags[best_tagid] = set(tag_hist['itemID'].values) - {best_tagid}
tagname_to_tagid.loc[tagname] = tags_df.loc[best_tagid]
# for col in tagname_to_tagid.columns:
# tagname_to_tagid.loc[tagname][col] = tags_df.loc[best_tagid][col]
# tags_df.loc[best_tagid]
if False:
# Update tagIds
for newid, oldids in new_to_oldtags.items():
for oldid in oldids:
# cur.execute('SELECT itemID, tagID FROM itemTags WHERE tagID=?', (oldid,))
import sqlite3
try:
cmd = 'UPDATE itemTags SET tagID=? WHERE tagID=?'
args = (newid, oldid)
print('(%s) args = %r' % (cmd, args,))
cur.execute(cmd, args)
print(cur.fetchall())
except sqlite3.IntegrityError:
print('error')
pass
# tags_df.groupby('name', sort=True)
# itemTag_df.groupby('itemID')
# duptags = tags_df.iloc[tags_df.groupby('name', sort=True).indices['animals']]
# duptags['tagID']
# flags = itemTag_df['tagID'].isin(duptags['tagID'])
# dup_rel = itemTag_df[flags]
# item_df['title'].loc[dup_rel['itemID']].values
# tags_df.iloc[tags_df.groupby('name', sort=True).indices['animals']]
# tags_df[tags_df['type'] == 1]
# tags_df[tags_df['type'] == 0]
# tags_df['libraryID'].unique()
# tags_df['type'].unique()
'''
SELECT
SELECT FROM itemTags WHERE name in (animals)
'''
item_tag_pairs = ut.get_table_rows(cur, 'itemTags', ('itemID', 'tagID'))
# Group tags by item
itemid_to_tagids = ut.group_pairs(item_tag_pairs)
# Group items by tags
tagid_to_itemids = ut.group_pairs(map(tuple, map(reversed, item_tag_pairs)))
# mapping from tagid to name
tagid_to_name = dict(ut.get_table_rows(cur, 'tags', ('tagID', 'name')))
tagid_freq = list(ut.sort_dict(ut.map_vals(len, tagid_to_itemids), 'vals').items())
ut.sort_dict(ut.map_vals(sum, ut.group_pairs([(freq, tagid_to_name.get(tagid, tagid)) for tagid, freq in tagid_freq])), 'vals')
tagname_freq = ut.map_keys(lambda k: tagid_to_name.get(k, k), tagid_freq)
print('g = {!r}'.format(g))
ignore = {
'conference': ['eventtitle', 'doi', 'urldate', 'location', 'volume'],
'journal': ['doi', 'urldate', 'issue', 'number', 'volume'],
'book': ['urldate'],
'thesis': ['urldate'],
'online': ['type'],
'report': ['urldate'],
}
for v in ignore.values():
v.append('eprinttype')
v.append('eprint')
print('Entry type freq:')
print(ut.map_vals(len, entrytypes))
for e, g in entrytypes.items():
print('\n --- TYPE = %r' % (e.upper(), ))
g = g[g.columns[~np.all(pd.isnull(g), axis=0)]]
missing_cols = g.columns[np.any(pd.isnull(g), axis=0)]
if e in ignore:
missing_cols = missing_cols.difference(ignore[e])
print('missing_cols = {!r}'.format(missing_cols.tolist()))
for col in missing_cols:
print('col = {!r}'.format(col))
print(g[pd.isnull(g[col])].index.tolist())
for e, g in entrytypes.items():
print('e = %r' % (e, ))
g = g[g.columns[~np.all(pd.isnull(g), axis=0)]]