def populate_annos():
from protcur import document as ptcdoc
from protcur.core import annoSync
from protcur.analysis import Hybrid, protc
from hyputils.hypothesis import HypothesisHelper, group_to_memfile
from hyputils import hypothesis as hyp
ProtcurData.protc = protc
group = auth.get('hypothesis-group')
get_annos, annos, stream_thread, exit_loop = annoSync(
group_to_memfile(group + 'sparcur'),
helpers=(HypothesisHelper, Hybrid, protc),
group=group,
sync=False)
# FIXME hack to workaround bad api key init for hyutils until we can integrate orthauth
get_annos.api_token = auth.get('hypothesis-api-key')
annos.clear()
annos.extend([ptcdoc.Annotation(a) for a in get_annos()])
# reset classes in case some other class has populated them
# (e.g. during testing) FIXME this is a bad hack
protc.reset()
Hybrid.reset()
# FIXME this is expensive and slow to continually recompute
[protc(a, annos) for a in annos]
[Hybrid(a, annos) for a in annos]
def setUpClass(cls):
cls.get_annos = Memoizer(group_to_memfile(group), api_token, username,
group)
world_get = Memoizer(bad_memfile, api_token, username, '__world__')
world_annos = world_get.get_annos_from_api(max_results=10)
world_get.memoize_annos(world_annos)
def populate_annos(group_name='sparc-curation'):
from hyputils import hypothesis as hyp
if hyp.api_token == 'TOKEN': # FIXME does not work
hyp.api_token = auth.get('hypothesis-api-key')
group = auth.get('hypothesis-group')
get_annos, annos, stream_thread, exit_loop = annoSync(group_to_memfile(group + 'sparcur'),
helpers=(HypothesisHelper, Hybrid, protc),
group=group,
sync=False)
[protc(a, annos) for a in annos]
[Hybrid(a, annos) for a in annos]
def populate_annos(group_name='sparc-curation'):
from hyputils import hypothesis as hyp
if hyp.api_token == 'TOKEN': # FIXME does not work
hyp.api_token = devconfig.secrets('hypothesis', 'api',
devconfig.hypothesis_api_user)
group = devconfig.secrets('hypothesis', 'group', group_name)
get_annos, annos, stream_thread, exit_loop = annoSync(
group_to_memfile(group + 'sparcur'),
helpers=(HypothesisHelper, Hybrid, protc),
group=group,
sync=False)
[protc(a, annos) for a in annos]
[Hybrid(a, annos) for a in annos]
def populate_annos():
from protcur.core import annoSync
from protcur.analysis import Hybrid, protc
from hyputils.hypothesis import HypothesisHelper, group_to_memfile
from hyputils import hypothesis as hyp
ProtcurData.protc = protc
group = auth.get('hypothesis-group')
get_annos, annos, stream_thread, exit_loop = annoSync(
group_to_memfile(group + 'sparcur'),
helpers=(HypothesisHelper, Hybrid, protc),
group=group,
sync=False)
# FIXME hack to workaround bad api key init for hyutils until we can integrate orthauth
get_annos.api_token = auth.get('hypothesis-api-key')
annos.clear()
annos.extend(get_annos())
# FIXME this is expensive and slow to continually recompute
[protc(a, annos) for a in annos]
[Hybrid(a, annos) for a in annos]
def populate_annos():
from protcur import document as ptcdoc
from protcur.core import annoSync
from protcur.analysis import Hybrid, protc
from hyputils.hypothesis import HypothesisHelper, group_to_memfile, AnnoReader
from hyputils import hypothesis as hyp
ProtcurData.protc = protc
group = auth.get('hypothesis-group')
memfile = group_to_memfile(group + 'sparcur')
#ar = AnnoReader(memfile, group)
#annos = ar.get_annos()
#ud = [a.updated for a in annos]
#lud_before = max(ud) if ud else None
#del ud
get_annos, annos, stream_thread, exit_loop = annoSync(
memfile,
helpers=(HypothesisHelper, Hybrid, protc),
group=group,
sync=False)
# FIXME hack to workaround bad api key init for hyutils until we can integrate orthauth
get_annos.api_token = auth.get('hypothesis-api-key')
annos.clear()
annos.extend([ptcdoc.Annotation(a) for a in get_annos()])
#lud_after = max([a.updated for a in annos])
# reset classes in case some other class has populated them
# (e.g. during testing) FIXME this is a bad hack
protc.reset()
Hybrid.reset()
#if lud_before == lud_after: # no new annos
# yeah we can use pickled ... but how :/
#return ???
# FIXME this is expensive and slow to continually recompute
[protc(a, annos) for a in annos]
[Hybrid(a, annos) for a in annos]
import unittest
from hyputils.hypothesis import Memoizer
from hyputils.hypothesis import api_token, username, group, group_to_memfile
get_annos = Memoizer(group_to_memfile(group), api_token, username, group)
bad_memfile = '/tmp/test-bad-memfile.json'
world_get = Memoizer(bad_memfile, api_token, username, '__world__')
world_annos = world_get.get_annos_from_api(max_results=10)
world_get.memoize_annos(world_annos)
class TestMemoize(unittest.TestCase):
def test_0_get_start(self):
annos = get_annos.get_annos_from_api(max_results=400)
get_annos.memoize_annos(annos)
def test_1_get_file(self):
annos, lsu = get_annos.get_annos_from_file()
def test_2_add_missin(self):
annos, lsu = get_annos.get_annos_from_file()
more_annos = get_annos.add_missing_annos(annos, lsu)
assert len(more_annos) > 800 > len(annos)
def test_3_get_rest(self):
annos = get_annos.get_annos_from_api(max_results=400)
get_annos.memoize_annos(annos)
more_annos = get_annos()
assert len(more_annos) > 800 > len(annos)
import unittest
from hyputils.hypothesis import Memoizer, HypothesisHelper
from hyputils.hypothesis import api_token, username, group, group_to_memfile
get_annos = Memoizer(group_to_memfile(group), api_token, username, group)
annos = get_annos()
class TestZHelper(unittest.TestCase):
""" NOTE these tests are stateful """
def test_0_partial(self):
partial = [HypothesisHelper(a, annos) for a in annos[:100]]
repr(partial)
assert len(partial) == 100
def test_1_helper(self):
[HypothesisHelper(a, annos) for a in annos[100:]]
hh = list(HypothesisHelper)
assert len(hh) > 800
repr(hh)
def test_2_by_id(self):
h = next(iter(HypothesisHelper))
hanno = HypothesisHelper.byId(h.id)
assert hanno == h # that silent failure on passing in h instead of h.id though ...
def test_3_by_id_missing(self):
hanno = HypothesisHelper.byId('not a real id')
assert hanno is None
# hypothesis
api_token = environ.get('SCIBOT_API_TOKEN', 'TOKEN') # Hypothesis API token
username = environ.get('SCIBOT_USERNAME', 'USERNAME') # Hypothesis username
group = environ.get('SCIBOT_GROUP', '__world__')
group2 = environ.get('SCIBOT_GROUP2', '__world__')
group_staging = environ.get('SCIBOT_GROUP_STAGING', '__world__')
syncword = environ.get('SCIBOT_SYNC')
READ_ONLY = True
if group_staging == '__world__' and not READ_ONLY:
raise IOError('WARNING YOU ARE DOING THIS FOR REAL PLEASE COMMENT OUT THIS LINE')
def _post(group_hash):
if group_hash.startswith('f'):
print('Real annos')
elif group_hash.startswith('9'):
print('Test annos')
memfile = group_to_memfile(group, _post)
pmemfile = f'/tmp/annos-{UID}-__world__-{username}.pickle'
if group_staging == '__world__':
smemfile = f'/tmp/annos-{UID}-__world__-{username}.pickle'
else:
smemfile = group_to_memfile(group_staging)
# rrid resolver
resolver_xml_filepath = Path('~/ni/dev/rrid/scibot/scibot_rrid_xml.pickle').expanduser() # FIXME