def make_anno(data, dbdocs):
#document_uri_dicts = data['document']['document_uri_dicts']
#document_meta_dicts = data['document']['document_meta_dicts']
#del data['document']
#data = {k:v for k, v in data.items() if k != 'document'} # prevent overwrite on batch load
annotation = models.Annotation(
**data) # FIXME for batch the overhead here is stupid beyond belief
annotation.document_id = dbdocs[uri_normalize(annotation.target_uri)].id
#for k, v in data.items():
#print(k, v)
#setattr(annotation, k, v)
#id, created, updated = extra
#annotation.id = id
#annotation.created = created
#annotation.updated = updated
return annotation
# this baby is super slow
document = update_document_metadata(session,
annotation.target_uri,
document_meta_dicts,
document_uri_dicts,
created=created,
updated=updated)
annotation.document = document
return annotation
def find_by_uris(cls, session, uris):
"""Find documents by a list of uris."""
query_uris = [uri_normalize(u) for u in uris]
matching_claims = (session.query(DocumentURI).filter(
DocumentURI.uri_normalized.in_(query_uris)).distinct(
DocumentURI.document_id).subquery())
return session.query(Document).join(matching_claims)
def find_by_uris(cls, session, uris):
"""Find documents by a list of uris."""
query_uris = [uri_normalize(u) for u in uris]
matching_claims = (
session.query(DocumentURI)
.filter(DocumentURI.uri_normalized.in_(query_uris))
.distinct(DocumentURI.document_id)
.subquery()
)
return session.query(Document).join(matching_claims)
def sync_anno_stream(self, search_after=None, stop_at=None):
""" streaming one anno at a time version of sync """
for row in self.yield_from_api(search_after=last_updated,
stop_at=stop_at):
yield row, 'TODO'
continue
# TODO
datum = validate(row) # roughly 30x slower than quickload
# the h code I'm calling assumes these are new annos
datum['id'] = row['id']
datum['created'] = row['created']
datum['updated'] = row['updated']
document_dict = datum.pop('document')
document_uri_dicts = document_dict['document_uri_dicts']
document_meta_dicts = document_dict['document_meta_dicts']
a = [
models.Annotation(**d,
document_id=dbdocs[uri_normalize(
d['target_uri'])].id) for d in datas
] # slow
self.log.debug('making annotations')
self.session.add_all(a)
self.log.debug('adding all annotations')
def do_check():
api_rows # so that it is accessible in function scope
self.log.debug('checking for consistency')
annos = self.session.query(models.Annotation).\
filter(models.Annotation.groupid == self.group).all()
#docs = self.session.query(models.Document).all()
durs = self.session.query(models.DocumentURI).all()
doc_uris = defaultdict(set)
_ = [doc_uris[d.document_id].add(d.uri) for d in durs]
doc_uris = dict(doc_uris)
#dms = self.session.query(models.DocumentMeta).all()
#doc_mismatch = [a for a in annos if anno_id_to_doc_id[a.id] != a.document.id] # super slow due to orm fetches
doc_missing = [a for a in annos if a.id not in anno_id_to_doc_id]
assert not doc_missing
doc_mismatch = [
a for a in annos if anno_id_to_doc_id[a.id] != a.document_id
]
assert not doc_mismatch, doc_mismatch
# don't use the orm to do this, it is too slow even if you send the other queries above
#embed()
uri_mismatch = [(a.target_uri, doc_uris[a.document_id], a)
for a in annos
if a.target_uri not in doc_uris[a.document_id]]
# NOTE hypothesis only allows 1 record per normalized uri, so we have to normalize here as well
maybe_mismatch = set(
frozenset(s) for u, s, a in uri_mismatch if not s.add(u))
h_mismatch = set(s for s in maybe_mismatch
if len(frozenset(uri_normalize(u)
for u in s)) > 1)
self.log.debug(f'h mismatch has {len(h_mismatch)} cases')
# the above normalization is not sufficient for cases where there are two
# hypothes.is normalized uris AND a scibot normalized uri as well
super_mismatch = set(
s for s in h_mismatch
if len(frozenset(uri_normalization(u) for u in s)) > 1)
assert not super_mismatch, super_mismatch
def uri_normalization(uri):
""" NOTE: this does NOT produce uris """
try:
# strip hypothesis extension prefix
if uri.startswith('chrome-extension://bjfhmglciegochdpefhhlphglcehbmek/content/web/viewer.html?file='):
junk, uri = uri.split('=', 1)
# universal fixes
no_fragment, *_frag = uri.rsplit('#', 1)
no_trailing_slash = no_fragment.rstrip('/') # annoying
_scheme, no_scheme = no_trailing_slash.split('://', 1)
# special cases
if 'frontiersin.org' in no_scheme:
# og:url on frontiers is incorrect
no_scheme = no_scheme.replace('article/', 'articles/')
elif 'fasebj.org' in no_scheme: # FIXME this one has _all_ the variants :/
no_scheme = (no_scheme
.replace('.abstract', '')
.replace('.full', '')
.replace('.pdf', '')
)
elif no_scheme.endswith('?needAccess=true'):
no_scheme = no_scheme[:-len('?needAccess=true')]
elif '?systemMessage' in no_scheme:
no_scheme, junk = no_scheme.rsplit('?systemMessage', 1)
# specific fixes
if anyMembers(no_scheme,
'acs.org',
'ahajournals.org',
'biologicalpsychiatryjournal.com',
'ebiomedicine.com',
'fasebj.org',
'frontiersin.org',
'future-science.com',
'hindawi.com',
'ieee.org',
'jclinepi.com',
'jpeds.com',
'liebertpub.com',
'mitpressjournals.org',
'molbiolcell.org',
'molmetab.com',
'neurobiologyofaging.org',
'physiology.org',
'sagepub.com',
'sciencedirect.com',
'tandfonline.com',
'theriojournal.com',
'wiley.com',):
# NOTE not all the above hit all of these
# almost all still resolve
normalized = (no_scheme
.replace('/abstract', '')
.replace('/abs', '')
.replace('/fulltext', '')
.replace('/full', '')
.replace('/pdf', ''))
#elif ('sciencedirect.com' in no_scheme):
#normalized = (no_scheme
#.replace('/abs', ''))
elif ('cell.com' in no_scheme):
normalized = (no_scheme # FIXME looks like cell uses /abstract in og:url
.replace('/abstract', '/XXX')
.replace('/fulltext', '/XXX'))
elif 'jneurosci.org' in no_scheme:
# TODO content/early -> resolution_chain(doi)
normalized = (no_scheme
.replace('.short', '')
.replace('.long', '')
.replace('.full', '')
.replace('.pdf', '')
# note .full.pdf is a thing
)
elif 'pnas.org' in no_scheme:
normalized = (no_scheme
.replace('.short', '')
.replace('.long', '')
.replace('.full', ''))
elif 'mdpi.com' in no_scheme:
normalized = (no_scheme
.replace('/htm', ''))
elif 'f1000research.com' in no_scheme:
# you should be ashamed of yourselves for being in here for this reason
normalized, *maybe_version = no_scheme.rsplit('/v', 1)
elif 'academic.oup.com' in no_scheme:
normalized, *maybesr = no_scheme.rsplit('?searchresult=', 1)
_normalized, maybe_junk = normalized.rsplit('/', 1)
numbers = '0123456789'
if (maybe_junk[0] not in numbers or # various ways to detect the human readable junk after the id
maybe_junk[-1] not in numbers or
'-' in maybe_junk or
len(maybe_junk) > 20):
normalized = _normalized
elif anyMembers(no_scheme,
'jci.org',
'nature.com'):
# cases where safe to remove query fragment
normalized, *_query = no_scheme.rsplit('?', 1)
normalized, *table_number = normalized.rsplit('/tables/', 1)
elif 'pubmed/?term=' in no_scheme and noneMembers(no_scheme, ' ', '+'):
normalized = no_scheme.replace('?term=', '')
elif 'nih.gov/pubmed/?' in no_scheme:
# FIXME scibot vs client norm?
normalized = no_scheme.replace(' ', '+')
elif 'govhttp' in no_scheme:
# lol oh dear
hrm, oops = no_scheme.split('govhttp')
ded, wat = oops.split('//', 1)
blargh, suffix = wat.split('/', 1)
normalized = hrm + 'gov/pmc/' + suffix
elif 'table/undtbl' in no_scheme:
normalized, table_number = no_scheme.rsplit('table/undtbl')
elif anyMembers(no_scheme,
'index.php?',
):
# cases where we just use hypothes.is normalization
_scheme, normalized = uri_normalize(uri).split('://') # FIXME h dependency
else:
normalized = no_scheme
'onlinelibrary.wiley.com/doi/10.1002/cne.23727?wol1URL=/doi/10.1002/cne.23727®ionCode=US-CA&identityKey=e2523300-b934-48c9-b08e-940de05d7335'
'www.jove.com/video/55441/?language=Japanese'
'www.nature.com/neuro/journal/v19/n5/full/nn.4282.html'
'www.nature.com/cr/journal/vaop/ncurrent/full/cr201669a.html'
'https://www.nature.com/articles/cr201669'
#{'www.ingentaconnect.com/content/umrsmas/bullmar/2017/00000093/00000002/art00006':
#[OntId('DOI:10.5343/bms.2016.1044'), OntId('DOI:info:doi/10.5343/bms.2016.1044')]}
# pmid extract from pmc
#<meta name="citation_pmid" content="28955177">
return normalized
except ValueError as e: # split fail
pdf_prefix = 'urn:x-pdf:'
if uri.startswith(pdf_prefix):
return uri
elif uri in bad_uris:
print('AAAAAAAAAAAAAAAAAAAAAAAAAAA', uri)
return 'THIS URI IS GARBAGE AND THIS IS ITS NORMALIZED FORM'
else:
raise TypeError(uri) from e
def q_prepare_docs(self, rows):
existing_unnormed = {
r.uri: (r.document_id, self.convert[0](r.created),
self.convert[0](r.updated))
for r in self.session.execute('SELECT uri, document_id, created, '
'updated FROM document_uri')
}
created_updated = {
docid: (created, updated)
for _, (docid, created, updated) in existing_unnormed.items()
}
_existing = defaultdict(set)
_ = [
_existing[uri_normalization(uri)].add(docid)
for uri, (docid, created, updated) in existing_unnormed.items()
]
assert not [_ for _ in _existing.values() if len(_) > 1
] # TODO proper handling for this case
h_existing_unnormed = {
uri_normalize(uri): docid
for uri, (docid, created, updated) in existing_unnormed.items()
}
existing = {k: next(iter(v))
for k, v in _existing.items()
} # FIXME issues when things get big
latest_existing = max(
u
for c, u in created_updated.values()) if created_updated else None
new_docs = {
} # FIXME this is completely opaque since it is not persisted anywhere
for row in sorted(rows, key=lambda r: r['created']):
id = row['id']
uri, uri_normed, (created, updated, claims) = self.uri_records(row)
try:
docid = existing[uri_normed]
dc, du = created_updated[docid]
doc = models.Document(id=docid, created=dc, updated=du)
if doc.updated < updated:
# FIXME TODO update the record?
#self.log.warning('YOU ARE NOT UPDATING A DOC WHEN YOU SHOULD!!!!!!\n'
#f'{docid} {doc.updated} {updated}')
pass
do_claims = False
except KeyError as e:
if existing:
if row['updated'] <= latest_existing:
# only need to worry if we are recreating
raise e
if uri_normed not in new_docs:
do_claims = True
doc = models.Document(created=created, updated=updated)
self.session.add(doc) # TODO perf testing vs add_all
new_docs[uri_normed] = doc
else:
do_claims = False
doc = new_docs[uri_normed]
#if type(doc.created) == str:
#embed()
yield id, doc
if uri_normalize(uri) not in h_existing_unnormed:
# NOTE allowing only the normalized uri can cause confusion (i.e. see checks in sync_annos)
h_existing_unnormed[uri_normalize(uri)] = doc
# TODO do these get added automatically if their doc gets added but exists?
doc_uri = models.DocumentURI(document=doc,
claimant=uri,
uri=uri,
type='self-claim',
created=created,
updated=updated)
yield None, doc_uri
# because of how this schema is designed
# the only way that this can be fast is
# if we assume that all claims are identical
# FIXME if there is a new claim type then we are toast though :/
# the modelling here assumes that title etc can't change
#print(id, uri, uri_normed, row['user'], row['uri'], row['created'])
if do_claims:
for claim in claims:
#print(id, uri, uri_normed, claim['claimant'], claim['type'], claim['value'])
dm = models.DocumentMeta(document=doc,
created=created,
updated=updated,
**claim)
yield None, dm
def create_or_update_document_meta(
session, claimant, type, value, document, created, updated
):
"""
Create or update a DocumentMeta with the given parameters.
If an equivalent DocumentMeta already exists in the database then its value
and updated time will be updated.
If no equivalent DocumentMeta exists in the database then a new one will be
created and added to the database.
To be considered "equivalent" an existing DocumentMeta must have the given
claimant and type, but its value, document and created and updated times
needn't match the given ones.
:param session: the database session
:type session: sqlalchemy.orm.session.Session
:param claimant: the value to use for the DocumentMeta's claimant attribute
if a new DocumentMeta is created
:type claimant: unicode
:param type: the value of the new or existing DocumentMeta's type attribute
:type type: unicode
:param value: the value to set the new or existing DocumentMeta's value
attribute to
:type value: list of unicode strings
:param document: the value to use for the DocumentMeta's document if a new
DocumentMeta is created
:type document: h.models.Document
:param created: the value to use for the DocumentMeta's created attribute
if a new DocumentMeta is created
:type created: datetime.datetime
:param updated: the value to set the new or existing DocumentMeta's updated
attribute to
:type updated: datetime.datetime
"""
existing_dm = (
session.query(DocumentMeta)
.filter(
DocumentMeta.claimant_normalized == uri_normalize(claimant),
DocumentMeta.type == type,
)
.one_or_none()
)
if existing_dm is None:
session.add(
DocumentMeta(
claimant=claimant,
type=type,
value=value,
document=document,
created=created,
updated=updated,
)
)
else:
existing_dm.value = value
existing_dm.updated = updated
if not existing_dm.document == document:
log.warning(
"Found DocumentMeta (id: %d)'s document_id (%d) doesn't "
"match given Document's id (%d)",
existing_dm.id,
existing_dm.document_id,
document.id,
)
if type == "title" and value and not document.title:
document.title = value[0]
try:
session.flush()
except sa.exc.IntegrityError:
raise ConcurrentUpdateError("concurrent document meta updates")
def create_or_update_document_uri(
session, claimant, uri, type, content_type, document, created, updated
):
"""
Create or update a DocumentURI with the given parameters.
If an equivalent DocumentURI already exists in the database then its
updated time will be updated.
If no equivalent DocumentURI exists in the database then a new one will be
created and added to the database.
To be considered "equivalent" an existing DocumentURI must have the same
claimant, uri, type and content_type, but the Document object that it
belongs to may be different. The claimant and uri are normalized before
comparing.
:param session: the database session
:type session: sqlalchemy.orm.session.Session
:param claimant: the .claimant property of the DocumentURI
:type claimant: unicode
:param uri: the .uri property of the DocumentURI
:type uri: unicode
:param type: the .type property of the DocumentURI
:type type: unicode
:param content_type: the .content_type property of the DocumentURI
:type content_type: unicode
:param document: the Document that the new DocumentURI will belong to, if a
new DocumentURI is created
:type document: h.models.Document
:param created: the time that will be used as the .created time for the new
DocumentURI, if a new one is created
:type created: datetime.datetime
:param updated: the time that will be set as the .updated time for the new
or existing DocumentURI
:type updated: datetime.datetime
"""
docuri = (
session.query(DocumentURI)
.filter(
DocumentURI.claimant_normalized == uri_normalize(claimant),
DocumentURI.uri_normalized == uri_normalize(uri),
DocumentURI.type == type,
DocumentURI.content_type == content_type,
)
.first()
)
if docuri is None:
docuri = DocumentURI(
claimant=claimant,
uri=uri,
type=type,
content_type=content_type,
document=document,
created=created,
updated=updated,
)
session.add(docuri)
elif not docuri.document == document:
log.warning(
"Found DocumentURI (id: %d)'s document_id (%d) doesn't match "
"given Document's id (%d)",
docuri.id,
docuri.document_id,
document.id,
)
docuri.updated = updated
try:
session.flush()
except sa.exc.IntegrityError:
raise ConcurrentUpdateError("concurrent document uri updates")
def claimant(self, value):
self._claimant = value
self._claimant_normalized = uri_normalize(value)
def uri(self, value):
self._uri = value
self._uri_normalized = uri_normalize(value)
def create_or_update_document_meta(session, claimant, type, value, document,
created, updated):
"""
Create or update a DocumentMeta with the given parameters.
If an equivalent DocumentMeta already exists in the database then its value
and updated time will be updated.
If no equivalent DocumentMeta exists in the database then a new one will be
created and added to the database.
To be considered "equivalent" an existing DocumentMeta must have the given
claimant and type, but its value, document and created and updated times
needn't match the given ones.
:param session: the database session
:type session: sqlalchemy.orm.session.Session
:param claimant: the value to use for the DocumentMeta's claimant attribute
if a new DocumentMeta is created
:type claimant: unicode
:param type: the value of the new or existing DocumentMeta's type attribute
:type type: unicode
:param value: the value to set the new or existing DocumentMeta's value
attribute to
:type value: list of unicode strings
:param document: the value to use for the DocumentMeta's document if a new
DocumentMeta is created
:type document: h.models.Document
:param created: the value to use for the DocumentMeta's created attribute
if a new DocumentMeta is created
:type created: datetime.datetime
:param updated: the value to set the new or existing DocumentMeta's updated
attribute to
:type updated: datetime.datetime
"""
existing_dm = (session.query(DocumentMeta).filter(
DocumentMeta.claimant_normalized == uri_normalize(claimant),
DocumentMeta.type == type,
).one_or_none())
if existing_dm is None:
session.add(
DocumentMeta(
claimant=claimant,
type=type,
value=value,
document=document,
created=created,
updated=updated,
))
else:
existing_dm.value = value
existing_dm.updated = updated
if not existing_dm.document == document:
log.warning(
"Found DocumentMeta (id: %d)'s document_id (%d) doesn't "
"match given Document's id (%d)",
existing_dm.id,
existing_dm.document_id,
document.id,
)
if type == "title" and value and not document.title:
document.title = value[0]
try:
session.flush()
except sa.exc.IntegrityError:
raise ConcurrentUpdateError("concurrent document meta updates")
def create_or_update_document_uri(session, claimant, uri, type, content_type,
document, created, updated):
"""
Create or update a DocumentURI with the given parameters.
If an equivalent DocumentURI already exists in the database then its
updated time will be updated.
If no equivalent DocumentURI exists in the database then a new one will be
created and added to the database.
To be considered "equivalent" an existing DocumentURI must have the same
claimant, uri, type and content_type, but the Document object that it
belongs to may be different. The claimant and uri are normalized before
comparing.
:param session: the database session
:type session: sqlalchemy.orm.session.Session
:param claimant: the .claimant property of the DocumentURI
:type claimant: unicode
:param uri: the .uri property of the DocumentURI
:type uri: unicode
:param type: the .type property of the DocumentURI
:type type: unicode
:param content_type: the .content_type property of the DocumentURI
:type content_type: unicode
:param document: the Document that the new DocumentURI will belong to, if a
new DocumentURI is created
:type document: h.models.Document
:param created: the time that will be used as the .created time for the new
DocumentURI, if a new one is created
:type created: datetime.datetime
:param updated: the time that will be set as the .updated time for the new
or existing DocumentURI
:type updated: datetime.datetime
"""
docuri = (session.query(DocumentURI).filter(
DocumentURI.claimant_normalized == uri_normalize(claimant),
DocumentURI.uri_normalized == uri_normalize(uri),
DocumentURI.type == type,
DocumentURI.content_type == content_type,
).first())
if docuri is None:
docuri = DocumentURI(
claimant=claimant,
uri=uri,
type=type,
content_type=content_type,
document=document,
created=created,
updated=updated,
)
session.add(docuri)
elif not docuri.document == document:
log.warning(
"Found DocumentURI (id: %d)'s document_id (%d) doesn't match "
"given Document's id (%d)",
docuri.id,
docuri.document_id,
document.id,
)
docuri.updated = updated
try:
session.flush()
except sa.exc.IntegrityError:
raise ConcurrentUpdateError("concurrent document uri updates")
def claimant(self, value):
self._claimant = value
self._claimant_normalized = uri_normalize(value)
def uri(self, value):
self._uri = value
self._uri_normalized = uri_normalize(value)