def create_db():
"""Create the ElasticSearch index for Annotations and Documents"""
try:
es.conn.indices.create(es.index)
except elasticsearch_exceptions.RequestError as e:
if not (e.error.startswith('IndexAlreadyExistsException')
or e.error.startswith('InvalidIndexNameException')):
raise
except elasticsearch_exceptions.ConnectionError as e:
msg = ('Can not access ElasticSearch at {0}! '
'Check to ensure it is running.').format(es.host)
raise elasticsearch_exceptions.ConnectionError('N/A', msg, e)
# Pylint issue #258: https://bitbucket.org/logilab/pylint/issue/258
#
# pylint: disable=unexpected-keyword-arg
es.conn.cluster.health(wait_for_status='yellow')
# pylint: enable=unexpected-keyword-arg
try:
Annotation.update_settings()
Annotation.create_all()
Document.create_all()
except elasticsearch_exceptions.RequestError as e:
if e.error.startswith('MergeMappingException'):
date = time.strftime('%Y-%m-%d')
message = ("Elasticsearch index mapping is incorrect! Please "
"reindex it. For example, run: "
"./bin/hypothesis reindex {0} {1} {1}-{2}"
.format('yourconfig.ini', es.index, date)
)
log.critical(message)
raise RuntimeError(message)
raise
def session(self, db_session, added_ann_id, changed_ann_id,
deleted_ann_id):
# Populate the DB session with different types of change relative to the
# last-committed state. We could use any model object for this purpose
# but annotations are the primary object in the system.
doc = Document(web_uri="https://example.org")
changed = Annotation(id=changed_ann_id,
userid="foo",
groupid="wibble",
document=doc)
deleted = Annotation(id=deleted_ann_id,
userid="foo",
groupid="wibble",
document=doc)
db_session.add(changed)
db_session.add(deleted)
db_session.commit()
changed.text = "changed text"
db_session.delete(deleted)
added = Annotation(id=added_ann_id,
userid="foo",
groupid="wibble",
document=doc)
db_session.add(added)
return db_session
def test_generate_notifications_only_if_author_can_read_reply(Subscriptions,
render_reply_notification,
effective_principals):
"""
If the annotation is not readable by the parent author, no notifications
should be generated.
"""
private_annotation = Annotation.fetch(6)
shared_annotation = Annotation.fetch(7)
request = _create_request()
effective_principals.return_value = [
security.Everyone,
security.Authenticated,
'acct:[email protected]',
'group:wibble',
]
Subscriptions.get_active_subscriptions_for_a_type.return_value = [
MockSubscription(id=1, uri='acct:[email protected]')
]
render_reply_notification.return_value = (
'Dummy subject',
'Dummy text',
'Dummy HTML',
['*****@*****.**']
)
notifications = rt.generate_notifications(request, private_annotation, 'create')
assert list(notifications) == []
notifications = rt.generate_notifications(request, shared_annotation, 'create')
assert list(notifications) != []
def parent_values(annotation):
if 'references' in annotation:
parent = Annotation.fetch(annotation['references'][-1])
if 'references' in parent:
grandparent = Annotation.fetch(parent['references'][-1])
parent['quote'] = grandparent['text']
return parent
else:
return {}
def parent_values(annotation):
if 'references' in annotation:
parent = Annotation.fetch(annotation['references'][-1])
if 'references' in parent:
grandparent = Annotation.fetch(parent['references'][-1])
parent['quote'] = grandparent['text']
return parent
else:
return {}
def create_db():
"""Create the ElasticSearch index for Annotations and Documents"""
try:
es.conn.indices.create(es.index)
except elasticsearch_exceptions.RequestError as e:
if not (e.error.startswith('IndexAlreadyExistsException')
or e.error.startswith('InvalidIndexNameException')):
raise
except elasticsearch_exceptions.ConnectionError as e:
msg = ('Can not access ElasticSearch at {0}! '
'Check to ensure it is running.').format(es.host)
raise elasticsearch_exceptions.ConnectionError('N/A', msg, e)
es.conn.cluster.health(wait_for_status='yellow')
Annotation.update_settings()
Annotation.create_all()
Document.create_all()
def reply(self, annotations, parent):
# We need to create a document object to provide the title, and
# ensure it is associated with the annotation through the
# annotation's `target_uri`
doc = Document.find_or_create_by_uris(db.Session,
claimant_uri='http://example.net/foo',
uris=[]).one()
doc.meta.append(DocumentMeta(type='title',
value=['Some document'],
claimant='http://example.com/foo'))
reply = Annotation(**FIXTURE_DATA['reply'])
reply.target_uri = 'http://example.net/foo'
reply.references = [parent.id]
db.Session.add(reply)
db.Session.flush()
annotations[reply.id] = reply
return reply
def reply(self):
common = {
"id": "bar456",
"created": datetime.datetime.utcnow(),
"updated": datetime.datetime.utcnow(),
"text": "No it is not!",
}
return Annotation(target_uri="http://example.org/", **common)
def reply(self, annotations, db_session, parent):
# We need to create a document object to provide the title, and
# ensure it is associated with the annotation through the
# annotation's `target_uri`
doc = Document.find_or_create_by_uris(db_session,
claimant_uri='http://example.net/foo',
uris=[]).one()
doc.meta.append(DocumentMeta(type='title',
value=['Some document'],
claimant='http://example.com/foo'))
reply = Annotation(**FIXTURE_DATA['reply'])
reply.target_uri = 'http://example.net/foo'
reply.references = [parent.id]
db_session.add(reply)
db_session.flush()
annotations[reply.id] = reply
return reply
def test_generate_notifications_empty_if_action_not_create():
"""If the action is not 'create', no notifications should be generated."""
annotation = Annotation()
request = DummyRequest()
notifications = rt.generate_notifications(request, annotation, 'update')
assert list(notifications) == []
def annotation(db_session):
from h.models import Annotation
ann = Annotation(userid='acct:testuser@localhost',
groupid='__world__',
shared=True)
db_session.add(ann)
db_session.commit()
return ann
def parent(self):
common = {
"id": "foo123",
"created": datetime.datetime.utcnow(),
"updated": datetime.datetime.utcnow(),
"text": "Foo is true",
}
return Annotation(target_uri="http://example.org/", **common)
def annotations_index(context, request):
"""Do a search for all annotations on anything and return results.
This will use the default limit, 20 at time of writing, and results
are ordered most recent first.
"""
user = get_user(request)
return Annotation.search(user=user)
def test_generate_notifications_empty_if_annotation_has_no_parent():
"""If the annotation has no parent no notifications should be generated."""
annotation = Annotation.fetch(0)
request = DummyRequest()
notifications = rt.generate_notifications(request, annotation, 'create')
assert list(notifications) == []
def parent(self):
common = {
'id': 'foo123',
'created': datetime.datetime.utcnow(),
'updated': datetime.datetime.utcnow(),
'text': 'Foo is true',
}
return Annotation(target_uri='http://example.org/', **common)
def __getitem__(self, key):
annotation = Annotation.fetch(key)
if annotation is None:
raise KeyError(key)
annotation.__name__ = key
annotation.__parent__ = self
return annotation
def annotations_index(context, request):
"""Do a search for all annotations on anything and return results.
This will use the default limit, 20 at time of writing, and results
are ordered most recent first.
"""
user = get_user(request)
return Annotation.search(user=user)
def reply(self):
common = {
'id': 'bar456',
'created': datetime.datetime.utcnow(),
'updated': datetime.datetime.utcnow(),
'text': 'No it is not!',
}
return Annotation(target_uri='http://example.org/', **common)
def send_annotations(self):
request = self.request
user = get_user(request)
annotations = Annotation.search_raw(query=self.query.query, user=user)
self.received = len(annotations)
packet = _annotation_packet(annotations, 'past')
data = json.dumps(packet)
self.send(data)
def session(self, db_session, added_ann_id, changed_ann_id, deleted_ann_id):
# Populate the DB session with different types of change relative to the
# last-committed state. We could use any model object for this purpose
# but annotations are the primary object in the system.
doc = Document(web_uri='https://example.org')
changed = Annotation(id=changed_ann_id, userid='foo', groupid='wibble', document=doc)
deleted = Annotation(id=deleted_ann_id, userid='foo', groupid='wibble', document=doc)
db_session.add(changed)
db_session.add(deleted)
db_session.commit()
changed.text = 'changed text'
db_session.delete(deleted)
added = Annotation(id=added_ann_id, userid='foo', groupid='wibble', document=doc)
db_session.add(added)
return db_session
def test_og_no_document(pyramid_request, group_service, links_service, sidebar_app):
annotation = Annotation(id='123', userid='foo', target_uri='http://example.com')
context = AnnotationResource(annotation, group_service, links_service)
sidebar_app.side_effect = _fake_sidebar_app
ctx = main.annotation_page(context, pyramid_request)
def test(d):
return 'foo' in d['content']
assert any(test(d) for d in ctx['meta_attrs'])
def group_with_two_users(db_session, factories):
"""
Create a group with two members and an annotation created by each.
"""
creator = factories.User()
member = factories.User()
group = Group(authority=creator.authority, creator=creator, members=[creator, member],
name='test', pubid='group_with_two_users')
db_session.add(group)
doc = Document(web_uri='https://example.org')
creator_ann = Annotation(userid=creator.userid, groupid=group.pubid, document=doc)
member_ann = Annotation(userid=member.userid, groupid=group.pubid, document=doc)
db_session.add(creator_ann)
db_session.add(member_ann)
db_session.flush()
return (group, creator, member, creator_ann, member_ann)
def search(context, request):
"""Search the database for annotations matching with the given query."""
# The search results are filtered for the authenticated user
user = get_user(request)
# Compile search parameters
search_params = _search_params(request.params, user=user)
log.debug("Searching with user=%s, for uri=%s",
user.id if user else 'None',
search_params.get('uri'))
results = Annotation.search(**search_params)
total = Annotation.count(**search_params)
return {
'rows': results,
'total': total,
}
def _create_annotation(fields, user):
"""Create and store an annotation"""
# Some fields are not to be set by the user, ignore them
for field in PROTECTED_FIELDS:
fields.pop(field, None)
# Create Annotation instance
annotation = Annotation(fields)
annotation['user'] = user.id
annotation['consumer'] = user.consumer.key
# Save it in the database
annotation.save()
log.debug('Created annotation; user: %s, consumer key: %s',
annotation['user'], annotation['consumer'])
return annotation
def create_db():
"""Create the ElasticSearch index for Annotations and Documents"""
try:
es.conn.indices.create(es.index)
except elasticsearch_exceptions.RequestError as e:
if not (e.error.startswith('IndexAlreadyExistsException')
or e.error.startswith('InvalidIndexNameException')):
raise
except elasticsearch_exceptions.ConnectionError as e:
msg = ('Can not access ElasticSearch at {0}! '
'Check to ensure it is running.').format(es.host)
raise elasticsearch_exceptions.ConnectionError('N/A', msg, e)
# Pylint issue #258: https://bitbucket.org/logilab/pylint/issue/258
#
# pylint: disable=unexpected-keyword-arg
es.conn.cluster.health(wait_for_status='yellow')
# pylint: enable=unexpected-keyword-arg
Annotation.update_settings()
Annotation.create_all()
Document.create_all()
def _create_annotation(fields, user):
"""Create and store an annotation"""
# Some fields are not to be set by the user, ignore them
for field in PROTECTED_FIELDS:
fields.pop(field, None)
# Create Annotation instance
annotation = Annotation(fields)
annotation['user'] = user.id
annotation['consumer'] = user.consumer.key
# Save it in the database
annotation.save()
log.debug('Created annotation; user: %s, consumer key: %s',
annotation['user'], annotation['consumer'])
return annotation
def test_og_no_document(
pyramid_request, groupfinder_service, links_service, sidebar_app
):
annotation = Annotation(id="123", userid="foo", target_uri="http://example.com")
context = AnnotationContext(annotation, groupfinder_service, links_service)
sidebar_app.side_effect = _fake_sidebar_app
ctx = main.annotation_page(context, pyramid_request)
expected = Any.dict.containing({"content": Any.string.containing("foo")})
assert expected in ctx["meta_attrs"]
def parent(self, storage_driver):
common = {
'id': 'foo123',
'created': datetime.datetime.utcnow(),
'updated': datetime.datetime.utcnow(),
'text': 'Foo is true',
}
uri = 'http://example.org/'
if storage_driver == 'elastic':
return elastic.Annotation(uri=uri, **common)
else:
return Annotation(target_uri=uri, **common)
def reply(self, storage_driver):
common = {
'id': 'bar456',
'created': datetime.datetime.utcnow(),
'updated': datetime.datetime.utcnow(),
'text': 'No it is not!',
}
uri = 'http://example.org/'
if storage_driver == 'elastic':
return elastic.Annotation(uri=uri, **common)
else:
return Annotation(target_uri=uri, **common)
def test_generate_notifications_checks_subscriptions(Subscriptions):
"""If the annotation has a parent, then proceed to check subscriptions."""
request = _create_request()
annotation = Annotation.fetch(1)
Subscriptions.get_active_subscriptions_for_a_type.return_value = []
notifications = rt.generate_notifications(request, annotation, "create")
# Read the generator
list(notifications)
Subscriptions.get_active_subscriptions_for_a_type.assert_called_with(REPLY_TYPE)
def handle_annotation_event(message, socket):
"""
Get message about annotation event `message` to be sent to `socket`.
Inspects the embedded annotation event and decides whether or not the
passed socket should receive notification of the event.
Returns None if the socket should not receive any message about this
annotation event, otherwise a dict containing information about the event.
"""
action = message['action']
annotation = Annotation(**message['annotation'])
if action == 'read':
return None
if message['src_client_id'] == socket.client_id:
return None
if annotation.get('nipsa') and (socket.request.authenticated_userid !=
annotation.get('user', '')):
return None
if not _authorized_to_read(socket.request, annotation):
return None
# We don't send anything until we have received a filter from the client
if socket.filter is None:
return None
if not socket.filter.match(annotation, action):
return None
return {
'payload': [annotation],
'type': 'annotation-notification',
'options': {
'action': action
},
}
def reply(self, annotations, storage_driver, parent):
if storage_driver == 'elastic':
reply = elastic.Annotation(**FIXTURE_DATA['reply_elastic'])
reply['document'] = {'title': ['Some document']}
reply['references'] = [parent.id]
else:
# We need to create a document object to provide the title, and
# ensure it is associated with the annotation through the
# annotation's `target_uri`
doc = Document.find_or_create_by_uris(db.Session,
claimant_uri='http://example.net/foo',
uris=[]).one()
doc.meta.append(DocumentMeta(type='title',
value=['Some document'],
claimant='http://example.com/foo'))
reply = Annotation(**FIXTURE_DATA['reply_postgres'])
reply.target_uri = 'http://example.net/foo'
reply.references = [parent.id]
db.Session.add(reply)
db.Session.flush()
annotations[reply.id] = reply
return reply
def test_generate_notifications_checks_subscriptions(Subscriptions):
"""If the annotation has a parent, then proceed to check subscriptions."""
request = _create_request()
annotation = Annotation.fetch(1)
Subscriptions.get_active_subscriptions_for_a_type.return_value = []
notifications = rt.generate_notifications(request, annotation, 'create')
# Read the generator
list(notifications)
Subscriptions.get_active_subscriptions_for_a_type.assert_called_with(
REPLY_TYPE)
def handle_annotation_event(message, socket):
"""
Get message about annotation event `message` to be sent to `socket`.
Inspects the embedded annotation event and decides whether or not the
passed socket should receive notification of the event.
Returns None if the socket should not receive any message about this
annotation event, otherwise a dict containing information about the event.
"""
action = message['action']
annotation = Annotation(**message['annotation'])
if action == 'read':
return None
if message['src_client_id'] == socket.client_id:
return None
if annotation.get('nipsa') and (
socket.request.authenticated_userid != annotation.get('user', '')):
return None
if not _authorized_to_read(socket.request, annotation):
return None
# We don't send anything until we have received a filter from the client
if socket.filter is None:
return None
if not socket.filter.match(annotation, action):
return None
return {
'payload': [annotation],
'type': 'annotation-notification',
'options': {'action': action},
}
def broadcast_from_queue(queue, sockets):
"""
Pulls messages from a passed queue object, and handles dispatching them to
appropriate active sessions.
"""
for message in queue:
data_in = json.loads(message.body)
action = data_in['action']
annotation = Annotation(**data_in['annotation'])
payload = _annotation_packet([annotation], action)
data_out = json.dumps(payload)
for socket in list(sockets):
if should_send_event(socket, annotation, data_in):
socket.send(data_out)
def test_og_no_document(pyramid_request, group_service, links_service,
sidebar_app):
annotation = Annotation(id="123",
userid="foo",
target_uri="http://example.com")
context = AnnotationContext(annotation, group_service, links_service)
sidebar_app.side_effect = _fake_sidebar_app
ctx = main.annotation_page(context, pyramid_request)
def test(d):
return "foo" in d["content"]
assert any(test(d) for d in ctx["meta_attrs"])
def _search(request_params, user=None):
# Compile search parameters
search_params = _search_params(request_params, user=user)
log.debug("Searching with user=%s, for uri=%s",
user.id if user else 'None', request_params.get('uri'))
if 'any' in search_params['query']:
# Handle any field parameters
query = _add_any_field_params_into_query(search_params)
results = Annotation.search_raw(query)
params = {'search_type': 'count'}
count = Annotation.search_raw(query, params, raw_result=True)
total = count['hits']['total']
else:
results = Annotation.search(**search_params)
total = Annotation.count(**search_params)
return {
'rows': results,
'total': total,
}
def test_check_conditions_false_stops_sending():
"""If the check conditions() returns False, no notifications are generated"""
request = _create_request()
annotation = Annotation.fetch(1)
with patch('h.notification.reply_template.Subscriptions') as mock_subs:
mock_subs.get_active_subscriptions_for_a_type.return_value = [
MockSubscription(id=1, uri='acct:[email protected]')
]
with patch('h.notification.reply_template.check_conditions') as mock_conditions:
mock_conditions.return_value = False
with pytest.raises(StopIteration):
msgs = rt.generate_notifications(request, annotation, 'create')
msgs.next()
def send_annotations(self):
request = self.request
user = get_user(request)
annotations = Annotation.search_raw(query=self.query.query, user=user)
self.received = len(annotations)
# Can send zero to indicate that no past data is matched
packet = {
'payload': annotations,
'type': 'annotation-notification',
'options': {
'action': 'past',
}
}
self.send(packet)
def test_send_if_everything_is_okay():
"""Test whether we generate notifications if every condition is okay"""
request = _create_request()
annotation = Annotation.fetch(1)
with patch('h.notification.reply_template.Subscriptions') as mock_subs:
mock_subs.get_active_subscriptions_for_a_type.return_value = [
MockSubscription(id=1, uri='acct:[email protected]')
]
with patch('h.notification.reply_template.check_conditions') as mock_conditions:
mock_conditions.return_value = True
with patch('h.notification.reply_template.render') as mock_render:
mock_render.return_value = ''
with patch('h.notification.reply_template.get_user_by_name') as mock_user_db:
user = Mock()
user.email = '*****@*****.**'
mock_user_db.return_value = user
msgs = rt.generate_notifications(request, annotation, 'create')
msgs.next()
def _add_any_field_params_into_query(search_params):
"""Add any_field parameters to ES query"""
any_terms = search_params['query'].getall('any')
del search_params['query']['any']
offset = search_params.get('offset', None)
limit = search_params.get('limit', None)
query = Annotation._build_query(search_params['query'], offset, limit)
multi_match_query = {
'multi_match': {
'query': any_terms,
'type': 'cross_fields',
'fields': ['quote', 'tags', 'text', 'uri', 'user']
}
}
# Remove match_all if we add the multi-match part
if 'match_all' in query['query']['bool']['must'][0]:
query['query']['bool']['must'] = []
query['query']['bool']['must'].append(multi_match_query)
return query
def parent_values(annotation):
if 'references' in annotation:
parent = Annotation.fetch(annotation['references'][-1])
return parent
else:
return {}
def delete_db():
Annotation.drop_all()
Document.drop_all()
def parent(self, annotations):
parent = Annotation(**FIXTURE_DATA['parent'])
annotations[parent.id] = parent
return parent
def delete_db():
Annotation.drop_all()
Document.drop_all()