def post(self):
newUser = Responder(name = self.request.get('name'), email = self.request.get('email'), company = self.request.get('company'))
newUser.put()
set = ResponseSet(product = 'ADSync', responder = newUser)
set.put()
adQuestions = Question.gql('WHERE product = :1', 'ADSync')
htmlBody = '<h2>Response to ADSync Questionnaire</h2><p><i>Submitted by ' + newUser.name +', ' + newUser.email + '</i></p>'
for adQuestion in adQuestions:
responseText = self.request.get('response' + str(adQuestion.key().id()))
response = Response(text = responseText, question = adQuestion, responseSet = set)
response.put()
htmlBody += '<h3>' + adQuestion.text + '</h3>' + '<p>' + response.text + '</p>'
#send email notification
sender = '*****@*****.**'
recipients = ['*****@*****.**', '*****@*****.**', '*****@*****.**']
sub = newUser.name + ' from ' + newUser.company + ' responded to the ADSync Questionnaire'
plainBody = 'Get response here: http://yammerie.appspot.com/responsesets?id=' + str(set.key().id())
mail.send_mail(sender, recipients, sub, plainBody, html = htmlBody)
self.redirect('/adsuccess')
def process_requests():
global account
global last_processed_request_id
# process request data
ubr_list = UBRList()
ubr_list.deserialize(request.get_json())
app.logger.debug('Processing received bank requests.')
# sort request data
sorted_bank_requests = ubr_list.get_sorted()
# perform bank requests in order
for req in sorted_bank_requests:
if last_processed_request_id < req.id:
if req.operation == 'CREDIT':
account.increase_balance(req.amount)
else:
account.decrease_balance(req.amount)
last_processed_request_id = req.id
# response
response = Response(code=status.HTTP_200_OK,
description='Bankovnà operace úspěšně zpracovány.')
return make_response(response.serialize(), status.HTTP_200_OK)
def take_survey(request, survey_id):
survey = get_object_or_404(Survey, pk=survey_id)
questions = survey.question_set.all()
AnswerFormset = get_answer_formset(questions)
formset = AnswerFormset(request.POST or None, queryset=Answer.objects.none())
if formset.is_valid():
answers = formset.save(commit=False)
response = Response(survey=survey)
response.save()
for answer in answers:
answer.response = response
answer.save()
return HttpResponseRedirect(reverse('simple_survey.views.list_surveys'))
for index in range(len(questions)):
question = questions[index]
form = formset.forms[index]
form.fields['answer'].label = question.question
form.fields['question'].initial = question
responseParameters = {
"survey" : survey,
"formset" : formset,
}
return render_to_response('simple_survey/take_survey.html', responseParameters, context_instance=RequestContext(request))
def _on_rmq_message(self, *args, **kwargs):
if 'rmq_message' in kwargs:
rmq_message = kwargs['rmq_message']
else:
return
if rmq_message is not None:
channel, user, message, plugin_name, plugin_response = \
self._process_rmq_message(rmq_message)
if channel is not None:
if user is not None:
if message is not None:
if plugin_name is not None:
if plugin_response is not None:
is_approved = True if channel.is_secure is False \
else False
response = Response(
text=plugin_response,
from_plugin=plugin_name,
in_response_to=message,
to_channel=channel,
to_user=user,
is_approved=is_approved,
is_sent=False,
)
response.save()
def post_bot_message(response_bot_params):
bot_params = {}
bot_params['bot_id'] = vars(response_bot_params)['bot_id']
bot_params['text'] = vars(response_bot_params)['text']
if response_bot_params.picture_url is not None:
bot_params['picture_url'] = vars(response_bot_params)['picture_url']
result = requests.post('https://api.groupme.com/v3/bots/post', params=bot_params)
if result.status_code == 201:
print("EVENT: POST 201 SUCCESSFUL RESPONSE")
created_response = result.json()['response']['message']
response = Response(201, created_response['id'], None)
return response
if result.status_code == 202:
print("EVENT: POST 202 SUCCESSFUL RESPONSE")
response = Response(202, None, bot_params['text'])
return response
if result.status_code == 400:
print("ERROR: POST 400 BAD REQUEST")
write_error_to_logfile("ERROR: POST 400 BAD REQUEST")
return False
if result.status_code == 401:
print("ERROR: POST 401 UNAUTHORIZED")
write_error_to_logfile("ERROR: POST 401 UNAUTHORIZED")
return False
if result.status_code == 409:
print("ERROR: POST 409 CONFLICT")
write_error_to_logfile("ERROR: POST 409 CONFLICT")
return False
else:
print("ERROR: REQUEST EXCEPTION")
raise requests.exceptions.RequestException
async def add_player(self, name, ws):
player = Player(name=name, ws=ws)
action = "add_player"
if len(self._players) > 4:
response = Response(action=action,
message='"Game is already full"')
elif player in self._players:
response = Response(action=action,
message=f"{player.name} already exists")
else:
self._players.append(player)
player.turn_id = self._players.index(player)
player_id = player.turn_id
setattr(self.game, f"player_{player_id}", player)
response = Response(
action=action,
message=f'Player added! Player ID: {player_id}',
body={
"player_id": player_id,
**player.to_dict()
})
await self.send_response(response=response, player=player)
await self.distribute_game(to_waiting=True)
def answer(request, poll_id, user_uuid):
poll = get_object_or_404(Poll, pk=poll_id)
participant = get_object_or_404(Participant, unique_id=user_uuid)
if participant.completed:
return redirect(reverse('polls:thanks',
args=(poll.id,
user_uuid,)))
questions = poll.question_set.all()
if request.method == 'POST':
form = DetailForm(request.POST, questions=questions)
if form.is_valid():
to_delete = Response.objects.filter(participant=participant)
to_delete.delete()
for choice_id in form.answers():
response = Response(participant=participant,
choice_id=choice_id)
response.save()
return HttpResponseRedirect(reverse('polls:sign',
args=(poll.id,
user_uuid,)))
else:
form = DetailForm(questions=questions)
return render(request, 'polls/detail.html',
{
'poll': poll,
'form': form,
'user_uuid': user_uuid,
})
def volumedown():
"""Decreases the Sonos volume by 10"""
sonos.volume = sonos.volume - 10;
resp = Response("Sonos volume now set to " + str(sonos.volume))
resp.volume = sonos.volume
return jsonpickle.encode(resp, unpicklable=False)
def volume():
"""Returns (GET) or sets (POST) the current Sonos volume (0 - 100)"""
if request.method == 'POST':
sonos.volume = request.form.get('volume')
resp = Response("Sonos volume now set to " + str(sonos.volume))
resp.volume = sonos.volume
return jsonpickle.encode(resp, unpicklable=False)
def mutate(self, info, **kwargs):
req_kw = kwargs['request']
request_instance = RequestModel.objects.get(**req_kw)
kwargs['request'] = request_instance
del kwargs['request']
new_response_instance = ResponseModel(**kwargs)
print 'new response instance: '
print new_response_instance
new_response_instance.save()
return CreateResponse(response=new_response_instance)
def test_get_type(self):
self.assertEqual(
'repost',
Response.get_type({
'objectType': 'activity',
'verb': 'share'
}))
self.assertEqual('rsvp', Response.get_type({'verb': 'rsvp-no'}))
self.assertEqual('rsvp', Response.get_type({'verb': 'invite'}))
self.assertEqual('comment',
Response.get_type({'objectType': 'comment'}))
self.assertEqual('post', Response.get_type({'verb': 'post'}))
self.assertEqual('post', Response.get_type({'objectType': 'event'}))
self.assertEqual('post', Response.get_type({'objectType': 'image'}))
self.assertEqual(
'comment',
Response.get_type({
'objectType': 'note',
'context': {
'inReplyTo': {
'foo': 'bar'
}
},
}))
self.assertEqual(
'comment',
Response.get_type({
'objectType': 'comment',
'verb': 'post',
}))
def test_hooks(self):
resp = Response(id='x', activity_json='{"foo": "bar"}')
self.assertRaises(AssertionError, resp.put)
pre_put = Response._pre_put_hook
del Response._pre_put_hook
resp.put()
Response._pre_put_hook = pre_put
got = resp.key.get()
self.assertEqual(['{"foo": "bar"}'], got.activities_json)
self.assertIsNone(got.activity_json)
def send_webmentions(handler, activity, **response_props):
"""Sends webmentions for an incoming Salmon slap or ActivityPub inbox delivery.
Args:
handler: RequestHandler
activity: dict, AS1 activity
response_props: passed through to the newly created Responses
"""
verb = activity.get('verb')
if verb and verb not in SUPPORTED_VERBS:
error(handler, '%s activities are not supported yet.' % verb)
# extract source and targets
source = activity.get('url') or activity.get('id')
obj = activity.get('object')
obj_url = util.get_url(obj)
targets = util.get_list(activity, 'inReplyTo')
if isinstance(obj, dict):
if not source:
source = obj_url or obj.get('id')
targets.extend(util.get_list(obj, 'inReplyTo'))
if verb in ('like', 'share'):
targets.append(obj_url)
targets = util.dedupe_urls(util.get_url(t) for t in targets)
if not source:
error(handler, "Couldn't find original post URL")
if not targets:
error(handler, "Couldn't find target URLs (inReplyTo or object)")
# send webmentions and store Responses
errors = []
for target in targets:
if not target:
continue
response = Response(source=source, target=target, direction='in',
**response_props)
response.put()
wm_source = response.proxy_url() if verb in ('like', 'share') else source
logging.info('Sending webmention from %s to %s', wm_source, target)
wm = send.WebmentionSend(wm_source, target)
if wm.send(headers=HEADERS):
logging.info('Success: %s', wm.response)
response.status = 'complete'
else:
logging.warning('Failed: %s', wm.error)
errors.append(wm.error)
response.status = 'error'
response.put()
if errors:
msg = 'Errors:\n' + '\n'.join(json.dumps(e, indent=2) for e in errors)
error(handler, msg, status=errors[0].get('http_status'))
def test_good(self):
self.expect_requests_get(
'http://foo.com/', """
<html class="h-feed">
<div class="h-entry">
<a class="u-url" href="http://foo.com/post"></a>
<a class="u-syndication" href="https://www.facebook.com/snarfed.org/posts/123"></a>
</div>
</html>""")
self.mox.ReplayAll()
self.handler.receive(self.mail)
self.assert_equals(200, self.response.status_code)
emails = list(FacebookEmail.query())
self.assertEquals(1, len(emails))
self.assert_equals('SMTP-123-xyz', emails[0].key.id())
self.assert_equals(self.fea.key, emails[0].source)
self.assert_equals([COMMENT_EMAIL_USERNAME], emails[0].htmls)
resp_id = EMAIL_COMMENT_OBJ_USERNAME['id']
self.assert_equals(ndb.Key('Response', resp_id), emails[0].response)
expected = Response(
id=resp_id,
source=self.fea.key,
type='comment',
response_json=json.dumps(EMAIL_COMMENT_OBJ_USERNAME),
activities_json=[
json.dumps({
'id': '123',
'numeric_id': '123',
'url': 'https://www.facebook.com/212038/posts/123',
'author': {
'id': 'snarfed.org'
},
})
],
unsent=['http://foo.com/post'])
self.assert_entities_equal([expected],
list(Response.query()),
ignore=('created', 'updated'))
tasks = self.taskqueue_stub.GetTasks('propagate')
self.assertEquals(1, len(tasks))
self.assert_equals(expected.key.urlsafe(),
testutil.get_task_params(tasks[0])['response_key'])
self.assert_equals(EMAIL_COMMENT_OBJ_USERNAME,
self.fea.get_comment('123_789'))
async def distribute_game(self, to_waiting=False):
response = Response(action=Actions.UPDATE_GAME.value,
body=self.game.to_dict())
serialized_response = response.serialize()
if to_waiting:
tasks = [
asyncio.ensure_future(ws.send(serialized_response))
for ws in self._waiting
]
else:
tasks = [
asyncio.ensure_future(player.ws.send(serialized_response))
for player in self._players
]
await asyncio.gather(*tasks)
def mutate(self, info, **kwargs):
print "getting request info: "
req_kw = kwargs['request']
print req_kw
request_instance = RequestModel.objects.get(**req_kw)
print "request instance: "
print request_instance.id
#kwargs['request'] = request_instance
del kwargs['request']
new_response_instance = ResponseModel(**kwargs)
new_response_instance.request = request_instance
print 'new response instance: '
print new_response_instance
new_response_instance.save()
return CreateResponse(response=new_response_instance)
def respond(request):
"""
Request handler when someone posts a response
1. Add response content to the database
2. Send push notification to client device
3. Update the credit of the responder
"""
if request.method == 'POST':
json_data = json.loads(request.body)
try:
thread_id = json_data['thread_id']
response_content = json_data['content']
device_id = json_data['device_id']
except KeyError:
print "Error: A posted response did not have a JSON object with the required properties"
else:
# check that the thread id and the device ids are valid
thread = Thread.objects.filter(id=thread_id)
device = Device.objects.filter(device_id=device_id)
print "Passed parameter validation"
print thread.count()
print device.count()
if thread.exists() and device.exists():
# add response to database
response = Response(thread=thread[0], responder_device=device[0], response_content=response_content)
response.save()
# add update to the other device
asker_device = thread[0].asker_device
answerer_device = thread[0].answerer_device
print "Thread and device actually exist"
print device_id
print asker_device.device_id
print answerer_device.device_id
if asker_device.device_id == device_id:
ResponseUpdates.add_update(answerer_device, response)
print "Adding an update to the answerers queue"
elif answerer_device.device_id == device_id:
ResponseUpdates.add_update(asker_device, response)
print "Adding an update to the askers queue"
return HttpResponse(json.dumps({}), content_type="application/json")
def add_response(**kwargs):
response = Response(**kwargs)
try:
db.session.add(response)
db.session.commit()
except SQLAlchemyError as e:
print 'ERROR adding response', e
def test_salmon_like(self, mock_get, mock_post):
mock_get.side_effect = [self.like, self.orig_html_atom, self.orig_atom]
got = self.client.post('/webmention',
data={
'source': 'http://a/like',
'target': 'http://orig/post',
})
self.assertEqual(200, got.status_code)
mock_get.assert_has_calls((
self.req('http://a/like'),
self.req('http://orig/post', headers=CONNEG_HEADERS_AS2_HTML),
self.req('http://orig/atom'),
))
data = self.verify_salmon(mock_post)
parsed = feedparser.parse(data)
entry = parsed.entries[0]
self.assertEqual('http://a/like', entry['id'])
self.assertIn(
{
'rel': 'alternate',
'href': 'http://a/like',
'type': 'text/html',
}, entry['links'])
self.assertEqual('http://orig/post', entry['activity_object'])
resp = Response.get_by_id('http://a/like http://orig/post')
self.assertEqual('out', resp.direction)
self.assertEqual('ostatus', resp.protocol)
self.assertEqual('complete', resp.status)
self.assertEqual(self.like_mf2, json_loads(resp.source_mf2))
def test_activitypub_create_repost(self, mock_get, mock_post):
mock_get.side_effect = [self.repost, self.orig_as2, self.actor]
mock_post.return_value = requests_response('abc xyz')
got = self.client.post('/webmention',
data={
'source': 'http://a/repost',
'target': 'https://fed.brid.gy/',
})
self.assertEqual(200, got.status_code)
mock_get.assert_has_calls((
self.req('http://a/repost'),
self.req('http://orig/post', headers=CONNEG_HEADERS_AS2_HTML),
self.req('http://orig/author', headers=CONNEG_HEADERS_AS2_HTML),
))
args, kwargs = mock_post.call_args
self.assertEqual(('https://foo.com/inbox', ), args)
self.assertEqual(self.repost_as2, json_loads(kwargs['data']))
headers = kwargs['headers']
self.assertEqual(CONTENT_TYPE_AS2, headers['Content-Type'])
rsa_key = kwargs['auth'].header_signer._rsa._key
self.assertEqual(self.key.private_pem(), rsa_key.exportKey())
resp = Response.get_by_id('http://a/repost http://orig/as2')
self.assertEqual('out', resp.direction)
self.assertEqual('activitypub', resp.protocol)
self.assertEqual('complete', resp.status)
self.assertEqual(self.repost_mf2, json_loads(resp.source_mf2))
def get(self):
source = util.get_required_param(self, 'source')
target = util.get_required_param(self, 'target')
id = '%s %s' % (source, target)
resp = Response.get_by_id(id)
if not resp:
self.abort(404, 'No stored response for %s' % id)
if resp.source_mf2:
as1 = microformats2.json_to_object(json.loads(resp.source_mf2))
elif resp.source_as2:
as1 = as2.to_as1(json.loads(resp.source_as2))
elif resp.source_atom:
as1 = atom.atom_to_activity(resp.source_atom)
else:
self.abort(404, 'Stored response for %s has no data' % id)
# add HTML meta redirect to source page. should trigger for end users in
# browsers but not for webmention receivers (hopefully).
html = microformats2.activities_to_html([as1])
utf8 = '<meta charset="utf-8">'
refresh = '<meta http-equiv="refresh" content="0;url=%s">' % source
html = html.replace(utf8, utf8 + '\n' + refresh)
self.response.write(html)
def try_salmon(self):
"""
Returns Flask response (string body or tuple) if we attempted OStatus
delivery (whether successful or not), None if we didn't attempt, raises
an exception otherwise.
"""
target = None
if self.target_resp:
target = self.target_resp.url
else:
targets = self._targets()
if targets:
target = targets[0]
if not target:
logging.warning("No targets or followers. Ignoring.")
return
status = None
try:
ret = self._try_salmon(target)
if isinstance(ret, str):
status = 'complete'
return ret
except:
status = 'error'
raise
finally:
if status:
Response(source=self.source_url,
target=target,
status=status,
direction='out',
protocol='ostatus',
source_mf2=json_dumps(self.source_mf2)).put()
def _test_inbox_mention(self, as2, mock_head, mock_get, mock_post):
mock_head.return_value = requests_response(url='http://target')
mock_get.return_value = requests_response(
'<html><head><link rel="webmention" href="/webmention"></html>')
mock_post.return_value = requests_response()
with self.client:
got = self.client.post('/foo.com/inbox', json=as2)
self.assertEqual(200, got.status_code, got.get_data(as_text=True))
mock_get.assert_called_once_with(
'http://target/', headers=common.HEADERS, timeout=15, stream=True)
expected_headers = copy.deepcopy(common.HEADERS)
expected_headers['Accept'] = '*/*'
mock_post.assert_called_once_with(
'http://target/webmention',
data={
'source': 'http://localhost/render?source=http%3A%2F%2Fthis%2Fmention&target=http%3A%2F%2Ftarget%2F',
'target': 'http://target/',
},
allow_redirects=False, timeout=15, stream=True,
headers=expected_headers)
resp = Response.get_by_id('http://this/mention http://target/')
self.assertEqual('in', resp.direction)
self.assertEqual('activitypub', resp.protocol)
self.assertEqual('complete', resp.status)
self.assertEqual(common.redirect_unwrap(as2), json_loads(resp.source_as2))
def test_proxy_url(self):
resp = Response.get_or_create('abc', 'xyz')
self.assertIsNone(resp.proxy_url())
resp.source_as2 = 'as2'
self.assertEquals('http://localhost/render?source=abc&target=xyz',
resp.proxy_url())
def test_activitypub_follow(self, mock_get, mock_post):
mock_get.side_effect = [self.follow, self.actor]
mock_post.return_value = requests_response('abc xyz')
got = app.get_response('/webmention',
method='POST',
body=urllib.urlencode({
'source':
'http://a/follow',
'target':
'https://fed.brid.gy/',
}))
self.assertEquals(200, got.status_int)
mock_get.assert_has_calls((
self.req('http://a/follow'),
self.req('http://followee', headers=CONNEG_HEADERS_AS2_HTML),
))
args, kwargs = mock_post.call_args
self.assertEqual(('https://foo.com/inbox', ), args)
self.assertEqual(self.follow_as2, kwargs['json'])
headers = kwargs['headers']
self.assertEqual(CONTENT_TYPE_AS2, headers['Content-Type'])
rsa_key = kwargs['auth'].header_signer._rsa._key
self.assertEqual(self.key.private_pem(), rsa_key.exportKey())
resp = Response.get_by_id('http://a/follow http://followee')
self.assertEqual('out', resp.direction)
self.assertEqual('activitypub', resp.protocol)
self.assertEqual('complete', resp.status)
self.assertEqual(self.follow_mf2, json.loads(resp.source_mf2))
def test_activitypub_create_reply(self, mock_get, mock_post):
mock_get.side_effect = self.activitypub_gets
mock_post.return_value = requests_response('abc xyz', status=203)
got = application.get_response('/webmention',
method='POST',
body=urlencode({
'source':
'http://a/reply',
'target':
'https://fed.brid.gy/',
}).encode())
self.assertEqual(203, got.status_int)
mock_get.assert_has_calls((
self.req('http://a/reply'),
self.req('http://orig/post', headers=CONNEG_HEADERS_AS2_HTML),
self.req('http://orig/author', headers=CONNEG_HEADERS_AS2_HTML),
))
args, kwargs = mock_post.call_args
self.assertEqual(('https://foo.com/inbox', ), args)
self.assertEqual(self.as2_create, kwargs['json'])
headers = kwargs['headers']
self.assertEqual(CONTENT_TYPE_AS2, headers['Content-Type'])
rsa_key = kwargs['auth'].header_signer._rsa._key
self.assertEqual(self.key.private_pem(), rsa_key.exportKey())
resp = Response.get_by_id('http://a/reply http://orig/as2')
self.assertEqual('out', resp.direction)
self.assertEqual('activitypub', resp.protocol)
self.assertEqual('complete', resp.status)
self.assertEqual(self.reply_mf2, json_loads(resp.source_mf2))
def test_render_atom(self):
Response(id='abc xyz', source_atom=self.atom).put()
resp = application.get_response('/render?source=abc&target=xyz')
self.assertEqual(200, resp.status_int)
self.assert_multiline_equals(self.html,
resp.body.decode(),
ignore_blanks=True)
def try_salmon(self):
"""Returns True if we attempted OStatus delivery. Raises otherwise."""
target = None
if self.target_resp:
target = self.target_resp.url
else:
targets = self._targets()
if targets:
target = targets[0]
if not target:
logging.warning("No targets or followers. Ignoring.")
return False
resp = Response.get_or_create(source=self.source_url,
target=target,
direction='out',
source_mf2=json_dumps(self.source_mf2))
resp.protocol = 'ostatus'
try:
ret = self._try_salmon(resp)
resp.status = 'complete'
return ret
except:
resp.status = 'error'
raise
finally:
resp.put()
def api_view_request(request_rid):
check_admin()
request = Request.find_by('where rid = ?', request_rid)
response = Response.find_by('where rid = ?', request_rid)
if request is None or response is None:
raise notfound()
return dict(request=content_escape(request), response=html_encode(response))
def test_render_mf2(self):
Response(id='abc xyz', source_mf2=json.dumps(self.mf2)).put()
resp = app.get_response('/render?source=abc&target=xyz')
self.assertEquals(200, resp.status_int)
self.assert_multiline_equals(self.html,
resp.body.decode('utf-8'),
ignore_blanks=True)
def test_reply(self, mock_urlopen, mock_head, mock_get, mock_post):
atom_reply = """\
<?xml version='1.0' encoding='UTF-8'?>
<entry xmlns='http://www.w3.org/2005/Atom'>
<id>https://my/reply</id>
<uri>https://my/reply</uri>
<author>
<name>Alice</name>
<uri>[email protected]</uri>
</author>
<thr:in-reply-to xmlns:thr="http://purl.org/syndication/thread/1.0">
http://orig/post
</thr:in-reply-to>
<content>I hereby reply.</content>
<title>My Reply</title>
<updated>%s</updated>
</entry>""" % datetime.datetime.now().isoformat('T')
self.send_slap(mock_urlopen, mock_head, mock_get, mock_post, atom_reply)
# check webmention post
mock_post.assert_called_once_with(
'http://orig/webmention',
data={'source': 'https://my/reply', 'target': 'http://orig/post'},
allow_redirects=False, timeout=15, stream=True,
headers=self.expected_headers)
# check stored response
resp = Response.get_by_id('https://my/reply http://orig/post')
self.assertEqual('in', resp.direction)
self.assertEqual('ostatus', resp.protocol)
self.assertEqual('complete', resp.status)
self.assertEqual(atom_reply, resp.source_atom)
def render():
"""Fetches a stored Response and renders it as HTML."""
source = flask_util.get_required_param('source')
target = flask_util.get_required_param('target')
id = f'{source} {target}'
resp = Response.get_by_id(id)
if not resp:
error(f'No stored response for {id}', status=404)
if resp.source_mf2:
as1 = microformats2.json_to_object(json_loads(resp.source_mf2))
elif resp.source_as2:
as1 = as2.to_as1(json_loads(resp.source_as2))
elif resp.source_atom:
as1 = atom.atom_to_activity(resp.source_atom)
else:
error(f'Stored response for {id} has no data', status=404)
# add HTML meta redirect to source page. should trigger for end users in
# browsers but not for webmention receivers (hopefully).
html = microformats2.activities_to_html([as1])
utf8 = '<meta charset="utf-8">'
refresh = f'<meta http-equiv="refresh" content="0;url={source}">'
return html.replace(utf8, utf8 + '\n' + refresh)
def test_inbox_like(self, mock_head, mock_get, mock_post):
mock_head.return_value = requests_response(url='http://orig/post')
mock_get.side_effect = [
# source actor
requests_response(LIKE_WITH_ACTOR['actor'], headers={'Content-Type': common.CONTENT_TYPE_AS2}),
# target post webmention discovery
requests_response(
'<html><head><link rel="webmention" href="/webmention"></html>'),
]
mock_post.return_value = requests_response()
got = self.client.post('/foo.com/inbox', json=LIKE)
self.assertEqual(200, got.status_code)
as2_headers = copy.deepcopy(common.HEADERS)
as2_headers.update(common.CONNEG_HEADERS_AS2_HTML)
mock_get.assert_has_calls((
call('http://orig/actor', headers=as2_headers, stream=True, timeout=15),
call('http://orig/post', headers=common.HEADERS, stream=True, timeout=15),
))
args, kwargs = mock_post.call_args
self.assertEqual(('http://orig/webmention',), args)
self.assertEqual({
# TODO
'source': 'http://localhost/render?source=http%3A%2F%2Fthis%2Flike__ok&target=http%3A%2F%2Forig%2Fpost',
'target': 'http://orig/post',
}, kwargs['data'])
resp = Response.get_by_id('http://this/like__ok http://orig/post')
self.assertEqual('in', resp.direction)
self.assertEqual('activitypub', resp.protocol)
self.assertEqual('complete', resp.status)
self.assertEqual(LIKE_WITH_ACTOR, json_loads(resp.source_as2))
def join_quiz(quiz_id, user_id=None):
## Get user from request
if request:
user = User.query.get(request.form['user_id'])
else:
user = User.query.get(user_id)
quiz = Quiz.query.get(quiz_id)
if Timer.query.filter(Timer.quiz_id == quiz_id).filter(
Timer.user_id == user.id).first() is None:
timer = Timer(quiz_id=quiz_id,
user_id=user.id,
last_answered=datetime.now())
db.session.add(timer)
if Response.query.filter(Response.quiz_id == quiz_id).filter(
Response.user_id == user.id).first() is None:
## Create responses for user
for index in range(0, 10):
questions = json.loads(quiz.questions)
response = Response(question_id=questions[index],
user_id=user.id,
quiz_id=quiz_id)
db.session.add(response)
## Store responses for user
db.session.commit()
status_dict = dict()
status_dict['status'] = 'JOINED'
status_dict['quiz_id'] = quiz_id
return jsonify(status_dict)
def api_view_request(request_rid):
check_admin()
request = Request.find_by('where rid = ?', request_rid)
response = Response.find_by('where rid = ?', request_rid)
if request is None or response is None:
raise notfound()
return dict(request=content_escape(request), response=html_encode(response))
def _test_inbox_mention(self, as2, mock_head, mock_get, mock_post):
mock_head.return_value = requests_response(url='http://target')
mock_get.return_value = requests_response(
'<html><head><link rel="webmention" href="/webmention"></html>')
mock_post.return_value = requests_response()
got = app.get_response('/foo.com/inbox',
method='POST',
body=json.dumps(as2))
self.assertEquals(200, got.status_int, got.body)
mock_get.assert_called_once_with('http://target/',
headers=common.HEADERS,
verify=False)
expected_headers = copy.deepcopy(common.HEADERS)
expected_headers['Accept'] = '*/*'
mock_post.assert_called_once_with(
'http://target/webmention',
data={
'source':
'http://localhost/render?source=http%3A%2F%2Fthis%2Fmention&target=http%3A%2F%2Ftarget%2F',
'target': 'http://target/',
},
allow_redirects=False,
headers=expected_headers,
verify=False)
resp = Response.get_by_id('http://this/mention http://target/')
self.assertEqual('in', resp.direction)
self.assertEqual('activitypub', resp.protocol)
self.assertEqual('complete', resp.status)
self.assertEqual(common.redirect_unwrap(as2),
json.loads(resp.source_as2))
def test_like(self, mock_urlopen, mock_head, mock_get, mock_post):
atom_like = """\
<?xml version='1.0' encoding='UTF-8'?>
<entry xmlns='http://www.w3.org/2005/Atom'
xmlns:activity='http://activitystrea.ms/spec/1.0/'>
<uri>https://my/like</uri>
<author>
<name>Alice</name>
<uri>[email protected]</uri>
</author>
<activity:verb>http://activitystrea.ms/schema/1.0/like</activity:verb>
<activity:object>http://orig/post</activity:object>
<updated>%s</updated>
</entry>""" % datetime.datetime.now().isoformat('T')
self.send_slap(mock_urlopen, mock_head, mock_get, mock_post, atom_like)
# check webmention post
mock_post.assert_called_once_with(
'http://orig/webmention',
data={
'source':
'http://localhost/render?source=https%3A%2F%2Fmy%2Flike&target=http%3A%2F%2Forig%2Fpost',
'target': 'http://orig/post',
},
allow_redirects=False,
headers=self.expected_headers,
verify=False)
# check stored response
resp = Response.get_by_id('https://my/like http://orig/post')
self.assertEqual('in', resp.direction)
self.assertEqual('ostatus', resp.protocol)
self.assertEqual('complete', resp.status)
self.assertEqual(atom_like, resp.source_atom)
def get(self):
qId = int(self.request.get('id'))
question = Question.get_by_id(qId)
responses = Response.all()
responses.filter("question =", question)
values = {'responses': responses, 'question': question}
self.response.out.write(template.render('templates/responses.html', values))
def get(self):
qId = int(self.request.get('id'))
responseSet = ResponseSet.get_by_id(qId)
responses = Response.all()
responses.filter('responseSet =', responseSet)
values = {'responseSet': responseSet, 'responses': responses}
self.response.out.write(template.render('templates/responseSet.html', values))
def refetch_hfeed(self, source):
"""refetch and reprocess the author's url, looking for
new or updated syndication urls that we may have missed the first
time we looked for them.
"""
logging.debug('refetching h-feed for source %s', source.label())
relationships = original_post_discovery.refetch(source)
if not relationships:
return
logging.debug('refetch h-feed found %d new rel=syndication relationships',
len(relationships))
# grab the Responses and see if any of them have a a syndication
# url matching one of the newly discovered relationships. We'll
# check each response until we've seen all of them or until
# the 60s timer runs out.
# TODO maybe add a (canonicalized) url field to Response so we can
# query by it instead of iterating over all of them
for response in (Response.query(Response.source == source.key)
.order(-Response.created)):
if response.activity_json: # handle old entities
response.activities_json.append(response.activity_json)
response.activity_json = None
new_orig_urls = set()
for activity_json in response.activities_json:
activity = json.loads(activity_json)
activity_url = activity.get('url') or activity.get('object', {}).get('url')
if not activity_url:
logging.warning('activity has no url %s', activity_json)
continue
activity_url = source.canonicalize_syndication_url(activity_url)
# look for activity url in the newly discovered list of relationships
for relationship in relationships.get(activity_url, []):
# won't re-propagate if the discovered link is already among
# these well-known upstream duplicates
if relationship.original in response.sent:
logging.info(
'%s found a new rel=syndication link %s -> %s, but the '
'relationship had already been discovered by another method',
response.label(), relationship.original,
relationship.syndication)
else:
logging.info(
'%s found a new rel=syndication link %s -> %s, and '
'will be repropagated with a new target!',
response.label(), relationship.original,
relationship.syndication)
new_orig_urls.add(relationship.original)
if new_orig_urls:
# re-open a previously 'complete' propagate task
response.status = 'new'
response.unsent.extend(list(new_orig_urls))
response.put()
response.add_task()
def test_get_type(self):
self.assertEqual('repost', Response.get_type(
{'objectType': 'activity', 'verb': 'share'}))
self.assertEqual('rsvp', Response.get_type({'verb': 'rsvp-no'}))
self.assertEqual('rsvp', Response.get_type({'verb': 'invite'}))
self.assertEqual('comment', Response.get_type({'objectType': 'comment'}))
self.assertEqual('post', Response.get_type({'verb': 'post'}))
self.assertEqual('post', Response.get_type({'objectType': 'event'}))
self.assertEqual('post', Response.get_type({'objectType': 'image'}))
self.assertEqual('comment', Response.get_type({
'objectType': 'note',
'context': {'inReplyTo': {'foo': 'bar'}},
}))
self.assertEqual('comment', Response.get_type({
'objectType': 'comment',
'verb': 'post',
}))
def add_response():
username = request.form.get('username')
username = username or 'anonymous'
board_id = request.form.get('board_id')
items = request.form.getlist('item')
# print 'Items: {} // {}'.format(type(items), items)
items = [int(x) for x in items if x]
# print 'Items: {} // {}'.format(type(items), items)
response = Response(username=username, board_id=int(board_id),
items=items)
saved = response.save()
# print 'saved response? : {}'.format(saved)
if saved == True:
flash('Response saved. Thanks {}.'.format(username))
else:
flash('Could not save response')
return redirect(url_for('response', response_id=response.id))
def repropagate_old_responses(self, source, relationships):
"""Find old Responses that match a new SyndicatedPost and repropagate them.
We look through as many responses as we can until the datastore query expires.
Args:
source: :class:`models.Source`
relationships: refetch result
"""
for response in Response.query(Response.source == source.key).order(-Response.updated):
new_orig_urls = set()
for activity_json in response.activities_json:
activity = json.loads(activity_json)
activity_url = activity.get("url") or activity.get("object", {}).get("url")
if not activity_url:
logging.warning("activity has no url %s", activity_json)
continue
activity_url = source.canonicalize_url(activity_url, activity=activity)
if not activity_url:
continue
# look for activity url in the newly discovered list of relationships
for relationship in relationships.get(activity_url, []):
# won't re-propagate if the discovered link is already among
# these well-known upstream duplicates
if relationship.original in response.sent or relationship.original in response.original_posts:
logging.info(
"%s found a new rel=syndication link %s -> %s, but the "
"relationship had already been discovered by another method",
response.label(),
relationship.original,
relationship.syndication,
)
else:
logging.info(
"%s found a new rel=syndication link %s -> %s, and "
"will be repropagated with a new target!",
response.label(),
relationship.original,
relationship.syndication,
)
new_orig_urls.add(relationship.original)
if new_orig_urls:
# re-open a previously 'complete' propagate task
response.status = "new"
response.unsent.extend(list(new_orig_urls))
response.put()
response.add_task()
def get(self):
id = int(self.request.get('id'))
responseSet = ResponseSet.get_by_id(id)
#First, need to delete all the responses that are associated with the response set that I'm deleting
responses = Response.all()
responses.filter('responseSet =', responseSet)
for response in responses:
response.delete()
responseSet.delete()
self.redirect('/qadmin')
def post(self, request):
if request.POST.get('response', False):
response = Response()
question = Question.objects.get(id=request.POST['question'])
response.message = request.POST['response']
response.question = question
response.user = request.user
response.save()
return HttpResponseRedirect(request.META['HTTP_REFERER'])
def repropagate_old_responses(self, source, relationships):
"""Find old Responses that match a new SyndicatedPost and repropagate them.
We look through as many responses as we can until the datastore query expires.
"""
for response in (Response.query(Response.source == source.key)
.order(-Response.updated)):
if response.activity_json: # handle old entities
response.activities_json.append(response.activity_json)
response.activity_json = None
new_orig_urls = set()
for activity_json in response.activities_json:
activity = json.loads(activity_json)
activity_url = activity.get('url') or activity.get('object', {}).get('url')
if not activity_url:
logging.warning('activity has no url %s', activity_json)
continue
activity_url = source.canonicalize_syndication_url(activity_url,
activity=activity)
# look for activity url in the newly discovered list of relationships
for relationship in relationships.get(activity_url, []):
# won't re-propagate if the discovered link is already among
# these well-known upstream duplicates
if (relationship.original in response.sent or
relationship.original in response.original_posts):
logging.info(
'%s found a new rel=syndication link %s -> %s, but the '
'relationship had already been discovered by another method',
response.label(), relationship.original,
relationship.syndication)
else:
logging.info(
'%s found a new rel=syndication link %s -> %s, and '
'will be repropagated with a new target!',
response.label(), relationship.original,
relationship.syndication)
new_orig_urls.add(relationship.original)
if new_orig_urls:
# re-open a previously 'complete' propagate task
response.status = 'new'
response.unsent.extend(list(new_orig_urls))
response.put()
response.add_task()
def test_get_or_save(self):
response = self.responses[0]
self.assertEqual(0, Response.query().count())
self.assert_no_propagate_task()
# new. should add a propagate task.
saved = response.get_or_save(self.sources[0])
self.assertEqual(response.key, saved.key)
self.assertEqual(response.source, saved.source)
self.assertEqual('comment', saved.type)
self.assertEqual([], saved.old_response_jsons)
self.assert_propagate_task()
# existing. no new task.
same = saved.get_or_save(self.sources[0])
self.assert_entities_equal(saved, same)
self.assert_no_propagate_task()
def template_vars(self):
responses = []
# Find the most recently propagated responses with error URLs
for r in Response.query().order(-Response.updated):
if (len(responses) >= self.NUM_RESPONSES or
r.updated < datetime.datetime.now() - datetime.timedelta(hours=1)):
break
elif not r.error or r.status == 'complete':
continue
# r.source = r.source.get()
r.links = [util.pretty_link(u, new_tab=True) for u in r.error + r.failed]
r.response = json.loads(r.response_json)
r.activities = [json.loads(a) for a in r.activities_json]
responses.append(r)
responses.sort(key=lambda r: (r.source, r.activities, r.response))
return {'responses': responses}
def test_get_or_save(self):
self.sources[0].put()
response = self.responses[0]
self.assertEqual(0, Response.query().count())
self.assertEqual(0, len(self.taskqueue_stub.GetTasks('propagate')))
# new. should add a propagate task.
saved = response.get_or_save()
self.assertEqual(response.key, saved.key)
self.assertEqual(response.source, saved.source)
self.assertEqual('comment', saved.type)
tasks = self.taskqueue_stub.GetTasks('propagate')
self.assertEqual(1, len(tasks))
self.assertEqual(response.key.urlsafe(),
testutil.get_task_params(tasks[0])['response_key'])
self.assertEqual('/_ah/queue/propagate', tasks[0]['url'])
# existing. no new task.
same = saved.get_or_save()
self.assertEqual(saved.source, same.source)
self.assertEqual(1, len(tasks))
def template_vars(self):
if not self.source:
return {}
vars = super(UserHandler, self).template_vars()
vars.update({
'source': self.source,
'epoch': util.EPOCH,
})
# Blog webmention promos
if 'webmention' not in self.source.features:
if self.source.SHORT_NAME in ('blogger', 'tumblr', 'wordpress'):
vars[self.source.SHORT_NAME + '_promo'] = True
else:
for domain in self.source.domains:
if ('.blogspot.' in domain and # Blogger uses country TLDs
not Blogger.query(Blogger.domains == domain).get()):
vars['blogger_promo'] = True
elif (domain.endswith('tumblr.com') and
not Tumblr.query(Tumblr.domains == domain).get()):
vars['tumblr_promo'] = True
elif (domain.endswith('wordpress.com') and
not WordPress.query(WordPress.domains == domain).get()):
vars['wordpress_promo'] = True
# Responses
if 'listen' in self.source.features:
vars['responses'] = []
for i, r in enumerate(Response.query()
.filter(Response.source == self.source.key)\
.order(-Response.updated)):
r.response = json.loads(r.response_json)
if r.activity_json: # handle old entities
r.activities_json.append(r.activity_json)
r.activities = [json.loads(a) for a in r.activities_json]
if (not gr_source.Source.is_public(r.response) or
not all(gr_source.Source.is_public(a) for a in r.activities)):
continue
r.actor = r.response.get('author') or r.response.get('actor', {})
if not r.response.get('content'):
phrases = {
'like': 'liked this',
'repost': 'reposted this',
'rsvp-yes': 'is attending',
'rsvp-no': 'is not attending',
'rsvp-maybe': 'might attend',
'invite': 'is invited',
}
r.response['content'] = '%s %s.' % (
r.actor.get('displayName') or '',
phrases.get(r.type) or phrases.get(r.response.get('verb')))
# convert image URL to https if we're serving over SSL
image_url = r.actor.setdefault('image', {}).get('url')
if image_url:
r.actor['image']['url'] = util.update_scheme(image_url, self)
# generate original post links
r.links = self.process_webmention_links(r)
vars['responses'].append(r)
if len(vars['responses']) >= 10 or i > 200:
break
# Publishes
if 'publish' in self.source.features:
publishes = Publish.query().filter(Publish.source == self.source.key)\
.order(-Publish.updated)\
.fetch(10)
for p in publishes:
p.pretty_page = util.pretty_link(
p.key.parent().id(), a_class='original-post', new_tab=True)
vars['publishes'] = publishes
if 'webmention' in self.source.features:
# Blog posts
blogposts = BlogPost.query().filter(BlogPost.source == self.source.key)\
.order(-BlogPost.created)\
.fetch(10)
for b in blogposts:
b.links = self.process_webmention_links(b)
try:
text = b.feed_item.get('title')
except ValueError:
text = None
b.pretty_url = util.pretty_link(b.key.id(), text=text,
a_class='original-post', max_length=40,
new_tab=True)
# Blog webmentions
webmentions = BlogWebmention.query()\
.filter(BlogWebmention.source == self.source.key)\
.order(-BlogWebmention.updated)\
.fetch(10)
for w in webmentions:
w.pretty_source = util.pretty_link(w.source_url(), a_class='original-post',
new_tab=True)
try:
target_is_source = (urlparse.urlparse(w.target_url()).netloc in
self.source.domains)
except BaseException:
target_is_source = False
w.pretty_target = util.pretty_link(w.target_url(), a_class='original-post',
new_tab=True, keep_host=target_is_source)
vars.update({'blogposts': blogposts, 'webmentions': webmentions})
return vars
import instagram
import tumblr
import twitter
import wordpress_rest
domains = collections.defaultdict(int) # maps domain to # of users
for cls in models.sources.values():
for src in cls.query(cls.domains > ''):
for domain in src.domains:
print domain
domains[domain] += 1
with open('domains.txt', 'w') as f:
f.write('domain,num_users\n')
f.write('\n'.join(str(item) for item in reversed(sorted(
'%s,%s' % (item[1], item[0]) for item in domains.items()))))
with open('domains_sent.txt', 'w') as f:
url = ''
while True:
resp = Response.query(Response.sent > url).get(projection=['sent'])
if not resp:
break
domain = None
for sent in resp.sent:
parsed = urlparse.urlparse(sent)
if sent > url and (domain is None or parsed.netloc < domain):
domain = parsed.netloc
url = urlparse.urlunparse(parsed[:2] + ('', '', '', '')) + chr(ord('/') + 1)
print domain
def template_vars(self):
vars = super(UserHandler, self).template_vars()
vars.update({
'source': self.source,
'EPOCH': util.EPOCH,
'REFETCH_HFEED_TRIGGER': models.REFETCH_HFEED_TRIGGER,
'RECENT_PRIVATE_POSTS_THRESHOLD': RECENT_PRIVATE_POSTS_THRESHOLD,
})
if not self.source:
return vars
if isinstance(self.source, instagram.Instagram):
auth = self.source.auth_entity
vars['indieauth_me'] = (
auth.id if isinstance(auth, indieauth.IndieAuth)
else self.source.domain_urls[0] if self.source.domain_urls
else None)
# Blog webmention promos
if 'webmention' not in self.source.features:
if self.source.SHORT_NAME in ('blogger', 'tumblr', 'wordpress'):
vars[self.source.SHORT_NAME + '_promo'] = True
else:
for domain in self.source.domains:
if ('.blogspot.' in domain and # Blogger uses country TLDs
not Blogger.query(Blogger.domains == domain).get()):
vars['blogger_promo'] = True
elif (domain.endswith('tumblr.com') and
not Tumblr.query(Tumblr.domains == domain).get()):
vars['tumblr_promo'] = True
elif (domain.endswith('wordpress.com') and
not WordPress.query(WordPress.domains == domain).get()):
vars['wordpress_promo'] = True
# Responses
if 'listen' in self.source.features:
vars['responses'] = []
query = Response.query().filter(Response.source == self.source.key)
# if there's a paging param (responses_before or responses_after), update
# query with it
def get_paging_param(param):
val = self.request.get(param)
try:
return util.parse_iso8601(val) if val else None
except:
msg = "Couldn't parse %s %r as ISO8601" % (param, val)
logging.exception(msg)
self.abort(400, msg)
before = get_paging_param('responses_before')
after = get_paging_param('responses_after')
if before and after:
self.abort(400, "can't handle both responses_before and responses_after")
elif after:
query = query.filter(Response.updated > after).order(Response.updated)
elif before:
query = query.filter(Response.updated < before).order(-Response.updated)
else:
query = query.order(-Response.updated)
query_iter = query.iter()
for i, r in enumerate(query_iter):
r.response = json.loads(r.response_json)
r.activities = [json.loads(a) for a in r.activities_json]
if (not self.source.is_activity_public(r.response) or
not all(self.source.is_activity_public(a) for a in r.activities)):
continue
elif r.type == 'post':
r.activities = []
r.actor = r.response.get('author') or r.response.get('actor', {})
for a in r.activities + [r.response]:
if not a.get('content'):
a['content'] = a.get('object', {}).get('content')
if not r.response.get('content'):
phrases = {
'like': 'liked this',
'repost': 'reposted this',
'rsvp-yes': 'is attending',
'rsvp-no': 'is not attending',
'rsvp-maybe': 'might attend',
'rsvp-interested': 'is interested',
'invite': 'is invited',
}
r.response['content'] = '%s %s.' % (
r.actor.get('displayName') or '',
phrases.get(r.type) or phrases.get(r.response.get('verb')))
# convert image URL to https if we're serving over SSL
image_url = r.actor.setdefault('image', {}).get('url')
if image_url:
r.actor['image']['url'] = util.update_scheme(image_url, self)
# generate original post links
r.links = self.process_webmention_links(r)
r.original_links = [util.pretty_link(url, new_tab=True)
for url in r.original_posts]
vars['responses'].append(r)
if len(vars['responses']) >= 10 or i > 200:
break
vars['responses'].sort(key=lambda r: r.updated, reverse=True)
# calculate new paging param(s)
new_after = (
before if before else
vars['responses'][0].updated if
vars['responses'] and query_iter.probably_has_next() and (before or after)
else None)
if new_after:
vars['responses_after_link'] = ('?responses_after=%s#responses' %
new_after.isoformat())
new_before = (
after if after else
vars['responses'][-1].updated if
vars['responses'] and query_iter.probably_has_next()
else None)
if new_before:
vars['responses_before_link'] = ('?responses_before=%s#responses' %
new_before.isoformat())
vars['next_poll'] = max(
self.source.last_poll_attempt + self.source.poll_period(),
# lower bound is 1 minute from now
util.now_fn() + datetime.timedelta(seconds=90))
# Publishes
if 'publish' in self.source.features:
publishes = Publish.query().filter(Publish.source == self.source.key)\
.order(-Publish.updated)\
.fetch(10)
for p in publishes:
p.pretty_page = util.pretty_link(
p.key.parent().id(), attrs={'class': 'original-post u-url u-name'},
new_tab=True)
vars['publishes'] = publishes
if 'webmention' in self.source.features:
# Blog posts
blogposts = BlogPost.query().filter(BlogPost.source == self.source.key)\
.order(-BlogPost.created)\
.fetch(10)
for b in blogposts:
b.links = self.process_webmention_links(b)
try:
text = b.feed_item.get('title')
except ValueError:
text = None
b.pretty_url = util.pretty_link(
b.key.id(), text=text, attrs={'class': 'original-post u-url u-name'},
max_length=40, new_tab=True)
# Blog webmentions
webmentions = BlogWebmention.query()\
.filter(BlogWebmention.source == self.source.key)\
.order(-BlogWebmention.updated)\
.fetch(10)
for w in webmentions:
w.pretty_source = util.pretty_link(
w.source_url(), attrs={'class': 'original-post'}, new_tab=True)
try:
target_is_source = (urlparse.urlparse(w.target_url()).netloc in
self.source.domains)
except BaseException:
target_is_source = False
w.pretty_target = util.pretty_link(
w.target_url(), attrs={'class': 'original-post'}, new_tab=True,
keep_host=target_is_source)
vars.update({'blogposts': blogposts, 'webmentions': webmentions})
return vars
def test_get_type(self):
self.assertEqual('repost', Response.get_type(
{'objectType': 'activity', 'verb': 'share'}))
self.assertEqual('rsvp', Response.get_type({'verb': 'rsvp-no'}))
self.assertEqual('rsvp', Response.get_type({'verb': 'invite'}))
self.assertEqual('comment', Response.get_type({'objectType': 'other'}))
def poll(self, source):
"""Actually runs the poll.
Stores property names and values to update in source.updates.
"""
if source.last_activities_etag or source.last_activity_id:
logging.debug("Using ETag %s, last activity id %s", source.last_activities_etag, source.last_activity_id)
#
# Step 1: fetch activities:
# * posts by the user
# * search all posts for the user's domain URLs to find links
#
cache = util.CacheDict()
if source.last_activities_cache_json:
cache.update(json.loads(source.last_activities_cache_json))
# search for links first so that the user's activities and responses
# override them if they overlap
links = source.search_for_links()
# this user's own activities (and user mentions)
resp = source.get_activities_response(
fetch_replies=True,
fetch_likes=True,
fetch_shares=True,
fetch_mentions=True,
count=50,
etag=source.last_activities_etag,
min_id=source.last_activity_id,
cache=cache,
)
etag = resp.get("etag") # used later
user_activities = resp.get("items", [])
# these map ids to AS objects
responses = {a["id"]: a for a in links}
activities = {a["id"]: a for a in links + user_activities}
# extract silo activity ids, update last_activity_id
silo_activity_ids = set()
last_activity_id = source.last_activity_id
for id, activity in activities.items():
# maybe replace stored last activity id
parsed = util.parse_tag_uri(id)
if parsed:
id = parsed[1]
silo_activity_ids.add(id)
try:
# try numeric comparison first
greater = int(id) > int(last_activity_id)
except (TypeError, ValueError):
greater = id > last_activity_id
if greater:
last_activity_id = id
if last_activity_id and last_activity_id != source.last_activity_id:
source.updates["last_activity_id"] = last_activity_id
# trim cache to just the returned activity ids, so that it doesn't grow
# without bound. (WARNING: depends on get_activities_response()'s cache key
# format, e.g. 'PREFIX ACTIVITY_ID'!)
source.updates["last_activities_cache_json"] = json.dumps(
{k: v for k, v in cache.items() if k.split()[-1] in silo_activity_ids}
)
# Cache to make sure we only fetch the author's h-feed(s) the
# first time we see it
fetched_hfeeds = set()
# narrow down to just public activities
public = {}
private = {}
for id, activity in activities.items():
(public if source.is_activity_public(activity) else private)[id] = activity
logging.info("Found %d public activities: %s", len(public), public.keys())
logging.info("Found %d private activities: %s", len(private), private.keys())
last_public_post = (source.last_public_post or util.EPOCH).isoformat()
public_published = util.trim_nulls([a.get("published") for a in public.values()])
if public_published:
max_published = max(public_published)
if max_published > last_public_post:
last_public_post = max_published
source.updates["last_public_post"] = util.as_utc(util.parse_iso8601(max_published))
source.updates["recent_private_posts"] = len(
[a for a in private.values() if a.get("published", util.EPOCH_ISO) > last_public_post]
)
#
# Step 2: extract responses, store their activities in response['activities']
#
# WARNING: this creates circular references in link posts found by search
# queries in step 1, since they are their own activity. We use
# prune_activity() and prune_response() in step 4 to remove these before
# serializing to JSON.
#
for id, activity in public.items():
obj = activity.get("object") or activity
# handle user mentions
user_id = source.user_tag_id()
if obj.get("author", {}).get("id") != user_id:
for tag in obj.get("tags", []):
urls = tag.get("urls")
if tag.get("objectType") == "person" and tag.get("id") == user_id and urls:
activity["originals"], activity["mentions"] = original_post_discovery.discover(
source,
activity,
fetch_hfeed=True,
include_redirect_sources=False,
already_fetched_hfeeds=fetched_hfeeds,
)
activity["mentions"].update(u.get("value") for u in urls)
responses[id] = activity
break
# handle quote mentions
for att in obj.get("attachments", []):
if (
att.get("objectType") in ("note", "article")
and att.get("author", {}).get("id") == source.user_tag_id()
):
# now that we've confirmed that one exists, OPD will dig
# into the actual attachments
if "originals" not in activity or "mentions" not in activity:
activity["originals"], activity["mentions"] = original_post_discovery.discover(
source,
activity,
fetch_hfeed=True,
include_redirect_sources=False,
already_fetched_hfeeds=fetched_hfeeds,
)
responses[id] = activity
break
# extract replies, likes, reactions, reposts, and rsvps
replies = obj.get("replies", {}).get("items", [])
tags = obj.get("tags", [])
likes = [t for t in tags if Response.get_type(t) == "like"]
reactions = [t for t in tags if Response.get_type(t) == "react"]
reposts = [t for t in tags if Response.get_type(t) == "repost"]
rsvps = Source.get_rsvps_from_event(obj)
# coalesce responses. drop any without ids
for resp in replies + likes + reactions + reposts + rsvps:
id = resp.get("id")
if not id:
logging.error("Skipping response without id: %s", json.dumps(resp, indent=2))
continue
resp.setdefault("activities", []).append(activity)
# when we find two responses with the same id, the earlier one may have
# come from a link post or user mention, and this one is probably better
# since it probably came from the user's activity, so prefer this one.
# background: https://github.com/snarfed/bridgy/issues/533
existing = responses.get(id)
if existing:
if source.gr_source.activity_changed(resp, existing, log=True):
logging.warning("Got two different versions of same response!\n%s\n%s", existing, resp)
resp["activities"].extend(existing.get("activities", []))
responses[id] = resp
#
# Step 3: filter out responses we've already seen
#
# seen responses (JSON objects) for each source are stored in its entity.
unchanged_responses = []
if source.seen_responses_cache_json:
for seen in json.loads(source.seen_responses_cache_json):
id = seen["id"]
resp = responses.get(id)
if resp and not source.gr_source.activity_changed(seen, resp, log=True):
unchanged_responses.append(seen)
del responses[id]
#
# Step 4: store new responses and enqueue propagate tasks
#
pruned_responses = []
for id, resp in responses.items():
resp_type = Response.get_type(resp)
activities = resp.pop("activities", [])
if not activities and resp_type == "post":
activities = [resp]
too_long = set()
urls_to_activity = {}
for i, activity in enumerate(activities):
# we'll usually have multiple responses for the same activity, and the
# objects in resp['activities'] are shared, so cache each activity's
# discovered webmention targets inside its object.
if "originals" not in activity or "mentions" not in activity:
activity["originals"], activity["mentions"] = original_post_discovery.discover(
source,
activity,
fetch_hfeed=True,
include_redirect_sources=False,
already_fetched_hfeeds=fetched_hfeeds,
)
targets = original_post_discovery.targets_for_response(
resp, originals=activity["originals"], mentions=activity["mentions"]
)
if targets:
logging.info(
"%s has %d webmention target(s): %s", activity.get("url"), len(targets), " ".join(targets)
)
for t in targets:
if len(t) <= _MAX_STRING_LENGTH:
urls_to_activity[t] = i
else:
logging.warning("Giving up on target URL over %s chars! %s", _MAX_STRING_LENGTH, t)
too_long.add(t[: _MAX_STRING_LENGTH - 4] + "...")
# store/update response entity. the prune_*() calls are important to
# remove circular references in link responses, which are their own
# activities. details in the step 2 comment above.
pruned_response = util.prune_response(resp)
pruned_responses.append(pruned_response)
resp_entity = Response(
id=id,
source=source.key,
activities_json=[json.dumps(util.prune_activity(a, source)) for a in activities],
response_json=json.dumps(pruned_response),
type=resp_type,
unsent=list(urls_to_activity.keys()),
failed=list(too_long),
original_posts=resp.get("originals", []),
)
if urls_to_activity and len(activities) > 1:
resp_entity.urls_to_activity = json.dumps(urls_to_activity)
resp_entity.get_or_save(source)
# update cache
if pruned_responses:
source.updates["seen_responses_cache_json"] = json.dumps(pruned_responses + unchanged_responses)
source.updates.update({"last_polled": source.last_poll_attempt, "poll_status": "ok"})
if etag and etag != source.last_activities_etag:
source.updates["last_activities_etag"] = etag
#
# Step 5. possibly refetch updated syndication urls
#
# if the author has added syndication urls since the first time
# original_post_discovery ran, we'll miss them. this cleanup task will
# periodically check for updated urls. only kicks in if the author has
# *ever* published a rel=syndication url
if source.should_refetch():
logging.info("refetching h-feed for source %s", source.label())
relationships = original_post_discovery.refetch(source)
now = util.now_fn()
source.updates["last_hfeed_refetch"] = now
if relationships:
logging.info("refetch h-feed found new rel=syndication relationships: %s", relationships)
try:
self.repropagate_old_responses(source, relationships)
except BaseException, e:
if isinstance(
e, (datastore_errors.BadRequestError, datastore_errors.Timeout)
) or util.is_connection_failure(e):
logging.info("Timeout while repropagating responses.", exc_info=True)
else:
raise
def backfeed(self, source, responses=None, activities=None):
"""Processes responses and activities and generates propagate tasks.
Stores property names and values to update in source.updates.
Args:
source: Source
responses: dict mapping AS response id to AS object
activities: dict mapping AS activity id to AS object
"""
if responses is None:
responses = {}
if activities is None:
activities = {}
# Cache to make sure we only fetch the author's h-feed(s) the
# first time we see it
fetched_hfeeds = set()
# narrow down to just public activities
public = {}
private = {}
for id, activity in activities.items():
(public if source.is_activity_public(activity) else private)[id] = activity
logging.info('Found %d public activities: %s', len(public), public.keys())
logging.info('Found %d private activities: %s', len(private), private.keys())
last_public_post = (source.last_public_post or util.EPOCH).isoformat()
public_published = util.trim_nulls([a.get('published') for a in public.values()])
if public_published:
max_published = max(public_published)
if max_published > last_public_post:
last_public_post = max_published
source.updates['last_public_post'] = \
util.as_utc(util.parse_iso8601(max_published))
source.updates['recent_private_posts'] = \
len([a for a in private.values()
if a.get('published', util.EPOCH_ISO) > last_public_post])
#
# Step 2: extract responses, store their activities in response['activities']
#
# WARNING: this creates circular references in link posts found by search
# queries in step 1, since they are their own activity. We use
# prune_activity() and prune_response() in step 4 to remove these before
# serializing to JSON.
#
for id, activity in public.items():
obj = activity.get('object') or activity
# handle user mentions
user_id = source.user_tag_id()
if obj.get('author', {}).get('id') != user_id:
for tag in obj.get('tags', []):
urls = tag.get('urls')
if tag.get('objectType') == 'person' and tag.get('id') == user_id and urls:
activity['originals'], activity['mentions'] = \
original_post_discovery.discover(
source, activity, fetch_hfeed=True,
include_redirect_sources=False,
already_fetched_hfeeds=fetched_hfeeds)
activity['mentions'].update(u.get('value') for u in urls)
responses[id] = activity
break
# handle quote mentions
for att in obj.get('attachments', []):
if (att.get('objectType') in ('note', 'article')
and att.get('author', {}).get('id') == source.user_tag_id()):
# now that we've confirmed that one exists, OPD will dig
# into the actual attachments
if 'originals' not in activity or 'mentions' not in activity:
activity['originals'], activity['mentions'] = \
original_post_discovery.discover(
source, activity, fetch_hfeed=True,
include_redirect_sources=False,
already_fetched_hfeeds=fetched_hfeeds)
responses[id] = activity
break
# extract replies, likes, reactions, reposts, and rsvps
replies = obj.get('replies', {}).get('items', [])
tags = obj.get('tags', [])
likes = [t for t in tags if Response.get_type(t) == 'like']
reactions = [t for t in tags if Response.get_type(t) == 'react']
reposts = [t for t in tags if Response.get_type(t) == 'repost']
rsvps = Source.get_rsvps_from_event(obj)
# coalesce responses. drop any without ids
for resp in replies + likes + reactions + reposts + rsvps:
id = resp.get('id')
if not id:
logging.error('Skipping response without id: %s', json.dumps(resp, indent=2))
continue
if source.is_blocked(resp):
logging.info('Skipping response by blocked user: %s',
json.dumps(resp.get('author') or resp.get('actor'), indent=2))
continue
resp.setdefault('activities', []).append(activity)
# when we find two responses with the same id, the earlier one may have
# come from a link post or user mention, and this one is probably better
# since it probably came from the user's activity, so prefer this one.
# background: https://github.com/snarfed/bridgy/issues/533
existing = responses.get(id)
if existing:
if source.gr_source.activity_changed(resp, existing, log=True):
logging.warning('Got two different versions of same response!\n%s\n%s',
existing, resp)
resp['activities'].extend(existing.get('activities', []))
responses[id] = resp
#
# Step 3: filter out responses we've already seen
#
# seen responses (JSON objects) for each source are stored in its entity.
unchanged_responses = []
if source.seen_responses_cache_json:
for seen in json.loads(source.seen_responses_cache_json):
id = seen['id']
resp = responses.get(id)
if resp and not source.gr_source.activity_changed(seen, resp, log=True):
unchanged_responses.append(seen)
del responses[id]
#
# Step 4: store new responses and enqueue propagate tasks
#
pruned_responses = []
for id, resp in responses.items():
resp_type = Response.get_type(resp)
activities = resp.pop('activities', [])
if not activities and resp_type == 'post':
activities = [resp]
too_long = set()
urls_to_activity = {}
for i, activity in enumerate(activities):
# we'll usually have multiple responses for the same activity, and the
# objects in resp['activities'] are shared, so cache each activity's
# discovered webmention targets inside its object.
if 'originals' not in activity or 'mentions' not in activity:
activity['originals'], activity['mentions'] = \
original_post_discovery.discover(
source, activity, fetch_hfeed=True,
include_redirect_sources=False,
already_fetched_hfeeds=fetched_hfeeds)
targets = original_post_discovery.targets_for_response(
resp, originals=activity['originals'], mentions=activity['mentions'])
if targets:
logging.info('%s has %d webmention target(s): %s', activity.get('url'),
len(targets), ' '.join(targets))
for t in targets:
if len(t) <= _MAX_STRING_LENGTH:
urls_to_activity[t] = i
else:
logging.info('Giving up on target URL over %s chars! %s',
_MAX_STRING_LENGTH, t)
too_long.add(t[:_MAX_STRING_LENGTH - 4] + '...')
# store/update response entity. the prune_*() calls are important to
# remove circular references in link responses, which are their own
# activities. details in the step 2 comment above.
pruned_response = util.prune_response(resp)
pruned_responses.append(pruned_response)
resp_entity = Response(
id=id,
source=source.key,
activities_json=[json.dumps(util.prune_activity(a, source))
for a in activities],
response_json=json.dumps(pruned_response),
type=resp_type,
unsent=list(urls_to_activity.keys()),
failed=list(too_long),
original_posts=resp.get('originals', []))
if urls_to_activity and len(activities) > 1:
resp_entity.urls_to_activity=json.dumps(urls_to_activity)
resp_entity.get_or_save(source, restart=self.RESTART_EXISTING_TASKS)
# update cache
if pruned_responses:
source.updates['seen_responses_cache_json'] = json.dumps(
pruned_responses + unchanged_responses)
