def test_no_duplicate_links(self): """Make sure that a link found by both original-post-discovery and posse-post-discovery will not result in two webmentions being sent. """ source = self.sources[0] source.domain_urls = ['http://target1'] activity = self.activities[0] activity['object'].update({ 'content': 'with a backlink http://target1/post/url', 'url': 'https://fa.ke/post/url', }) original = 'http://target1/post/url' syndicated = 'https://fa.ke/post/url' self.expect_requests_get('http://target1', """ <html class="h-feed"> <div class="h-entry"> <a class="u-url" href="%s"></a> </div> </html>""" % original) self.expect_requests_get(original, """ <div class="h-entry"> <a class="u-url" href="%s"></a> <a class="u-syndication" href="%s"></a> </div>""" % (original, syndicated)) self.mox.ReplayAll() logging.debug('Original post discovery %s -> %s', source, activity) wmtargets = tasks.get_webmention_targets(source, activity) self.assertEquals([original], activity['object']['upstreamDuplicates']) self.assertEquals(set([original]), wmtargets)
def test_no_duplicate_links(self): """Make sure that a link found by both original-post-discovery and posse-post-discovery will not result in two webmentions being sent. """ self.activity['object']['content'] = 'with a link http://author/post/url' original = 'http://author/post/url' self.expect_requests_get('http://author', """ <html class="h-feed"> <div class="h-entry"> <a class="u-url" href="%s"></a> </div> </html>""" % original) self.expect_requests_get(original, """ <div class="h-entry"> <a class="u-url" href="%s"></a> <a class="u-syndication" href="%s"></a> </div>""" % (original, 'https://fa.ke/post/url')) self.mox.ReplayAll() logging.debug('Original post discovery %s -> %s', self.source, self.activity) wmtargets = tasks.get_webmention_targets(self.source, self.activity) self.assertEquals([original], self.activity['object']['upstreamDuplicates']) self.assertEquals([original], wmtargets)
def on_data(self, raw_data): try: # logging.debug('Received streaming message: %s...', raw_data[:100]) data = json.loads(raw_data) if data.get('event') == 'favorite': response = self.source.as_source.streaming_event_to_object(data) if not response: logging.warning('Discarding malformed favorite event: %s', raw_data) return True tweet = data.get('target_object') activity = self.source.as_source.tweet_to_activity(tweet) elif (data.get('retweeted_status', {}).get('user', {}).get('screen_name') == self.source.key.string_id()): response = self.source.as_source.retweet_to_object(data) activity = self.source.as_source.tweet_to_activity(data['retweeted_status']) # not handling replies right now. i wish i could, but we only get the # individual tweet that it's replying too, not the original root tweet that # started the chain, which is the one that will have the original post # links and should be used as the activity. # # worse, we store this as a response, even though it has the wrong # activity, and so when the poll task later finds it and has the correct # root tweet, it sees that the response has already been saved to the # datastore and propagated, so it drops the good one on the floor. # # sigh. oh well. # # elif ('in_reply_to_status_id_str' in data and # data.get('in_reply_to_screen_name') == self.source.key.string_id()): # response = self.source.as_source.tweet_to_object(data) # activity = self.source.as_source.get_activities( # activity_id=data['in_reply_to_status_id_str'])[0] else: # logging.debug("Discarding message we don't handle: %s", data) return True targets = tasks.get_webmention_targets(self.source, activity) models.Response(id=response['id'], source=self.source.key, activities_json=[json.dumps(activity)], response_json=json.dumps(response), type = models.Response.get_type(response), unsent=list(targets), ).get_or_save() except: logging.warning('Error processing message: %s', raw_data, exc_info=True) return True