def setUp(self):
"""Set up a mock patcher."""
self.patcher = mock.patch('requests.post', fake_requests_post)
self.patcher.start()
import diffbot
self.client = diffbot.Client(token=TOKEN)
self.client_v1 = diffbot.Client(token=TOKEN, version=1)
self.client_v2 = diffbot.Client(token=TOKEN, version=2)
def setUp(self):
"""Set up a mock patcher.
This will make the `requests` library unavailable in `diffbot`.
"""
self.import_hook = ImportHook('requests')
sys.meta_path.insert(0, self.import_hook)
try:
self.patcher = mock.patch('urllib2.urlopen', fake_urllib2_urlopen)
self.patcher.start()
except ImportError:
self.patcher = mock.patch('urllib.request.urlopen',
fake_urllib2_urlopen)
self.patcher.start()
import diffbot
diffbot = imp.reload(diffbot)
self.client = diffbot.Client(token=TOKEN)
from pymongo import MongoClient
import diffbot
from hackernews import HackerNews
import os
envHost = os.environ['MONGO_DB_HOST']
envPort = os.environ['MONGO_DB_PORT']
envDB = os.environ['MONGO_DB_DATABASE']
dbURL = 'mongodb://' + envHost + ':' + envPort + '/' + envDB
client = MongoClient(dbURL)
db = client.get_database()
hn = HackerNews()
diffbot_token = os.environ['DIFFBOT_TOKEN']
diffbot_client = diffbot.Client(token=diffbot_token)
links = []
# Get new links
for story_id in hn.top_stories(limit=1000):
item = hn.get_item(story_id)
url = item.url
# Check if link is already in database
if db.unrelatedlinks.find_one({'url': item.url}) is not None:
continue
try:
result = diffbot_client.api('article', url)
title = result["objects"][0]["title"]
text = result["objects"][0]["text"]
date = result["objects"][0]["date"]
tags = []
try: