http_client = AsyncHTTPClient()
loop = ioloop.IOLoop.instance()
@gen.engine
def fetch_data(url, callback):
try:
r = yield gen.Task(http_client.fetch, url)
print "done"
callback(Scrapper.Blog(url=url, content=r.body[:100]))
except:
print "Something went wrong"
callback(Scrapper.Blog())
class ScraperHandler(object):
def __init__(self):
pass
def scrape(self, url):
return gen.Task(fetch_data, url)
handler = ScraperHandler()
processor = Scrapper.Processor(handler)
pfactory = TJSONProtocol.TJSONProtocolFactory()
server = TAMQPTornadoServer(processor, pfactory, prefetch=10)
server.start()
loop.start()
