def fetchandsavepage(links, username):
database = Manager.getDatabaseInstance()
crawler = Manager.getCrawlerInstance()
database.getElseMakeUser(username)
print("Database Made")
#rdb.set_trace()
links = json.loads(links)
links = links['list'].values()
for link in links:
try:
if not crawler:
crawler = Manager.getCrawlerInstance()
else:
page = crawler.make_a_page(link['resolved_url'])
page.get_page_contents()
# need a method to generate tags
tags = ["jvc", "pocket"]
database.MakeWebPageNode(username, link['resolved_url'],
page.title, page.content, tags)
print("Hardware Done")
print(page.url)
#print self.page.content
print(page.title)
except:
print("Exception occured")
continue
print("Fetching Articles code here")
def __init__(self, redirect_fn):
self.index = "http://localhost:8000/index"
self.username = None
self.database = Manager.getDatabaseInstance()
self.redirect_url = "http://localhost:9000/intermediate"
self.data = {
'consumer_key': '12160-a5732aa14bd49ef07c5a3628',
"redirect_uri": self.redirect_url
}
self.headers = {
'Content-Type': 'application/json; charset=UTF-8',
'X-Accept': 'application/json',
}
self.redirect_fn = redirect_fn
self.crawler = Manager.getCrawlerInstance()