def store_revisions(self, page_url): """ Retrieve all the revision of a give wikipedia page_url parameters: - page_url: a wikipedia page URL """ p = Page() d = Dataset( "%s:27017" % (mongodb_host) ) title = url2title(page_url) lang = url2lang(page_url) p.fetch_from_api_title(title, lang=lang) revisions = p.get_all_editors() i = 0 for revision in revisions: i += 1 # ex: en/crimea/revision/999999 key = "%s/%s/revision/%s" % (lang,title,revision["revid"]) # fetch the revision from the internet value = p.get_revisions(extra_params={ "rvstartid": revision["revid"], "rvlimit" : 1}) # write in it the database handler d.write(key, value) self.update_state( state='PROGRESS', meta= { 'current': i, 'total': len(revisions)})
def store_last_revisions(db_url): d = Dataset( "%s:27017" % (mongodb_host) ) url = db_url.replace("/timeline", "") (lang, page) = url.split("/") p = Page() p.fetch_from_api_title(page, lang=lang) last_rev = p.get_revisions(extra_params={ "rvlimit" : 1 }) print "last revisions: %s" % (url.encode("utf8")) t = list(d.find({ "url": db_url }, { "url" : 1, "dataset" : { "$slice": -1 } })) # print t[0] extra_params = { "rvstartid": t[0]["dataset"][0]["revid"], "rvendid": last_rev[0]["revid"], "rvdir": "newer" } print extra_params revs = p.get_revisions(extra_params=extra_params) print "%s new revisions since %s (%s)" % (len(revs), t[0]["dataset"][0]["timestamp"], t[0]["dataset"][0]["revid"]) print "%s ----> %s" % (t[0]["dataset"][0]["timestamp"], last_rev[0]["timestamp"]) for r in revs: key = "%s/%s/revision/%s" % (lang, page, r["revid"]) value = [ r ] d.write(key, value)