Ejemplo n.º 1
0
def store_revisions(self, page_url):
  """
  Retrieve all the revision of a give wikipedia page_url

  parameters:
    - page_url: a wikipedia page URL
  """

  p = Page()

  d = Dataset( "%s:27017" % (mongodb_host) )

  title = url2title(page_url)
  lang = url2lang(page_url)

  p.fetch_from_api_title(title, lang=lang)

  revisions = p.get_all_editors()

  i = 0

  for revision in revisions:
    i += 1

    # ex: en/crimea/revision/999999
    key = "%s/%s/revision/%s" % (lang,title,revision["revid"])

    # fetch the revision from the internet
    value = p.get_revisions(extra_params={ "rvstartid": revision["revid"], "rvlimit" : 1})

    # write in it the database handler
    d.write(key, value)
    self.update_state( state='PROGRESS',
      meta= { 'current': i, 'total': len(revisions)})
Ejemplo n.º 2
0
def get_page_info(url, length, index):
 #  print ""

	# print url

	title = url2title(url)
	lang = url2lang(url) 

	print "[%s] %s (%s/%s)" % (lang, title, index, length)

	wp = Page()

#	print wp

	if (lang != "www"):
		r = wp.fetch_from_api_title(title, lang=lang)

		file = "dataset/%s.info.json" % (wp.page_id)

		if not os.path.isfile(file):
			with open(file, "w") as out:
				data = {
					"edits": wp.get_all_editors(),
					"langs": wp.get_langlinks()
				}

				json.dump(data, out)
Ejemplo n.º 3
0
def dataset_timeline(url):
  print "timeline: %s" % (url)
  d = Dataset( "%s:27017" % (mongodb_host) )

  title = url2title(url)
  lang = url2lang(url)

  url = "%s/%s" % (lang, title)

  regex_string = "%s\/%s\/revision/([0-9]*$)" % (lang, title)

  r = d.find({ "url" : { "$regex" : regex_string } }, { "dataset.timestamp" : 1, "dataset.revid" : 1 })

  timeline = []

  for result in r:
    i = { "timestamp": result["dataset"][0]["timestamp"], "revid": result["dataset"][0]["revid"] }
    timeline.append(i)

  timeline = sorted( timeline, key=lambda rev: rev["timestamp"])

  print "start: %s" % (timeline[0])
  print "end: %s" % (timeline[-1])

  k = "%s/%s/timeline" % (lang, title)
  d.delete(k)
  d.write(k, timeline)

  print r.count()
Ejemplo n.º 4
0
users = {}

init()

extra_languages = [ "en", "simple", "uk", "fr", "ru", "de" ]

source_in = "data/in/wicrimea-seeds.txt"
source_ext = "data/out/wicrimea-seeds.extended.txt"

out = open(source_ext, "w")

with open(source_in, "r") as file:
  for l in file:
    p = Page()
    r = p.fetch_from_api_title(url2title(l.strip()))

    print ""
    print u"→ %s (%s)" % (p.title, l.strip())

    out.write(l.strip()+"\n")

    langs = p.get_langlinks()

    for l in langs:
      if l["lang"] in extra_languages:
#        print l

        p_lang = Page()
        p_lang.fetch_from_api_title(l["*"], lang=l["lang"])