def get(self, game_id, player_id, x, y):
self.response.headers['Content-Type'] = 'text/plain'
game = Game.get_by_id(game_id)
if not game:
self.response.status = 412
self.response.write(self.response.status)
return
p = game.player(player_id)
Cache.setPos(game_id, player_id, x, y)
self.response.write(p.output())
def get_bulk_soup( self, urls, bar=None ):
Cache.cache_bulk(urls, bar=bar)
if bar:
bar.label = 'Souping...'
bar.expected_size = len(urls)
bar.show(0)
ret = {}
for url in urls:
ret[url] = self.getsoup(url)
if bar:
bar.show(bar.last_progress + 1)
return ret
def get(self, game_id):
pos = Cache.getPos(game_id)
if pos:
self.response.headers['Content-Type'] = 'text/plain'
self.response.status = 200
self.response.out.write("|".join(
["%s:%s,%s" % (p, x, y) for p, (x, y) in pos.iteritems()]))
else:
self.response.headers['Content-Type'] = 'text/plain'
self.response.status = 404
def set_lang_id(self, chat_id, lang_id):
if lang_id is None:
lang_id = "en"
Cache().invalidate_lang_cache(chat_id)
try:
self.cursor.execute(
"INSERT INTO chats (chat_id, lang_id) VALUES(?, ?);",
[chat_id, lang_id])
except sqlite3.IntegrityError:
self.cursor.execute(
"UPDATE chats SET lang_id = ? WHERE chat_id = ?;",
[lang_id, chat_id])
self.connection.commit()
def get(self, game_id):
game = Game.get_by_id(game_id)
hist = Cache.getHist(game_id)
if not hist:
self.response.status = 404
self.response.write(self.response.status)
return
self.response.headers['Content-Type'] = 'text/plain'
self.response.status = 200
self.response.out.write("|".join([
"%s:%s" % (p, ",".join([str(h.coords) for h in hls]))
for p, hls in hist.iteritems()
]))
def get(self, game_id):
hist = Cache.getHist(game_id)
if not hist or not paramVal(self, "modes"):
self.response.status = 404
self.response.write(self.response.status)
return
modes = paramVal(self, "modes").split(" ")
self.response.headers['Content-Type'] = 'text/plain'
self.response.status = 200
self.response.out.write("|".join([
"%s:%s" % (p, ",".join(
Map.get_reachable_areas(
PlayerState([(h.pickup_code, h.pickup_id, h.removed)
for h in hls]), modes)))
for p, hls in hist.iteritems()
]))
def get(self, game_id, player_id):
seedlines = []
lines = paramVal(self, "seed").split(",")
game = Game.get_by_id(game_id)
hist = Cache.getHist(game_id)
if not hist:
Cache.setHist(game_id, player_id, [])
pos = Cache.getPos(game_id)
if not pos:
Cache.setPos(game_id, player_id, 0, 0)
if not game:
flags = lines[0].split("|")
mode_opt = [f[5:] for f in flags if f.lower().startswith("mode=")]
shared_opt = [
f[7:].split(" ") for f in flags
if f.lower().startswith("shared=")
]
mode = mode_opt[0].lower() if mode_opt else None
if mode:
mode = mode_map[mode] if mode in mode_map else int(mode)
shared = shared_opt[0] if shared_opt else None
game = get_new_game(_mode=mode, _shared=shared, id=game_id)
for l in lines[1:]:
line = l.split("|")
if len(line) < 3:
print "ERROR: malformed seed line %s, skipping" % l
else:
seedlines.append("%s:%s" %
(line[0], Pickup.name(line[1], line[2])))
player = game.player(player_id)
player.seed = "\n".join(seedlines)
player.put()
self.response.headers['Content-Type'] = 'text/plain'
self.response.status = 200
self.response.out.write("ok")
def scrape (self):
if not self.rp:
r = Cache.get(self.nation.cfg['entrypoint'])
soup = BS(str(r))
#find current release point
log.info("No release point specified, retreiving latest...")
# this failed fantastically - we'll get the RP from the zipurl
#self.rp = utf8(soup.findAll('h3', attrs={'class': 'releasepointinformation'})[0].text.split()[-1])
log.info("Found release point %s" % self.rp)
#find the download url
self.zipurl = self.nation.cfg['entrypoint'].rpartition('/')[0] + '/' + soup.findAll('a', title='All USC Titles in XML')[0]['href']
# new way to set the rp using the zipurl's filename
self.rp = utf8( self.zipurl.rpartition('@')[-1].partition('.')[0] )
else:
log.info('Using specified release point %s...' % self.rp)
# don't actually need this
# rpurl = 'http://uscode.house.gov/download/releasepoints/us/pl/%s/%s/usc-rp@%s.htm' % (tuple(self.rp.split()) + (self.rp,))
self.zipurl = 'http://uscode.house.gov/download/releasepoints/us/pl/%s/%s/xml_uscAll@%s.zip' % (tuple(self.rp.split('-')) + (self.rp,))
log.debug("Using zipurl: %s" % self.zipurl)
class FileNotThere (Exception): pass
class XMLNotThere( Exception ): pass
class AllGood( Exception ): pass
filename = self.zipurl.rpartition('/')[-1]
xmldir = self._workdir + os.sep + 'xml' + os.sep
# check to see if we have xml that works
# if we don't check to see if we have a zip file
# if we don't, download it
# if we do, extract it
# check the xml again, if it's good, proceed
# if it's not, error out
try:
assert os.path.exists(xmldir + 'usc01.xml')
soup = BS(open(xmldir + os.sep + 'usc01.xml', 'r').read())
xmlrp = soup.find('docpublicationname').text.split('@')[-1]
#old way to get rp, the new way is much better
# xmlrp = soup.title.first("note", topic="miscellaneous").text.split()[-1]
if xmlrp == self.rp:
raise AllGood
else:
raise XMLNotThere
except (XMLNotThere,AssertionError):
# delete directory if it exists
if os.path.exists(xmldir):
shutil.rmtree(xmldir)
# if there's no xml file, download it
if not os.path.exists(self._workdir + os.sep + filename):
log.info('No zipfile found for release point, downloading...')
self.downloadFile(self.zipurl, filename)
# now we should have a zipfile and no existing xmldir
log.info('Extracting file %s...' % filename)
zf = ZipFile(self._workdir + os.sep + filename, 'r')
# older release points do not have an interior xml/ dir
if not all( [ n.startswith('xml/') for n in zf.namelist()]):
zf.extractall(xmldir)
else:
zf.extractall(self._workdir)
# double check the xml now...
assert os.path.exists(xmldir + 'usc01.xml')
# it may be problematic to rely on the RP information in the XML documents provided
# rp 113-21 (the earliest presently available) does not include this in the
# docpublicationname meta tag
#soup = BS(open(xmldir + os.sep + 'usc01.xml', 'r').read())
#xmlrp = soup.find('docpublicationname').text.split('@')[-1]
#if xmlrp != self.rp:
# raise XMLNotThere('XML did not check out after extraction.')
except AllGood:
pass
except:
raise
log.info('All checks passed...')
xf = os.listdir(xmldir)
root = self.findOrCreateRoot()
xf = [xmldir + f for f in xf if f.endswith('.xml')]
xf.sort()
log.info("Processing %i files..." % len(xf))
self.bar = progress.Bar(label='US', expected_size=1000*len(xf))
self.progress( i=len(xf) )
for fn in xf:
self.processFile(fn, root)
self.progress(rollup=1000)
log.info('Analyzing code...')
self.progress(label="Analyzing")
root.analyze(commit=True, bar=self.bar)
store.commit()
log.info('Scrape completed.')
from urllib.parse import urlencode
from hashlib import md5
import json
import re
import datetime
import os
from tornado import gen
from tornado.httpclient import AsyncHTTPClient as HTTPClient
from tornado.httpclient import HTTPRequest
import bs4.element
import requests
from util import Cache, unicode_type, bytes_type
cache = Cache()
jwc_domain = 'http://202.119.81.112:9080'
def login_data(username, password):
if isinstance(password, unicode_type):
password = password.encode('utf-8')
data = {
'method': 'verify',
'USERNAME': username,
'PASSWORD': md5(password).hexdigest().upper()
}
return data
def login_session(username, password):
def geturl( self, url ):
return Cache.get( url ).resp
