def parse_error(path):
html = open(path).read(10000)
soup = BeautifulSoup(html)
h1 = web.htmlunquote(soup.body.h1.string or "")
h2 = web.htmlunquote(soup.body.h2.string or "")
message = h1.split("at")[0].strip() + ": " + (h2 and h2.splitlines()[0])
code, url = [
web.htmlunquote(td.string) for td in soup.body.table.findAll("td")
]
# strip common prefixes
code = web.re_compile(
".*/(?:staging|production)/(openlibrary|infogami|web)").sub(
r"\1", code)
m = web.re_compile(r"(\d\d)(\d\d)(\d\d)(\d{6})").match(
web.numify(os.path.basename(path)))
hh, mm, ss, microsec = m.groups()
return web.storage(url=url,
message=message,
code=code,
time="%s:%s:%s" % (hh, mm, ss))
def get_atimes(block_number):
atimes = {}
d = self._dirname(block_number) # _dirname works even with block_number
for f in os.listdir(d):
id = int(web.numify(f))
atime = os.stat(os.path.join(d, f)).st_atime
atimes[id] = max(self.atimes.get(id, 0), atime)
return atimes
def save_user(i):
pin = web.numify(i.pin)
volunteer = 'volunteer' in i
if 'donate' in i:
donate_amt = int(i.donate_amt or '0')
else:
donate_amt = 0
donate = 'donate' in i and int(i.donate_amt, 0)
db.insert('users', name=i.name, email=i.email, phone=i.phone, pin=pin, want2volunteer=volunteer, want2donate=donate_amt)
def POST(self):
i = dict((k, v) for k, v in web.input().items() if k.startswith("status-"))
store = web.ctx.site.store
for k, v in i.items():
key = 'talks/' + web.numify(k)
talk = store[key]
if talk.get('status') != v:
store[key] = dict(talk, status=v)
raise web.seeother("/talks")
def update_user_details(i, uid=None):
if not uid:
user = helpers.get_user_by_email(i.get('email'))
uid = user and user.id
i['phone'] = web.numify(i.get('phone'))
details = ['prefix', 'lname', 'fname', 'addr1', 'addr2', 'city', 'zip5', 'zip4', 'phone', 'state']
d = {}
for (k, v) in i.items():
if v and (k in details):
d[k] = v
db.update('users', where='id=$uid', vars=locals(), **d)
def update_user_details(i):
user = helpers.get_user_by_email(i.get("email"))
userid = user and user.id
i["zip5"] = i.get("zipcode")
i["phone"] = web.numify(i.get("phone"))
details = ["prefix", "lname", "fname", "addr1", "addr2", "city", "zip5", "zip4", "phone", "state"]
d = {}
for (k, v) in i.items():
if v and (k in details):
d[k] = v
db.update("users", where="id=$userid", vars=locals(), **d)
def get_sort_key(key):
"""Returns a sort key used to group urls in 10K batches.
>>> get_sort_key("/authors/OL123456A")
'authors_0012'
"""
m = re_key.match(key)
if not m:
return
prefix = m.group(1)
num = int(web.numify(key)) / 10000
return "%s_%04d" % (prefix, num)
def get_sort_key(key):
"""Returns a sort key used to group urls in 10K batches.
>>> get_sort_key("/authors/OL123456A")
'authors_0012'
"""
m = re_key.match(key)
if not m:
return
prefix = m.group(1)
num = int(web.numify(key)) / 10000
return "%s_%04d" % (prefix, num)
def hash(self, key):
"""Returns an integer in the range [0..999] for each key.
"""
if key.startswith("/authors/"):
n = int(web.numify(key))
return 100 + n / 1000000
elif key.startswith("/books/"):
n = int(web.numify(key))
return 200 + n / 1000000
elif key.startswith("person:"):
return 400
elif key.startswith("place:"):
return 410
elif key.startswith("subject:"):
return 420
elif key.startswith("time:"):
return 430
elif key.startswith("/works/"):
n = int(web.numify(key))
return 800 + n / 1000000
else:
return 900
def hash(self, key):
"""Returns an integer in the range [0..999] for each key.
"""
if key.startswith("/authors/"):
n = int(web.numify(key))
return 100 + n / 1000000
elif key.startswith("/books/"):
n = int(web.numify(key))
return 200 + n / 1000000
elif key.startswith("person:"):
return 400
elif key.startswith("place:"):
return 410
elif key.startswith("subject:"):
return 420
elif key.startswith("time:"):
return 430
elif key.startswith("/works/"):
n = int(web.numify(key))
return 800 + n / 1000000
else:
return 900
def parse_ward(number):
html = download_ward(number)
soup = BeautifulSoup(html, "lxml")
th = soup.find("th", {"colspan": "2"})
title = th.get_text()
print title
table = th.parent.parent
data = [[td.get_text() for td in tr.find_all("td")] for tr in table.find_all("tr")]
d = {'Ward Name': title, 'Ward Number': web.numify(title)}
for row in data:
if len(row) == 2:
d[row[0].strip(": ")] = row[1].strip()
return d
def update_user_details(i, uid=None):
if not uid:
user = helpers.get_user_by_email(i.get('email'))
uid = user and user.id
i['phone'] = web.numify(i.get('phone'))
details = [
'prefix', 'lname', 'fname', 'addr1', 'addr2', 'city', 'zip5', 'zip4',
'phone', 'state'
]
d = {}
for (k, v) in i.items():
if v and (k in details):
d[k] = v
db.update('users', where='id=$uid', vars=locals(), **d)
def parse_error(path):
html = open(path).read(10000)
soup = BeautifulSoup(html)
h1 = web.htmlunquote(soup.body.h1.string or "")
h2 = web.htmlunquote(soup.body.h2.string or "")
message = h1.split('at')[0].strip() + ': ' + (h2 and h2.splitlines()[0])
code, url = [web.htmlunquote(td.string) for td in soup.body.table.findAll('td')]
# strip common prefixes
code = web.re_compile(".*/(?:staging|production)/(openlibrary|infogami|web)").sub(r'\1', code)
m = web.re_compile('(\d\d)(\d\d)(\d\d)(\d{6})').match(web.numify(os.path.basename(path)))
hh, mm, ss, microsec = m.groups()
return web.storage(url=url, message=message, code=code, time="%s:%s:%s" % (hh, mm, ss))
def get_tarfile(self, name):
id = web.numify(name)
tarname = "covers_%s_%s.tar" % (id[:4], id[4:6])
# for id-S.jpg, id-M.jpg, id-L.jpg
if '-' in name:
size = name[len(id + '-'):][0].lower()
tarname = size + "_" + tarname
else:
size = ""
_tarname, _tarfile, _indexfile = self.tarfiles[size.upper()]
if _tarname != tarname:
_tarname and _tarfile.close()
_tarfile, _indexfile = self.open_tarfile(tarname)
self.tarfiles[size.upper()] = tarname, _tarfile, _indexfile
log('writing', tarname)
return _tarfile, _indexfile
def get_tarfile(self, name):
id = web.numify(name)
prefix = "covers_%s_%s" % (id[:4], id[4:6])
if '-' in name:
size = name.split('-')[1].split('.')[0]
tarname = size.lower() + "_" + prefix + ".tar"
else:
size = ''
tarname = prefix + ".tar"
_tarname, _tarfile = self.tarfiles[size]
if _tarname != tarname:
_tarname and _tarfile.close()
_tarfile = self.open_tarfile(tarname)
self.tarfiles[size] = tarname, _tarfile
log('writing', tarname)
return _tarfile
def get_tarfile(self, name):
id = web.numify(name)
prefix = "covers_%s_%s" % (id[:4], id[4:6])
if '-' in name:
size = name.split('-')[1].split('.')[0]
tarname = size.lower() + "_" + prefix + ".tar"
else:
size = ''
tarname = prefix + ".tar"
_tarname, _tarfile = self.tarfiles[size]
if _tarname != tarname:
_tarname and _tarfile.close()
_tarfile = self.open_tarfile(tarname)
self.tarfiles[size] = tarname, _tarfile
log('writing', tarname)
return _tarfile
def get_tarfile(self, name):
id = web.numify(name)
tarname = "covers_%s_%s.tar" % (id[:4], id[4:6])
# for id-S.jpg, id-M.jpg, id-L.jpg
if '-' in name:
size = name[len(id + '-'):][0].lower()
tarname = size + "_" + tarname
else:
size = ""
_tarname, _tarfile, _indexfile = self.tarfiles[size.upper()]
if _tarname != tarname:
_tarname and _tarfile.close()
_tarfile, _indexfile = self.open_tarfile(tarname)
self.tarfiles[size.upper()] = tarname, _tarfile, _indexfile
log('writing', tarname)
return _tarfile, _indexfile
def split(self, seq, dir, bucket_size):
M = 1024*1024
files = {}
def get_file(index):
if index not in files:
files[index] = open(os.path.join(dir, "%04d.txt" % index), "w", 2*M)
return files[index]
for key, json in seq:
try:
index = int(web.numify(key)) / bucket_size
except Exception:
print >> sys.stderr, "bad key %s" % key
continue
get_file(index).write("%s\t%s\n" % (key, json))
for f in files.values():
f.close()
def split(self, seq, dir, bucket_size):
M = 1024*1024
files = {}
def get_file(index):
if index not in files:
files[index] = open(os.path.join(dir, "%04d.txt" % index), "w", 2*M)
return files[index]
for key, json in seq:
try:
index = int(web.numify(key)) / bucket_size
except Exception:
print >> sys.stderr, "bad key %s" % key
continue
get_file(index).write("%s\t%s\n" % (key, json))
for f in files.values():
f.close()
def send_msg(self, i, wyrform, pform=None):
pol = self.pol
captcha_src = (not i.get('captcha')) and get_captcha_src(pol)
if captcha_src:
set_captcha(wyrform, captcha_src)
msg = 'Please fill in the captcha verification below'
helpers.set_msg(msg, msg_type='note')
raise CaptchaException
email = '*****@*****.**' % (self.msg_id)
try:
msg_sent = writerep(pol=pol,
prefix=i.prefix, lname=i.lname, fname=i.fname,
addr1=i.addr1, addr2=i.addr2, city=i.city,
zipcode=i.zipcode, zip4=i.zip4,
phone=web.numify(i.phone), email=email, subject=i.ptitle, msg=i.msg,
captcha=i.get('captcha', ''))
except:
msg_sent = False
if not pform: update_user_details(i)
return msg_sent
def key_int(rec):
# extract the number from a key like /a/OL1234A
return int(web.numify(rec['key']))
def check_len(phone):
return len(web.numify(phone)) <= 15
def key_int(rec):
return int(web.numify(rec['key']))
def check_len(phone):
return len(web.numify(phone)) <= 15
def key_int(key):
# extract the number from a key like /a/OL1234A
return int(web.numify(key))
def key_int(key):
# extract the number from a key like /a/OL1234A
return int(web.numify(key))
def key_int(rec):
return int(web.numify(rec['key']))
