def main():
proxy = getcfg('http_proxy')
if proxy:
print 'Using http proxy %s' % (proxy, )
pollers.requestoptions['proxies'] = {'http': proxy, 'https': proxy}
try:
threading.stack_size(512 * 1024)
except BaseException as e:
print 'Error changing stack size:', repr(e)
# Connect to the database...
Database.get()
parser = ParserThread()
parser.start()
for poller in POLLERS.values():
poller.start()
# wait for them to finish!
for poller in POLLERS.values():
while poller.is_alive():
time.sleep(1)
while parser.is_alive():
time.sleep(1)
return
def run(self):
while True:
try:
n = self.parse_new_leaks()
if n == 0:
time.sleep(10)
else:
time.sleep(0.2)
except OperationalError as e:
print 'mysql error while processing %s: %s' % (id, repr(e))
exc_type, exc_value, exc_traceback = sys.exc_info()
traceback.print_exception(exc_type,
exc_value,
exc_traceback,
file=sys.stderr)
if e.args[0] == 2006:
# MySQL server as gone away
Database.reconnect()
return
def main():
proxy = getcfg("http_proxy")
if proxy:
print "Using http proxy %s" % (proxy,)
pollers.requestoptions["proxies"] = {"http": proxy, "https": proxy}
try:
threading.stack_size(512 * 1024)
except BaseException as e:
print "Error changing stack size:", repr(e)
# Connect to the database...
Database.get()
parser = ParserThread()
parser.start()
for poller in POLLERS.values():
poller.start()
# wait for them to finish!
for poller in POLLERS.values():
while poller.is_alive():
time.sleep(1)
while parser.is_alive():
time.sleep(1)
return
def seen(req, **params):
""" Return all leak id that this user has seen. """
sess = Session(req)
if sess.is_new():
req.status = apache.HTTP_BAD_REQUEST
return 'not logged in'
db = Database.get()
c = db.cursor()
q = """
select
us.leak_id
from
user_seen us
where
us.user_id = {user_id}
""".format(user_id=sess['user_id'])
c.execute(q)
r = c.fetchall()
c.close()
# reformat the list...
r = [e[0] for e in r]
req.content_type = 'application/json'
return json.dumps(dict(items=r), ensure_ascii=False)
def keep(self):
""" Insert an entry in the database. This should be overridden by the subclass, and
the subclass should insert subclass-specific data in its own table. """
reason = ", ".join([m.name for m in self.matches])
print "Keeping %s, reason: %s" % (repr(self), reason)
db = Database.get()
c = db.cursor()
c.execute(
"""replace into leaks (date, data, nblines, isbad, reason, source)
values (%s, %s, %s, %s, %s, %s)""",
(self.timestamp, self.data, None, False, reason, self.source),
)
leak_id = c.lastrowid
metadata = []
for name in self.metadata:
metadata.append((leak_id, name, self.metadata[name]))
c.executemany("replace into leak_metadata (leak_id, name, value) values (%s, %s, %s)", metadata)
db.commit()
c.close()
return leak_id
def keep(self):
""" Insert an entry in the database. This should be overridden by the subclass, and
the subclass should insert subclass-specific data in its own table. """
reason = ', '.join([m.name for m in self.matches])
print 'Keeping %s, reason: %s' % (repr(self), reason)
db = Database.get()
c = db.cursor()
c.execute("""replace into leaks (date, data, nblines, isbad, reason, source)
values (%s, %s, %s, %s, %s, %s)""", (self.timestamp, self.data, None,
False, reason, self.source))
leak_id = c.lastrowid
metadata = []
for name in self.metadata:
metadata.append((leak_id, name, self.metadata[name]))
c.executemany('replace into leak_metadata (leak_id, name, value) values (%s, %s, %s)', metadata)
db.commit()
c.close()
return leak_id
def run(self):
while True:
try:
n = self.parse_new_leaks()
if n == 0:
time.sleep(10)
else:
time.sleep(0.2)
except OperationalError as e:
print 'mysql error while processing %s: %s' % (id, repr(e))
exc_type, exc_value, exc_traceback = sys.exc_info()
traceback.print_exception(exc_type, exc_value, exc_traceback, file=sys.stderr)
if e.args[0] == 2006:
# MySQL server as gone away
Database.reconnect()
return
def login(req, **params):
""" New login attempt. Clean out old session if present, and create new one. """
sess = Session(req)
if not sess.is_new():
sess.delete()
sess = Session(req)
if not sess.is_new():
req.status = apache.HTTP_BAD_REQUEST
return 'failed to create new session'
if 'u' not in params or 'p' not in params:
req.status = apache.HTTP_BAD_REQUEST
return 'some parameters were not provided'
ret = dict()
if params['u'] != 'einstein' or params['p'] != 'fuckbin':
ret['success'] = False
ret['error'] = 'bad username or password'
# note: session is not saved!
else:
ret['success'] = True
# keep some stuff in session...
sess['username'] = params['u']
sess['user_id'] = 1
sess.set_timeout(60 * 60 * 24 * 365 * 10) # 10 year
sess.save()
# grab the user's cookie, and save the seen leaks into the database
seen_ranges = urllib.unquote(Cookie.get_cookie(req, '__CJ_seen').value)
seen_ranges = json.loads(seen_ranges)
values = [[sess['user_id'], i] for seen_range in seen_ranges
for i in range(seen_range['start'], seen_range['end'] + 1)]
db = Database.get()
c = db.cursor()
c.executemany(
""" replace into user_seen (user_id, leak_id) values (%s, %s) """,
values)
db.commit()
c.close()
req.content_type = 'application/json'
return json.dumps(ret, ensure_ascii=False)
def login(req, **params):
""" New login attempt. Clean out old session if present, and create new one. """
sess = Session(req)
if not sess.is_new():
sess.delete()
sess = Session(req)
if not sess.is_new():
req.status = apache.HTTP_BAD_REQUEST
return 'failed to create new session'
if 'u' not in params or 'p' not in params:
req.status = apache.HTTP_BAD_REQUEST
return 'some parameters were not provided'
ret = dict()
if params['u'] != 'einstein' or params['p'] != 'fuckbin':
ret['success'] = False
ret['error'] = 'bad username or password'
# note: session is not saved!
else:
ret['success'] = True
# keep some stuff in session...
sess['username'] = params['u']
sess['user_id'] = 1
sess.set_timeout(60 * 60 * 24 * 365 * 10) # 10 year
sess.save()
# grab the user's cookie, and save the seen leaks into the database
seen_ranges = urllib.unquote(Cookie.get_cookie(req, '__CJ_seen').value)
seen_ranges = json.loads(seen_ranges)
values = [[sess['user_id'], i] for seen_range in seen_ranges for i in
range(seen_range['start'], seen_range['end'] + 1)]
db = Database.get()
c = db.cursor()
c.executemany(""" replace into user_seen (user_id, leak_id) values (%s, %s) """, values)
db.commit()
c.close()
req.content_type = 'application/json'
return json.dumps(ret, ensure_ascii=False)
def index(req, **params):
""" Show details of a certain leek.
Transfers the data and related information.
"""
db = Database.get()
if 'id' not in params or not params['id'].isdigit():
req.status = apache.HTTP_BAD_REQUEST
return '"id" param is not digits'
leak_id = int(params['id'])
c = db.cursor()
c.execute("""
select
l.data, l.htmldata
from
leaks l
where
l.leak_id = %s
""", (leak_id, ))
r = c.fetchall()
sess = Session(req)
if not sess.is_new():
c.execute("""
replace into user_seen (user_id, leak_id) values (%s, %s)
""", (sess['user_id'], leak_id))
db.commit()
c.close()
details = list(r[0])
# details[0] = urllib.quote(details[0])
details[0] = details[0].replace('&', '&').replace('<', '<').replace('>',
'>').replace('"', '"').replace("'", ''')
req.content_type = 'application/json'
return json.dumps(details, ensure_ascii=False)
def parse_new_leaks(self):
db = Database.get()
c = db.cursor()
c.execute("""
select
l.leak_id, l.data, l.reason
from
leaks l
where
isparsed = 0 and
data is not null
order by leak_id
limit 10
""")
r = c.fetchall()
for entry in r:
leak_id, data, reason = entry
if 'Mysql user/password' in reason:
self.parse_mysql_connect(c, leak_id, data)
else:
self.parse_leak(c, leak_id, data)
if len(r) == 0:
return 0
ids = ','.join([str(e[0]) for e in r])
c.execute(""" update leaks set isparsed=1 where leak_id in (%s) """ %
(ids, ))
print 'parsed!', repr(ids)
db.commit()
c.close()
return len(r)
def parse_new_leaks(self):
db = Database.get()
c = db.cursor()
c.execute("""
select
l.leak_id, l.data, l.reason
from
leaks l
where
isparsed = 0 and
data is not null
order by leak_id
limit 10
""")
r = c.fetchall()
for entry in r:
leak_id, data, reason = entry
if 'Mysql user/password' in reason:
self.parse_mysql_connect(c, leak_id, data)
else:
self.parse_leak(c, leak_id, data)
if len(r) == 0:
return 0
ids = ','.join([str(e[0]) for e in r])
c.execute(""" update leaks set isparsed=1 where leak_id in (%s) """ % (ids, ))
print 'parsed!', repr(ids)
db.commit()
c.close()
return len(r)
def index(req, **params):
""" List recent leaks. Limit of 300 leaks shown.
parameters:
- after:
show leaks after this timestamp
- period:
show leaks that occured within this period (today, week, month)
"""
limit = ''
if 'after' in params:
if not params['after'].isdigit():
req.status = apache.HTTP_BAD_REQUEST
return '"after" param is not digits'
leak_id = int(params['after'])
where = 'l.leak_id >= %u' % leak_id
elif 'before' in params:
if not params['after'].isdigit():
req.status = apache.HTTP_BAD_REQUEST
return '"before" param is not digits'
leak_id = int(params['after'])
where = 'l.leak_id <= %u' % leak_id
limit = 'limit 10'
elif 'period' in params:
if params['period'] == 'initial':
# By default, load today's leaks, otherwise load the last 300 leaks from whenever...
where = 'DATE(FROM_UNIXTIME(l.date)) = DATE(SYSDATE())'
else:
req.status = apache.HTTP_BAD_REQUEST
return '"period" param has unknown value'
else:
req.status = apache.HTTP_BAD_REQUEST
return 'bad request'
sess = Session(req)
if sess.is_new():
seen = 'false'
else:
seen = '(select true from user_seen us where us.user_id = {user_id} and us.leak_id = l.leak_id)'.format(
user_id=sess['user_id'], )
db = Database.get()
c = db.cursor()
q = """
select
l.leak_id,
({seen}) as seen,
l.date, l.comment, l.reason, l.source
from
leaks l
where
{where} and
l.isparsed = 1
order by leak_id
{limit}
""".format(seen=seen, where=where, limit=limit)
c.execute(q)
r = list(c.fetchall())
if len(r) == 0 and 'period' in params and params['period'] == 'initial':
# last chance to get some latest entry...
c.execute("""
select
l.leak_id,
({seen}) as seen,
l.date, l.comment, l.reason, l.source
from
leaks l
where
l.isparsed = 1
order by leak_id desc
limit 200
""".format(seen=seen, where=where, limit=limit))
r = list(c.fetchall())
r.reverse()
if len(r) > 0:
leak_ids = [str(e[0]) for e in r]
c.execute("""
select
leak_id, name, value
from
leak_metadata
where
leak_id in ({ids})
""".format(ids=','.join(leak_ids)))
metadata = c.fetchall()
# append a dictionary at the end of every entry
for i in range(len(r)):
r[i] = list(r[i])
r[i].append({})
# insert metadata into dictionary
for i in range(len(r)):
for data in metadata:
if data[0] != r[i][0]:
continue
r[i][-1][data[1]] = data[2]
c.close()
req.content_type = 'application/json'
return json.dumps(dict(items=r), ensure_ascii=False)