コード例 #1
0
ファイル: poll.py プロジェクト: InfoR3aper/pbl
def main():

    proxy = getcfg('http_proxy')
    if proxy:
        print 'Using http proxy %s' % (proxy, )
        pollers.requestoptions['proxies'] = {'http': proxy, 'https': proxy}

    try:
        threading.stack_size(512 * 1024)
    except BaseException as e:
        print 'Error changing stack size:', repr(e)

    # Connect to the database...
    Database.get()

    parser = ParserThread()
    parser.start()

    for poller in POLLERS.values():
        poller.start()

    # wait for them to finish!
    for poller in POLLERS.values():
        while poller.is_alive():
            time.sleep(1)

    while parser.is_alive():
        time.sleep(1)

    return
コード例 #2
0
    def run(self):

        while True:

            try:
                n = self.parse_new_leaks()

                if n == 0:
                    time.sleep(10)
                else:
                    time.sleep(0.2)

            except OperationalError as e:
                print 'mysql error while processing %s: %s' % (id, repr(e))

                exc_type, exc_value, exc_traceback = sys.exc_info()
                traceback.print_exception(exc_type,
                                          exc_value,
                                          exc_traceback,
                                          file=sys.stderr)

                if e.args[0] == 2006:
                    # MySQL server as gone away
                    Database.reconnect()

        return
コード例 #3
0
ファイル: poll.py プロジェクト: EiNSTeiN-/pbl
def main():

    proxy = getcfg("http_proxy")
    if proxy:
        print "Using http proxy %s" % (proxy,)
        pollers.requestoptions["proxies"] = {"http": proxy, "https": proxy}

    try:
        threading.stack_size(512 * 1024)
    except BaseException as e:
        print "Error changing stack size:", repr(e)

    # Connect to the database...
    Database.get()

    parser = ParserThread()
    parser.start()

    for poller in POLLERS.values():
        poller.start()

    # wait for them to finish!
    for poller in POLLERS.values():
        while poller.is_alive():
            time.sleep(1)

    while parser.is_alive():
        time.sleep(1)

    return
コード例 #4
0
def seen(req, **params):
    """ Return all leak id that this user has seen. """

    sess = Session(req)
    if sess.is_new():
        req.status = apache.HTTP_BAD_REQUEST
        return 'not logged in'

    db = Database.get()

    c = db.cursor()
    q = """
        select
            us.leak_id
        from
            user_seen us
        where
            us.user_id = {user_id}
    """.format(user_id=sess['user_id'])
    c.execute(q)
    r = c.fetchall()
    c.close()

    # reformat the list...
    r = [e[0] for e in r]

    req.content_type = 'application/json'
    return json.dumps(dict(items=r), ensure_ascii=False)
コード例 #5
0
ファイル: baseentry.py プロジェクト: EiNSTeiN-/pbl
    def keep(self):
        """ Insert an entry in the database. This should be overridden by the subclass, and
        the subclass should insert subclass-specific data in its own table. """

        reason = ", ".join([m.name for m in self.matches])
        print "Keeping %s, reason: %s" % (repr(self), reason)

        db = Database.get()
        c = db.cursor()

        c.execute(
            """replace into leaks (date, data, nblines, isbad, reason, source)
                        values (%s, %s, %s, %s, %s, %s)""",
            (self.timestamp, self.data, None, False, reason, self.source),
        )
        leak_id = c.lastrowid

        metadata = []
        for name in self.metadata:
            metadata.append((leak_id, name, self.metadata[name]))

        c.executemany("replace into leak_metadata (leak_id, name, value) values (%s, %s, %s)", metadata)

        db.commit()
        c.close()

        return leak_id
コード例 #6
0
    def keep(self):
        """ Insert an entry in the database. This should be overridden by the subclass, and
        the subclass should insert subclass-specific data in its own table. """

        reason = ', '.join([m.name for m in self.matches])
        print 'Keeping %s, reason: %s' % (repr(self), reason)

        db = Database.get()
        c = db.cursor()

        c.execute("""replace into leaks (date, data, nblines, isbad, reason, source)
                        values (%s, %s, %s, %s, %s, %s)""", (self.timestamp, self.data, None,
                        False, reason, self.source))
        leak_id = c.lastrowid

        metadata = []
        for name in self.metadata:
            metadata.append((leak_id, name, self.metadata[name]))

        c.executemany('replace into leak_metadata (leak_id, name, value) values (%s, %s, %s)', metadata)

        db.commit()
        c.close()

        return leak_id
コード例 #7
0
ファイル: parserthread.py プロジェクト: EiNSTeiN-/pbl
    def run(self):

        while True:

            try:
                n = self.parse_new_leaks()

                if n == 0:
                    time.sleep(10)
                else:
                    time.sleep(0.2)

            except OperationalError as e:
                print 'mysql error while processing %s: %s' % (id, repr(e))

                exc_type, exc_value, exc_traceback = sys.exc_info()
                traceback.print_exception(exc_type, exc_value, exc_traceback, file=sys.stderr)

                if e.args[0] == 2006:
                    # MySQL server as gone away
                    Database.reconnect()

        return
コード例 #8
0
ファイル: session.py プロジェクト: InfoR3aper/pbl
def login(req, **params):
    """ New login attempt. Clean out old session if present, and create new one. """

    sess = Session(req)
    if not sess.is_new():
        sess.delete()
        sess = Session(req)
        if not sess.is_new():
            req.status = apache.HTTP_BAD_REQUEST
            return 'failed to create new session'

    if 'u' not in params or 'p' not in params:
        req.status = apache.HTTP_BAD_REQUEST
        return 'some parameters were not provided'

    ret = dict()

    if params['u'] != 'einstein' or params['p'] != 'fuckbin':
        ret['success'] = False
        ret['error'] = 'bad username or password'

        # note: session is not saved!
    else:
        ret['success'] = True

        # keep some stuff in session...
        sess['username'] = params['u']
        sess['user_id'] = 1

        sess.set_timeout(60 * 60 * 24 * 365 * 10)  # 10 year
        sess.save()

        # grab the user's cookie, and save the seen leaks into the database
        seen_ranges = urllib.unquote(Cookie.get_cookie(req, '__CJ_seen').value)
        seen_ranges = json.loads(seen_ranges)
        values = [[sess['user_id'], i] for seen_range in seen_ranges
                  for i in range(seen_range['start'], seen_range['end'] + 1)]

        db = Database.get()
        c = db.cursor()
        c.executemany(
            """ replace into user_seen (user_id, leak_id) values (%s, %s) """,
            values)
        db.commit()
        c.close()

    req.content_type = 'application/json'
    return json.dumps(ret, ensure_ascii=False)
コード例 #9
0
ファイル: session.py プロジェクト: EiNSTeiN-/pbl
def login(req, **params):
    """ New login attempt. Clean out old session if present, and create new one. """

    sess = Session(req)
    if not sess.is_new():
        sess.delete()
        sess = Session(req)
        if not sess.is_new():
            req.status = apache.HTTP_BAD_REQUEST
            return 'failed to create new session'

    if 'u' not in params or 'p' not in params:
        req.status = apache.HTTP_BAD_REQUEST
        return 'some parameters were not provided'

    ret = dict()

    if params['u'] != 'einstein' or params['p'] != 'fuckbin':
        ret['success'] = False
        ret['error'] = 'bad username or password'

        # note: session is not saved!
    else:
        ret['success'] = True

        # keep some stuff in session...
        sess['username'] = params['u']
        sess['user_id'] = 1

        sess.set_timeout(60 * 60 * 24 * 365 * 10)  # 10 year
        sess.save()

        # grab the user's cookie, and save the seen leaks into the database
        seen_ranges = urllib.unquote(Cookie.get_cookie(req, '__CJ_seen').value)
        seen_ranges = json.loads(seen_ranges)
        values = [[sess['user_id'], i] for seen_range in seen_ranges for i in
            range(seen_range['start'], seen_range['end'] + 1)]

        db = Database.get()
        c = db.cursor()
        c.executemany(""" replace into user_seen (user_id, leak_id) values (%s, %s) """, values)
        db.commit()
        c.close()

    req.content_type = 'application/json'
    return json.dumps(ret, ensure_ascii=False)
コード例 #10
0
ファイル: details.py プロジェクト: EiNSTeiN-/pbl
def index(req, **params):
    """ Show details of a certain leek.

    Transfers the data and related information.
    """

    db = Database.get()

    if 'id' not in params or not params['id'].isdigit():
        req.status = apache.HTTP_BAD_REQUEST
        return '"id" param is not digits'

    leak_id = int(params['id'])

    c = db.cursor()
    c.execute("""
        select
            l.data, l.htmldata
        from
            leaks l
        where
            l.leak_id = %s
    """, (leak_id, ))
    r = c.fetchall()

    sess = Session(req)
    if not sess.is_new():
        c.execute("""
            replace into user_seen (user_id, leak_id) values (%s, %s)
        """, (sess['user_id'], leak_id))

    db.commit()
    c.close()

    details = list(r[0])
#    details[0] = urllib.quote(details[0])
    details[0] = details[0].replace('&', '&amp;').replace('<', '&lt;').replace('>',
        '&gt;').replace('"', '&quot;').replace("'", '&#39;')

    req.content_type = 'application/json'
    return json.dumps(details, ensure_ascii=False)
コード例 #11
0
    def parse_new_leaks(self):

        db = Database.get()

        c = db.cursor()
        c.execute("""
            select
                l.leak_id, l.data, l.reason
            from
                leaks l
            where
                isparsed = 0 and
                data is not null
            order by leak_id
            limit 10
        """)
        r = c.fetchall()

        for entry in r:
            leak_id, data, reason = entry

            if 'Mysql user/password' in reason:
                self.parse_mysql_connect(c, leak_id, data)
            else:
                self.parse_leak(c, leak_id, data)

        if len(r) == 0:
            return 0

        ids = ','.join([str(e[0]) for e in r])
        c.execute(""" update leaks set isparsed=1 where leak_id in (%s) """ %
                  (ids, ))
        print 'parsed!', repr(ids)

        db.commit()
        c.close()

        return len(r)
コード例 #12
0
ファイル: parserthread.py プロジェクト: EiNSTeiN-/pbl
    def parse_new_leaks(self):

        db = Database.get()

        c = db.cursor()
        c.execute("""
            select
                l.leak_id, l.data, l.reason
            from
                leaks l
            where
                isparsed = 0 and
                data is not null
            order by leak_id
            limit 10
        """)
        r = c.fetchall()

        for entry in r:
            leak_id, data, reason = entry

            if 'Mysql user/password' in reason:
                self.parse_mysql_connect(c, leak_id, data)
            else:
                self.parse_leak(c, leak_id, data)

        if len(r) == 0:
            return 0

        ids = ','.join([str(e[0]) for e in r])
        c.execute(""" update leaks set isparsed=1 where leak_id in (%s) """ % (ids, ))
        print 'parsed!', repr(ids)

        db.commit()
        c.close()

        return len(r)
コード例 #13
0
def index(req, **params):
    """ List recent leaks. Limit of 300 leaks shown.

    parameters:
        - after:
            show leaks after this timestamp
        - period:
            show leaks that occured within this period (today, week, month)
    """

    limit = ''
    if 'after' in params:
        if not params['after'].isdigit():
            req.status = apache.HTTP_BAD_REQUEST
            return '"after" param is not digits'

        leak_id = int(params['after'])
        where = 'l.leak_id >= %u' % leak_id

    elif 'before' in params:
        if not params['after'].isdigit():
            req.status = apache.HTTP_BAD_REQUEST
            return '"before" param is not digits'

        leak_id = int(params['after'])
        where = 'l.leak_id <= %u' % leak_id
        limit = 'limit 10'

    elif 'period' in params:
        if params['period'] == 'initial':
            # By default, load today's leaks, otherwise load the last 300 leaks from whenever...
            where = 'DATE(FROM_UNIXTIME(l.date)) = DATE(SYSDATE())'
        else:
            req.status = apache.HTTP_BAD_REQUEST
            return '"period" param has unknown value'
    else:
        req.status = apache.HTTP_BAD_REQUEST
        return 'bad request'

    sess = Session(req)
    if sess.is_new():
        seen = 'false'
    else:
        seen = '(select true from user_seen us where us.user_id = {user_id} and us.leak_id = l.leak_id)'.format(
            user_id=sess['user_id'], )

    db = Database.get()

    c = db.cursor()
    q = """
        select
            l.leak_id,
            ({seen}) as seen,
            l.date, l.comment, l.reason, l.source
        from
            leaks l
        where
            {where} and
            l.isparsed = 1
        order by leak_id
        {limit}
    """.format(seen=seen, where=where, limit=limit)
    c.execute(q)
    r = list(c.fetchall())

    if len(r) == 0 and 'period' in params and params['period'] == 'initial':
        # last chance to get some latest entry...
        c.execute("""
            select
                l.leak_id,
                ({seen}) as seen,
                l.date, l.comment, l.reason, l.source
            from
                leaks l
            where
                l.isparsed = 1
            order by leak_id desc
            limit 200
        """.format(seen=seen, where=where, limit=limit))
        r = list(c.fetchall())
        r.reverse()

    if len(r) > 0:
        leak_ids = [str(e[0]) for e in r]
        c.execute("""
            select
                leak_id, name, value
            from
                leak_metadata
            where
                leak_id in ({ids})
        """.format(ids=','.join(leak_ids)))
        metadata = c.fetchall()

        # append a dictionary at the end of every entry
        for i in range(len(r)):
            r[i] = list(r[i])
            r[i].append({})

        # insert metadata into dictionary
        for i in range(len(r)):
            for data in metadata:
                if data[0] != r[i][0]:
                    continue

                r[i][-1][data[1]] = data[2]

    c.close()

    req.content_type = 'application/json'
    return json.dumps(dict(items=r), ensure_ascii=False)