Exemplo n.º 1
0
def main():
    # reads a .env file
    konfab.read_env('../.env')

    global pg_connection, mongo_db
    pg_connection = konfab.getDBConnection(False)
    mongo_db = konfab.getMongoConnection()

    if pg_connection and mongo_db:
        application.listen(8080)
        tornado.ioloop.IOLoop.instance().start()
    else:
        sys.exit(0)
                    th.start()
                    th.join()



        print "Processing done..."




if __name__ == '__main__':
    sys.path.append(os.path.abspath('../'))
    import main as konfab
    try:
        urlid = sys.argv[1]
    except:
        urlid = None

    konfab.read_env('../.env')
    pg_connection = konfab.getDBConnection()

    if pg_connection:
        print "Starting reprocessing... "
        if urlid:
            ReprocessAttributes(pg_connection, setAttributesForUrl, count=1, url_id=urlid)
        else:
            ReprocessAttributes(pg_connection, setAttributesForUrl, count=500, url_id=None)
    else:
        print "No connection"

Exemplo n.º 3
0
                    th.setDaemon(True)
                    th.start()
                    th.join()

        print "Processing done..."


if __name__ == '__main__':
    sys.path.append(os.path.abspath('../'))
    import main as konfab
    try:
        urlid = sys.argv[1]
    except:
        urlid = None

    konfab.read_env('../.env')
    pg_connection = konfab.getDBConnection()

    if pg_connection:
        print "Starting reprocessing... "
        if urlid:
            ReprocessAttributes(pg_connection,
                                setAttributesForUrl,
                                count=1,
                                url_id=urlid)
        else:
            ReprocessAttributes(pg_connection,
                                setAttributesForUrl,
                                count=500,
                                url_id=None)
    else:
Exemplo n.º 4
0
#

import os,sys
from main import do_remove_oldest_tweets, getDBConnection, getMongoConnection, read_env, getIdTimestamp
import psycopg2
import psycopg2.extras
from psycopg2 import DataError, InternalError, DatabaseError

def remove_html(days=3):
    cur = pg_connection.cursor()
    q = '''DELETE FROM htmlpages USING messages
            WHERE htmlpages.url_id = CAST(messages.url_id as integer)
            AND to_timestamp(messages.time) < now() - interval '%s' day
        '''
    cur.execute(q,[days])
    pg_connection.commit()
    cur.close()

def main():
    remove_html(5)
    do_remove_oldest_tweets(5)


if __name__ == '__main__':
    read_env()

    pg_connection = getDBConnection()
    mongo_db = getMongoConnection()

    main()
Exemplo n.º 5
0
import os, sys
from main import do_remove_oldest_tweets, getDBConnection, getMongoConnection, read_env, getIdTimestamp
import psycopg2
import psycopg2.extras
from psycopg2 import DataError, InternalError, DatabaseError


def remove_html(days=3):
    cur = pg_connection.cursor()
    q = '''DELETE FROM htmlpages USING messages
            WHERE htmlpages.url_id = CAST(messages.url_id as integer)
            AND to_timestamp(messages.time) < now() - interval '%s' day
        '''
    cur.execute(q, [days])
    pg_connection.commit()
    cur.close()


def main():
    remove_html(5)
    do_remove_oldest_tweets(5)


if __name__ == '__main__':
    read_env()

    pg_connection = getDBConnection()
    mongo_db = getMongoConnection()

    main()