Esempio n. 1
0
def search_updates(day, database='openlibrary', user='******', pw=''):
    time.sleep(0.05)
    logger.info('Day %s: searching items...',day)
    db.setup_database(database='openlibrary', user='******', pw='')
    q = "SELECT key, last_modified FROM thing WHERE (type='17872418' OR type='9887992') AND last_modified >= '"+day+"' AND last_modified < date '"+day+"' + interval '1' day"
    rows = db.longquery(q,vars=locals())
    check_updates(rows,day)
Esempio n. 2
0
def search_updates(day, database='openlibrary', user='******', pw=''):
    '''Executes the query to the OL db searching for the items recently changed.'''
    time.sleep(0.05)
    logger.info('Day %s: searching items...', day)
    db.setup_database(database='openlibrary', user='******', pw='')
    q = "SELECT key, last_modified FROM thing WHERE (type='17872418' OR type='9887992') AND last_modified >= '" + day + "' AND last_modified < date '" + day + "' + interval '1' day"
    rows = db.longquery(q, vars=locals())
    check_updates(rows, day)
Esempio n. 3
0
def search_updates_hourly(timestamp, database='openlibrary', user='******', pw=''):
    time.sleep(0.05)
    logger.info('Timestamp %s: searching items...',timestamp)
    db.setup_database(database='openlibrary', user='******', pw='')
    now = datetime.utcnow()
    now_str = now.strftime("%Y-%m-%d %H:%M:%S")
    q = "SELECT key, last_modified FROM thing WHERE (type='17872418' OR type='9887992') AND last_modified >= '"+timestamp+"' AND last_modified < date'"+now_str+"'"
    rows = db.longquery(q,vars=locals())
    check_updates(rows,now_str)
Esempio n. 4
0
def longquery(query, vars, callback):
    """Executes a long query using SQL cursors and passing a chunk of rows to the callback function in each iteration.
    """
    for rows in db.longquery(query, vars=vars):
        t = db.db.transaction()
        try:
            callback(rows)
        except:
            t.rollback()
            raise
        else:
            t.commit()
Esempio n. 5
0
def longquery(query, vars, callback):
    """Executes a long query using SQL cursors and passing a chunk of rows to the callback function in each iteration.
    """
    for rows in db.longquery(query, vars=vars):
        t = db.db.transaction()
        try:
            callback(rows)
        except:
            t.rollback()
            raise
        else:
            t.commit()
Esempio n. 6
0
def generate_idump(day, **db_parameters):
    """Generate incremental dump for the given day.
    """
    db.setup_database(**db_parameters)
    rows = db.longquery(
        "SELECT data.* FROM data, version, transaction " +
        " WHERE data.thing_id=version.thing_id" +
        "     AND data.revision=version.revision" +
        "     AND version.transaction_id=transaction.id" +
        "     AND transaction.created >= $day AND transaction.created < date $day + interval '1 day'"
        + " ORDER BY transaction.created",
        vars=locals(),
        chunk_size=10000)
    print_dump(row.data for chunk in rows for row in chunk)