def search_updates(day, database='openlibrary', user='******', pw=''):
time.sleep(0.05)
logger.info('Day %s: searching items...',day)
db.setup_database(database='openlibrary', user='******', pw='')
q = "SELECT key, last_modified FROM thing WHERE (type='17872418' OR type='9887992') AND last_modified >= '"+day+"' AND last_modified < date '"+day+"' + interval '1' day"
rows = db.longquery(q,vars=locals())
check_updates(rows,day)
def search_updates(day, database='openlibrary', user='******', pw=''):
'''Executes the query to the OL db searching for the items recently changed.'''
time.sleep(0.05)
logger.info('Day %s: searching items...', day)
db.setup_database(database='openlibrary', user='******', pw='')
q = "SELECT key, last_modified FROM thing WHERE (type='17872418' OR type='9887992') AND last_modified >= '" + day + "' AND last_modified < date '" + day + "' + interval '1' day"
rows = db.longquery(q, vars=locals())
check_updates(rows, day)
def search_updates_hourly(timestamp, database='openlibrary', user='******', pw=''):
time.sleep(0.05)
logger.info('Timestamp %s: searching items...',timestamp)
db.setup_database(database='openlibrary', user='******', pw='')
now = datetime.utcnow()
now_str = now.strftime("%Y-%m-%d %H:%M:%S")
q = "SELECT key, last_modified FROM thing WHERE (type='17872418' OR type='9887992') AND last_modified >= '"+timestamp+"' AND last_modified < date'"+now_str+"'"
rows = db.longquery(q,vars=locals())
check_updates(rows,now_str)
def longquery(query, vars, callback):
"""Executes a long query using SQL cursors and passing a chunk of rows to the callback function in each iteration.
"""
for rows in db.longquery(query, vars=vars):
t = db.db.transaction()
try:
callback(rows)
except:
t.rollback()
raise
else:
t.commit()
def longquery(query, vars, callback):
"""Executes a long query using SQL cursors and passing a chunk of rows to the callback function in each iteration.
"""
for rows in db.longquery(query, vars=vars):
t = db.db.transaction()
try:
callback(rows)
except:
t.rollback()
raise
else:
t.commit()
def generate_idump(day, **db_parameters):
"""Generate incremental dump for the given day.
"""
db.setup_database(**db_parameters)
rows = db.longquery(
"SELECT data.* FROM data, version, transaction " +
" WHERE data.thing_id=version.thing_id" +
" AND data.revision=version.revision" +
" AND version.transaction_id=transaction.id" +
" AND transaction.created >= $day AND transaction.created < date $day + interval '1 day'"
+ " ORDER BY transaction.created",
vars=locals(),
chunk_size=10000)
print_dump(row.data for chunk in rows for row in chunk)