def check_scrapes_running(scrape_list):
"""Given a list of scrapes, check for actual running state and
return a new list containing only those which are NOT running"""
new_list = []
for item in scrape_list:
try:
db_name = idc.get_db_name(item['db'])['name']
coll_name = idc.get_coll_name(item['table'])['name']
prog = get_scrape_progress(db_name, coll_name)
if prog == (-1, -1):
new_list.append(item)
except:
pass
return new_list
def check_scrapes_running(scrape_list):
"""Given a list of scrapes, check for actual running state and
return a new list containing only those which are NOT running"""
new_list = []
for item in scrape_list:
db_name = idc.get_db_name(item['db'])
table_name = idc.get_table_name(item['table'])
try:
prog = get_scrape_progress(db_name, table_name)
if prog == (-1, -1):
new_list.append(item)
except Exception as e:
inv.log_dest.warning('Failed to get progress '+db_name+' '+table_name+' '+str(e))
return new_list
def check_scrapes_running(inv_db, scrape_list):
"""Given a list of scrapes, check for actual running state and
return a new list containing only those which are NOT running"""
new_list = []
for item in scrape_list:
try:
db_name = idc.get_db_name(inv_db, item['db'])['name']
coll_name = idc.get_coll_name(inv_db, item['coll'])['name']
prog = get_scrape_progress(db_name, coll_name, getDBConnection())
if prog == (-1, -1):
new_list.append(item)
except Exception as e:
inv.log_dest.warning('Failed to get progress '+db_name+' '+coll_name+' '+str(e))
return new_list
def get_progress_from_db(uid, db_id, coll_id):
"""Query db to see state of current scrape
NOTE: this function assumed that when it is called there was a running
process on db.coll, so if there no longer is, it must have finished
"""
db_name = idc.get_db_name(db_id)['name']
coll_name = idc.get_coll_name(coll_id)['name']
try:
live_progress = sf.get_scrape_progress(db_name, coll_name)
#Cheat here: we'll cap running to 99% and the last 1% is left for upload time
#If no running record found, this assumes it completed before
#we managed to check it, hence 99%
percent = (live_progress[0] *99)/live_progress[1]
except Exception as e:
percent = 0
raise e
return percent
def get_progress_from_db(uid, db_id, table_id):
"""Query db to see state of current scrape
NOTE: this function assumed that when it is called there was a running
process on db.table, so if there no longer is, it must have finished
"""
#db_name = idc.get_db_name(db_id)
table_name = idc.get_table_name(table_id)
try:
live_progress = sf.get_scrape_progress(table_name)
#Cheat here: we'll cap running to 99% and the last 1% is left for upload time
#If no running record found, this assumes it completed before
#we managed to check it, hence 99%
percent = (live_progress[0] *99)/live_progress[1]
except Exception as e:
inv.log_dest.warning(e)
percent = 0
raise e
return percent