def register_scrape(db, table, uid): """Create a suitable inventory entry for the scrape""" inv.log_dest.warning(db+' '+table+' '+str(uid)) try: db_id = idc.get_db_id(db) inv.log_dest.warning(str(db_id)) inprog = False had_err = False if not table: all_tables = idc.get_all_tables(db_id) for table in all_tables: table_id = table['_id'] tmp = check_and_insert_scrape_record(db_id, table_id, uid) inprog = tmp['inprog'] and inprog had_err = tmp['err'] and had_err else: table_id = idc.get_table_id(table) inv.log_dest.warning(str(table_id)) tmp = check_and_insert_scrape_record(db_id, table_id, uid) inprog = tmp['inprog'] and inprog had_err = tmp['err'] and had_err except Exception as e: #Either failed to connect etc, or are already scraping inv.log_dest.warning('Error resistering scrape '+str(e)) return {'err':True, 'inprog':False} return {'err':had_err, 'inprog':inprog}
def check_scrapes_on(spec=None): """If table given, check for scrapes in progress or queued on it. If only db, check all tables in it. If spec is None, check everything""" spec_ids = {} if spec is not None: if spec.get('db'): spec_ids['db'] = idc.get_db_id(spec['db']) if spec.get('table'): spec_ids['table'] = idc.get_table_id(spec['table']) return check_if_scraping(spec_ids) or check_if_scraping_queued(spec_ids)
def null_all_scrapes(table_name): """Update all scrapes on table to be 'complete' """ try: table_id = idc.get_table_id(table_name) rec_find = {'table_id':table_id} rec_set = {'complete':True, 'running':False} db.inv_ops.update(rec_find, rec_set) except Exception as e: inv.log_dest.error("Error updating progress "+ str(e)) return False
def is_valid_db_table(db_name, table_name): """Check if db and table_name name (if not None) exist""" try: db_id = idc.get_db_id(db_name) if db_id is None: return False if table_name: table_id = idc.get_table_id(table_name) if table_id is None: return False except Exception as e: inv.log_dest.error('Failed checking existence of '+db_name+' '+str(table_name)+' '+str(e)) return False return True
def is_valid_db_table(db_name, table_name): """Check if db and table_name name (if not None) exist""" try: db_id = idc.get_db_id(db_name) if db_id is None: return False if table_name: table_id = idc.get_table_id(table_name) if table_id is None: return False except Exception as e: inv.log_dest.error('Failed checking existence of '+db_name+' '+str(table_name)+' '+str(e)) return False return True
def update_scrape_progress(db_name, table_name, uid, complete=None, running=None): """Update progress of scrape from db/table names and uid """ try: db_id = idc.get_db_id(db_name) table_id = idc.get_table_id(table_name) rec_find = {'db_id':db_id, 'table_id':table_id, 'uid':uid} rec_set = {} if complete is not None: rec_set['complete'] = complete if running is not None: rec_set['running'] = running if rec_set: db.inv_ops.update(rec_find, rec_set) except Exception as e: inv.log_dest.error("Error updating progress "+ str(e)) return False
def check_locks(resp): """Check if request pertains to locked table or editing is locked globally """ if get_lockout_state(): raise EditLockError('Global Edit Lock') try: #db_name = resp['db'] table_name = resp['table'] #db_id = idc.get_db_id(db_name) table_id = idc.get_table_id(table_name) if check_locked(table_id): raise EditLockError('Table locked') except Exception as e: inv.log_dest.error("Error in locking "+str(e)) raise e
def check_locks(resp): """Check if request pertains to locked table or editing is locked globally """ if get_lockout_state(): raise EditLockError('Global Edit Lock') try: #db_name = resp['db'] table_name = resp['table'] #db_id = idc.get_db_id(db_name) table_id = idc.get_table_id(table_name) if check_locked(table_id): raise EditLockError('Table locked') except Exception as e: inv.log_dest.error("Error in locking "+str(e)) raise e
def delete_by_table(db_name, table_name): """Remove table entry and all its fields""" try: table_id = invc.get_table_id(table_name) except Exception as e: inv.log_dest.error("Error getting table " + str(e)) return {'err':True, 'id':0, 'exist':False} #Remove fields entries matching table_id delete_table_data(table_id, tbl='auto') delete_table_data(table_id, tbl='human') delete_table_data(table_id, tbl='records') try: db[inv.ALL_STRUC.table_ids[inv.STR_NAME]].delete({'_id':table_id}) except Exception as e: inv.log_dest.error("Error removing table " + str(e))
def update_gone_list(): """Set status of all removed tables to gone""" dbs = iv.retrieve_db_listing() all_tables = get_db_lists() gone_code = ih.status_to_code('gone') for db_rec in dbs: db_name = db_rec[0] tables = iv.retrieve_db_listing(db_name) #db_id = idc.get_db_id(db_name) for table in tables: table_name = table[0] gone = not (table_name in all_tables[db_name]) #Only mark if isn't already if gone and table[3] != 'gone': table_id = idc.get_table_id(table_name) idc.update_table(table_id, status=gone_code) inv.log_dest.info(str(table_name) + ' is now gone')
def update_gone_list(): """Set status of all removed tables to gone""" dbs = iv.retrieve_db_listing() all_tables = get_db_lists() gone_code = ih.status_to_code('gone') for db_rec in dbs: db_name = db_rec[0] tables = iv.retrieve_db_listing(db_name) #db_id = idc.get_db_id(db_name) for table in tables: table_name = table[0] gone = not (table_name in all_tables[db_name]) #Only mark if isn't already if gone and table[3] != 'gone': table_id = idc.get_table_id(table_name) idc.update_table(table_id, status=gone_code) inv.log_dest.info(str(table_name) +' is now gone')
def update_scrape_progress(db_name, table_name, uid, complete=None, running=None): """Update progress of scrape from db/table names and uid """ try: db_id = idc.get_db_id(db_name) table_id = idc.get_table_id(table_name) rec_find = {'db_id': db_id, 'table_id': table_id, 'uid': uid} rec_set = {} if complete is not None: rec_set['complete'] = complete if running is not None: rec_set['running'] = running if rec_set: db.inv_ops.update(rec_find, rec_set) except Exception as e: inv.log_dest.error("Error updating progress " + str(e)) return False
def upload_scraped_inventory(structure_dat, uid): """Upload a json structure document and store any oprhans structure_dat -- JSON document containing all db/tables to upload uid -- UID string for uploading process """ inv.log_dest.info("_____________________________________________________________________________________________") n_dbs = len(structure_dat.keys()) progress_tracker = 0 for db_name in structure_dat: progress_tracker += 1 inv.log_dest.info("Uploading "+db_name+" ("+str(progress_tracker)+" of "+str(n_dbs)+')') invc.set_db(db_name, db_name) for table_name in structure_dat[db_name]: inv.log_dest.info(" Uploading table "+table_name) orphaned_keys = upload_table_structure(db_name, table_name, structure_dat, fresh=False) if len(orphaned_keys) != 0: db_id = invc.get_db_id(db_name) table_id = invc.get_table_id(table_name) ild.store_orphans(db_id, table_id, uid, orphaned_keys)
def update_fields(diff, storeRollback=True): """Update a record from a diff object. diff -- should be a fully qualified difference, containing db, table names and then a list of changes, each being a dict containing the item, the field and the new content. Item corresponds to an entry in an object, field to the piece of information this specifies (for example, type, description, example) e.g. {"db":"hgcwa","table":"hgcwa_passports","diffs":[{"item":"total_label","field":"type","content":"string"}]} If this is a record entry, then the 'item' field will be a record hash. storeRollback -- determine whether to store the undiff and diff to allow rollback of the change """ try: if diff['table'] is not None: inv.log_dest.info("Updating descriptions for " + diff["table"]) else: inv.log_dest.info("Updating descriptions for " + diff["db"]) db_id = idc.get_db_id(diff["db"]) rollback = None try: for change in diff["diffs"]: if ih.is_special_field(change["item"]): if storeRollback: rollback = capture_rollback(db_id, diff["db"], diff["table"], change) change["item"] = change["item"][2:-2] #Trim special fields. TODO this should be done better somehow updated = idc.update_table_data(db_id, diff["table"], change["item"], change["field"], change["content"]) elif ih.is_toplevel_field(change["item"]): #Here we have item == "toplevel", field the relevant field, and change the new value if storeRollback: rollback = capture_rollback(db_id, diff["db"], diff["table"], change) #Only nice_name is currently an option if(change["field"] not in ['nice_name', 'status']): updated = {'err':True} else: if(diff["table"]): if(change['field']) == 'nice_name': new_nice = change['content'] new_stat = None else: new_nice = None new_stat = ih.status_to_code(change['content']) table_id = idc.get_table_id(diff['table']) updated = idc.update_table(table_id, nice_name=new_nice, status=new_stat) else: #Is database nice_name updated = idc.update_db(db_id, nice_name=change["content"]) else: table_id = idc.get_table_id(diff["table"]) if storeRollback: rollback = capture_rollback(db_id, diff["db"], diff["table"], change, table_id = table_id) updated = idc.update_field(table_id, change["item"], change["field"], change["content"], type="human") if updated['err']: raise KeyError("Cannot update, item not present") else: if storeRollback: store_rollback(rollback) except Exception as e: inv.log_dest.error("Error applying diff "+ str(change)+' '+str(e)) raise UpdateFailed(str(e)) except Exception as e: inv.log_dest.error("Error updating fields "+ str(e))
def upload_table_structure(db_name, table_name, structure_dat, fresh=False): """Upload the structure description for a single table Any entered descriptions for keys which still exist are preserved. Removed or renamed keys will be returned for handling Table entry is created if it doesn't exist, in which case Notes and Info are filled with dummies db_name -- Name of database (must exist) table_name -- Name of table to upload structure_dat -- lmfdb db structure as json object """ dummy_info = {} #Dummy per table info, containing basic fields we want included for field in inv.info_editable_fields: dummy_info[field] = None try: table_entry = structure_dat[table_name] db_entry = invc.get_db_id(db_name) if db_entry is None: #All dbs should have been added from the struc: if not is error inv.log_dest.error("ERROR: No inventory DB entry "+ db_name) inv.log_dest.error("Cannot add descriptions") return [] table_id = invc.get_table_id(table_name) if table_id is None: #Table doesn't exist, create it table_id = invc.set_table(db_entry, table_name, table_name, None, dummy_info, 0) else: #Delete existing auto-table entries (no table => no entries) delete_table_data(table_id, tbl='auto') try: scrape_date = datetime.datetime.strptime(structure_dat[db_name][table_name]['scrape_date'], '%Y-%m-%d %H:%M:%S.%f') except Exception as e: inv.log_dest.info("Scrape date parsing failed "+str(e)) scrape_date = datetime.datetime.min invc.set_table_scrape_date(table_id, scrape_date) except Exception as e: inv.log_dest.error("Failed to refresh table (db, table or scrape data) "+str(e)) try: for field in table_entry['fields']: inv.log_dest.info(" Processing "+field) invc.set_field(table_id, field, table_entry['fields'][field]) for record in table_entry['records']: inv.log_dest.info(" Processing record "+str(record)) invc.set_record(table_id, table_entry['records'][record]) #Cleanup any records which no longer exist invc.cleanup_records(db, table_id, table_entry['records']) inv.log_dest.info(" Processing indices") #FIXME upload_indices(db, table_id, table_entry['indices']) except Exception as e: inv.log_dest.error("Failed to refresh table entries "+str(e)) orphaned_keys = [] if not fresh: try: #Trim any human table keys which are now redundant orphaned_keys = invc.trim_human_table(db_entry, table_id) except Exception as e: inv.log_dest.error("Failed trimming table "+str(e)) #Ensure everything mandatory is present in human table try: invc.complete_human_table(db_entry, table_id) except Exception as e: inv.log_dest.error("Failed padding table "+str(e)) return orphaned_keys
def update_fields(diff, storeRollback=True): """Update a record from a diff object. diff -- should be a fully qualified difference, containing db, table names and then a list of changes, each being a dict containing the item, the field and the new content. Item corresponds to an entry in an object, field to the piece of information this specifies (for example, type, description, example) e.g. {"db":"hgcwa","table":"hgcwa_passports","diffs":[{"item":"total_label","field":"type","content":"string"}]} If this is a record entry, then the 'item' field will be a record hash. storeRollback -- determine whether to store the undiff and diff to allow rollback of the change """ try: if diff['table'] is not None: inv.log_dest.info("Updating descriptions for " + diff["table"]) else: inv.log_dest.info("Updating descriptions for " + diff["db"]) db_id = idc.get_db_id(diff["db"]) rollback = None try: for change in diff["diffs"]: if ih.is_special_field(change["item"]): if storeRollback: rollback = capture_rollback(db_id, diff["db"], diff["table"], change) change["item"] = change["item"][ 2: -2] #Trim special fields. TODO this should be done better somehow updated = idc.update_table_data(db_id, diff["table"], change["item"], change["field"], change["content"]) elif ih.is_toplevel_field(change["item"]): #Here we have item == "toplevel", field the relevant field, and change the new value if storeRollback: rollback = capture_rollback(db_id, diff["db"], diff["table"], change) #Only nice_name is currently an option if (change["field"] not in ['nice_name', 'status']): updated = {'err': True} else: if (diff["table"]): if (change['field']) == 'nice_name': new_nice = change['content'] new_stat = None else: new_nice = None new_stat = ih.status_to_code(change['content']) table_id = idc.get_table_id(diff['table']) updated = idc.update_table(table_id, nice_name=new_nice, status=new_stat) else: #Is database nice_name updated = idc.update_db( db_id, nice_name=change["content"]) else: table_id = idc.get_table_id(diff["table"]) if storeRollback: rollback = capture_rollback(db_id, diff["db"], diff["table"], change, table_id=table_id) updated = idc.update_field(table_id, change["item"], change["field"], change["content"], type="human") if updated['err']: raise KeyError("Cannot update, item not present") else: if storeRollback: store_rollback(rollback) except Exception as e: inv.log_dest.error("Error applying diff " + str(change) + ' ' + str(e)) raise UpdateFailed(str(e)) except Exception as e: inv.log_dest.error("Error updating fields " + str(e))