def get_records_for_display(full_name):
""" Get records descriptions
full_name -- fully qualified name, in form db.coll
"""
try:
inv.setup_internal_client()
db = inv.int_client[inv.ALL_STRUC.name]
except Exception as e:
raise ih.ConnectOrAuthFail("")
return None
try:
parts = ih.get_description_key_parts(full_name)
records = retrieve_records(db, parts[0], parts[1])
except Exception as e:
inv.log_dest.error("Unable to get requested inventory " + str(e))
return {'data': None, 'scrape_date': None}
try:
return {
'data': ih.diff_records(records['data']),
'scrape_date': records['scrape_date']
}
except Exception as e:
inv.log_dest.error("Error decoding inventory object " + str(e))
return {'data': None, 'scrape_date': None}
def retrieve_db_listing(db_name=None):
"""Retrieve listing for all or given database."""
inv.setup_internal_client()
try:
db = inv.int_client[inv.ALL_STRUC.name]
except Exception:
raise ih.ConnectOrAuthFail("")
return None
return gen_retrieve_db_listing(db, db_name)
def retrieve_db_listing(db_name=None):
"""Retrieve listing for all or given database."""
inv.setup_internal_client()
try:
db = inv.int_client[inv.ALL_STRUC.name]
except Exception:
raise ih.ConnectOrAuthFail("")
return None
return gen_retrieve_db_listing(db, db_name)
def generate_report():
"""Generate a JSON document describing current state
of the inventory database.
Can take a while to run
"""
report = {}
try:
got_client = inv.setup_internal_client(editor=True, remote=False)
assert(got_client == True)
idb = inv.int_client
inv_db = inv.int_client[inv.get_inv_db_name()]
except Exception as e:
inv.log_dest.error("Error getting Db connection "+ str(e))
return False
#Check connection, editor status etc
report['connection'] = report_connection(idb, inv_db)
all_colls=list(inv_db['collection_ids'].find())
report['fields'] = report_fields_tables(inv_db, all_colls)
report['latest'] = report_latest_changes(all_colls)
report['gone'] = report_gone(all_colls)
report['scrapes'] = report_scrapes(inv_db)
report_record = {'isa':'report', 'scan_date': datetime.datetime.now(), 'report':report}
inv_db['ops'].insert_one(report_record)
def collate_orphans_by_uid(uid):
"""Fetch all orphans with given uid and return summary"""
try:
got_client = inv.setup_internal_client(editor=True)
assert(got_client == True)
inv_db = inv.int_client[inv.get_inv_db_name()]
except Exception as e:
inv.log_dest.error("Error getting Db connection "+ str(e))
return False
#All orphans records for this uid
record = {'uid':uuid.UUID(uid), 'orphans':{"$exists":True}}
records = inv_db['ops'].find(record)
orph_data = {}
db_name = ''
try:
db_name = idc.get_db_name(inv_db, records[0]['db'])['name']
except:
record = {'uid':uuid.UUID(uid)}
tmp_record = inv_db['ops'].find_one(record)
try:
db_name = idc.get_db_name(inv_db, tmp_record['db'])['name']
except:
pass
orph_data[db_name] = {}
for entry in records:
coll = idc.get_coll_name(inv_db, entry['coll'])['name']
orph_data[db_name][coll] = split_orphans(entry)
return orph_data
def get_progress(uid):
"""Get progress of scrape with uid"""
try:
got_client = inv.setup_internal_client(editor=True)
assert(got_client == True)
inv_db = inv.int_client[inv.get_inv_db_name()]
except Exception as e:
inv.log_dest.error("Error getting Db connection "+ str(e))
return False
#NOTE what follows _is_ vulnerable to races but
# this will only affect the progress meter, and should be rare
scrapes = inv_db['ops'].find({'uid':uuid.UUID(uid), 'running':{"$exists":True}})
#Assume all ops records with correct uid and containing 'running' are relevant
n_scrapes = scrapes.count()
curr_coll = 0
curr_item = None
for item in scrapes:
if item['complete'] : curr_coll = curr_coll + 1
if item['running'] : curr_item = item
if curr_item:
try:
prog_in_curr = get_progress_from_db(inv_db, uid, curr_item['db'], curr_item['coll'])
except Exception as e:
#Web front or user can't do anything about errors here. If process
# is failing, will become evident later
prog_in_curr = 0
else:
#Nothing running. If not yet started, prog=0. If done prog=100.
prog_in_curr = 100 * (curr_coll == n_scrapes)
return {'n_colls':n_scrapes, 'curr_coll':curr_coll, 'progress_in_current':prog_in_curr}
def collate_collection_info(db_name):
"""Fetches and collates viewable info for collections in named db
"""
try:
inv.setup_internal_client()
db = inv.int_client[inv.ALL_STRUC.name]
except Exception:
raise ih.ConnectOrAuthFail("")
return None
db_info = idc.get_db(db, db_name)
if not db_info['exist']:
return
colls_info = idc.get_all_colls(db, db_info['id'])
for coll in colls_info:
rec_info = idc.count_records_and_types(db, coll['_id'])
coll['records'] = comma(rec_info)
return colls_info
def is_valid_db_collection(db_name, collection_name):
"""Check if db and collection name (if not None) exist"""
try:
inv.setup_internal_client()
db = inv.int_client[inv.ALL_STRUC.name]
except Exception as e:
raise ih.ConnectOrAuthFail("")
return False
try:
db_id = idc.get_db_id(db, db_name)
if not db_id['exist']:
return False
if collection_name:
coll_id = idc.get_coll_id(db, db_id['id'], collection_name)
if not coll_id['exist']:
return False
except Exception as e:
inv.log_dest.error('Failed checking existence of '+db_name+' '+collection_name+' '+str(e))
return False
return True
def collate_collection_info(db_name):
"""Fetches and collates viewable info for collections in named db
"""
try:
inv.setup_internal_client()
db = inv.int_client[inv.ALL_STRUC.name]
except Exception:
raise ih.ConnectOrAuthFail("")
return None
db_info = idc.get_db(db, db_name)
if not db_info['exist']:
return
colls_info = idc.get_all_colls(db, db_info['id'])
for coll in colls_info:
rec_info = idc.count_records_and_types(db, coll['_id'])
coll['records'] = comma(rec_info)
return colls_info
def check_locks(resp):
"""Check if request pertains to locked coll
or editing is locked globally
"""
inv.setup_internal_client()
try:
db = inv.int_client[inv.ALL_STRUC.name]
except Exception:
raise ih.ConnectOrAuthFail("")
if get_lockout_state():
raise EditLockError('Global Edit Lock')
try:
db_name = resp['db']
coll_name = resp['collection']
db_id = idc.get_db_id(db, db_name)
coll_id = idc.get_coll_id(db, db_id['id'], coll_name)
if check_locked(db, coll_id['id']):
raise EditLockError('Collection locked')
except Exception as e:
inv.log_dest.error("Error in locking " + str(e))
raise e
def check_locks(resp):
"""Check if request pertains to locked coll
or editing is locked globally
"""
inv.setup_internal_client()
try:
db = inv.int_client[inv.ALL_STRUC.name]
except Exception:
raise ih.ConnectOrAuthFail("")
if get_lockout_state():
raise EditLockError('Global Edit Lock')
try:
db_name = resp['db']
coll_name = resp['collection']
db_id = idc.get_db_id(db, db_name)
coll_id = idc.get_coll_id(db, db_id['id'], coll_name)
if check_locked(db, coll_id['id']):
raise EditLockError('Collection locked')
except Exception as e:
inv.log_dest.error("Error in locking "+str(e))
raise e
def is_valid_db_collection(db_name, collection_name):
"""Check if db and collection name (if not None) exist"""
try:
inv.setup_internal_client()
db = inv.int_client[inv.ALL_STRUC.name]
except Exception as e:
raise ih.ConnectOrAuthFail("")
return False
try:
db_id = idc.get_db_id(db, db_name)
if not db_id['exist']:
return False
if collection_name:
coll_id = idc.get_coll_id(db, db_id['id'], collection_name)
if not coll_id['exist']:
return False
except Exception as e:
inv.log_dest.error('Failed checking existence of ' + db_name + ' ' +
collection_name + ' ' + str(e))
return False
return True
def update_gone_list():
"""Check for any colections that are gone and mark
"""
try:
got_client = inv.setup_internal_client(editor=True)
assert(got_client == True)
main_db = inv.int_client
except Exception as e:
inv.log_dest.error("Error getting Db connection "+ str(e))
return False
mark_all_gone(main_db)
return True
def remove_gone_collections():
"""Remove any collections marked as gone
"""
try:
got_client = inv.setup_internal_client(editor=True)
assert(got_client == True)
inv_db = inv.int_client[inv.get_inv_db_name()]
except Exception as e:
inv.log_dest.error("Error getting Db connection "+ str(e))
return False
remove_all_gone(inv_db)
return True
def get_nicename(db_name, collection_name):
"""Return the nice_name string for given db/coll pair"""
try:
inv.setup_internal_client()
db = inv.int_client[inv.ALL_STRUC.name]
except Exception as e:
raise ih.ConnectOrAuthFail("")
return None
try:
if collection_name:
db_id = idc.get_db_id(db, db_name)
coll_rec = idc.get_coll(db, db_id['id'], collection_name)
nice_name = coll_rec['data']['nice_name']
else:
db_rec = idc.get_db(db, db_name)
#print db_rec
nice_name = db_rec['data']['nice_name']
return nice_name
except Exception as e:
inv.log_dest.error('Failed to get nice name for '+db_name+' '+collection_name+' '+str(e))
#Can't return nice name so return None
return None
def get_latest_report():
try:
got_client = inv.setup_internal_client(editor=True, remote=False)
assert(got_client == True)
idb = inv.int_client
inv_db = idb[inv.get_inv_db_name()]
except Exception as e:
inv.log_dest.error("Error getting Db connection "+ str(e))
return False
reports = inv_db['ops'].find({'isa':'report', 'scan_date':{'$exists':True}})
sorted_reports = sorted(reports, key=lambda s : s['scan_date'])
return sorted_reports[-1]
def get_indices_for_display(full_name):
""" Get indices descriptions
full_name -- fully qualified name, in form db.coll
"""
try:
inv.setup_internal_client()
db = inv.int_client[inv.ALL_STRUC.name]
except Exception as e:
raise ih.ConnectOrAuthFail("")
return None
try:
parts = ih.get_description_key_parts(full_name)
records = retrieve_indices(db, parts[0], parts[1])
except Exception as e:
inv.log_dest.error("Unable to get requested inventory "+ str(e))
return {'data': None, 'scrape_date': None}
try:
return {'data':records['data'], 'scrape_date' : records['scrape_date']}
except Exception as e:
inv.log_dest.error("Error decoding inventory object "+ str(e))
return {'data': None, 'scrape_date': None}
def set_lockout_state(state):
"""Swap state of lockout. If record exists, toggle, else create"""
try:
got_client = inv.setup_internal_client(editor=True)
assert(got_client == True)
inv_db = inv.int_client[inv.get_inv_db_name()]
except Exception as e:
inv.log_dest.error("Error getting Db connection "+ str(e))
return True
try:
assert(state == True or state == False)
rec_set = {'lockout':state}
inv_db['ops'].insert_one(rec_set)
except:
inv.log_dest.error('Failed to set lockout state')
def null_old_scrapes(time=DEFAULT_MAX_TIME):
"""Update any old, incomplete AND not running scrapes to be 'complete'"""
try:
got_client = inv.setup_internal_client(editor=True)
assert(got_client == True)
inv_db = inv.int_client[inv.get_inv_db_name()]
except Exception as e:
inv.log_dest.error("Error getting Db connection "+ str(e))
return {'err':True, 'found':0}
lst = get_live_scrapes_older_than(inv_db, time)
new_lst = check_scrapes_running(inv_db, lst)
null_scrapes_by_list(inv_db, new_lst)
return {'err':False, 'found':len(new_lst)}
def get_nicename(db_name, collection_name):
"""Return the nice_name string for given db/coll pair"""
try:
inv.setup_internal_client()
db = inv.int_client[inv.ALL_STRUC.name]
except Exception as e:
raise ih.ConnectOrAuthFail("")
return None
try:
if collection_name:
db_id = idc.get_db_id(db, db_name)
coll_rec = idc.get_coll(db, db_id['id'], collection_name)
nice_name = coll_rec['data']['nice_name']
else:
db_rec = idc.get_db(db, db_name)
print db_rec
nice_name = db_rec['data']['nice_name']
return nice_name
except Exception as e:
inv.log_dest.error('Failed to get nice name for ' + db_name + ' ' +
collection_name + ' ' + str(e))
#Can't return nice name so return None
return None
def upload_scraped_data(structure_data, uid):
"""Main entry point for scraper tool
structure_data -- the json data
uid -- string uuid from scraper process start call
"""
try:
got_client = inv.setup_internal_client(editor=True)
assert (got_client == True)
inv_db = inv.int_client[inv.get_inv_db_name()]
except Exception as e:
inv.log_dest.error("Error getting Db connection " + str(e))
return False
inv.log_dest.warning('In upload with ' + str(uid))
upload_scraped_inventory(inv_db, structure_data, uid)
def upload_scraped_data(structure_data, uid):
"""Main entry point for scraper tool
structure_data -- the json data
uid -- string uuid from scraper process start call
"""
try:
got_client = inv.setup_internal_client(editor=True)
assert(got_client == True)
inv_db = inv.int_client[inv.get_inv_db_name()]
except Exception as e:
inv.log_dest.error("Error getting Db connection "+ str(e))
return False
inv.log_dest.warning('In upload with '+str(uid))
upload_scraped_inventory(inv_db, structure_data, uid)
def update_scrape_progress(db, coll, uid, complete=None, running=None):
"""Update progress of scrape from db/coll names and uid """
try:
got_client = inv.setup_internal_client(editor=True)
assert(got_client == True)
inv_db = inv.int_client[inv.get_inv_db_name()]
except Exception as e:
inv.log_dest.error("Error getting Db connection "+ str(e))
return False
try:
db_id = idc.get_db_id(inv_db, db)
coll_id = idc.get_coll_id(inv_db, db_id['id'], coll)
update_scrape_progress_helper(inv_db, db_id['id'], coll_id['id'], uid, complete=complete, running=running)
except Exception as e:
inv.log_dest.error("Error updating progress "+ str(e))
return False
def get_lockout_state():
"""Get lockout status"""
try:
got_client = inv.setup_internal_client(editor=True)
assert (got_client == True)
inv_db = inv.int_client[inv.get_inv_db_name()]
except Exception as e:
inv.log_dest.error("Error getting Db connection " + str(e))
return True
try:
rec_find = {'lockout': {"$exists": True}}
res = inv_db['ops'].find(rec_find).sort('_id', -1).limit(1)
except:
pass
if res is None:
return False
else:
return res[0]['lockout']
def get_lockout_state():
"""Get global lockout status"""
try:
got_client = inv.setup_internal_client(editor=True)
assert(got_client == True)
inv_db = inv.int_client[inv.get_inv_db_name()]
except Exception as e:
inv.log_dest.error("Error getting Db connection "+ str(e))
return True
try:
rec_find = {'lockout':{"$exists":True}}
#Get latest lockout record
res = inv_db['ops'].find(rec_find).sort('_id', -1).limit(1)
except:
pass
if res is None:
return False
else:
return res[0]['lockout']
def check_scrapes_on(spec):
"""If collection given, check for scrapes in progress or
queued on it. If only db, check all collections in it"""
try:
got_client = inv.setup_internal_client(editor=True)
assert (got_client == True)
inv_db = inv.int_client[inv.get_inv_db_name()]
except Exception as e:
inv.log_dest.error("Error getting Db connection " + str(e))
return False
try:
db_id = idc.get_db_id(inv_db, spec['db'])
spec_ids = {'db': db_id['id']}
if spec['coll']:
coll_id = idc.get_coll_id(inv_db, db_id['id'], spec['coll'])
spec_ids['coll'] = coll_id['id']
result = check_if_scraping(
inv_db, spec_ids) or check_if_scraping_queued(inv_db, spec_ids)
return result
except Exception as e:
return False
def register_scrape(db, coll, uid):
"""Create a suitable inventory entry for the scrape"""
inv.log_dest.warning(db + ' ' + coll + ' ' + str(uid))
try:
got_client = inv.setup_internal_client(editor=True)
assert (got_client == True)
inv_db = inv.int_client[inv.get_inv_db_name()]
except Exception as e:
inv.log_dest.error("Error getting Db connection " + str(e))
return {'err': True, 'inprog': False}
try:
db_id = idc.get_db_id(inv_db, db)
inv.log_dest.warning(str(db_id))
db_id = db_id['id']
inprog = False
had_err = False
if not coll:
all_colls = idc.get_all_colls(inv_db, db_id)
for coll in all_colls:
coll_id = coll['_id']
tmp = check_and_insert_scrape_record(inv_db, db_id, coll_id,
uid)
inprog = tmp['inprog'] and inprog
had_err = tmp['err'] and had_err
else:
coll_id = idc.get_coll_id(inv_db, db_id, coll)
inv.log_dest.warning(str(coll_id))
coll_id = coll_id['id']
tmp = check_and_insert_scrape_record(inv_db, db_id, coll_id, uid)
inprog = tmp['inprog'] and inprog
had_err = tmp['err'] and had_err
except Exception as e:
#Either failed to connect etc, or are already scraping
inv.log_dest.warning('Error resistering scrape ' + str(e))
return {'err': True, 'inprog': False}
return {'err': had_err, 'inprog': inprog}
def update_scrape_progress(db, coll, uid, complete=None, running=None):
"""Update progress of scrape from db/coll names and uid """
try:
got_client = inv.setup_internal_client(editor=True)
assert (got_client == True)
inv_db = inv.int_client[inv.get_inv_db_name()]
except Exception as e:
inv.log_dest.error("Error getting Db connection " + str(e))
return False
try:
db_id = idc.get_db_id(inv_db, db)
coll_id = idc.get_coll_id(inv_db, db_id['id'], coll)
update_scrape_progress_helper(inv_db,
db_id['id'],
coll_id['id'],
uid,
complete=complete,
running=running)
except Exception as e:
inv.log_dest.error("Error updating progress " + str(e))
return False
def null_all_scrapes(db, coll):
"""Update all scrapes on db.coll to be 'complete' """
try:
got_client = inv.setup_internal_client(editor=True)
assert(got_client == True)
inv_db = inv.int_client[inv.get_inv_db_name()]
except Exception as e:
inv.log_dest.error("Error getting Db connection "+ str(e))
return False
try:
db_id = idc.get_db_id(inv_db, db)
coll_id = idc.get_coll_id(inv_db, db_id['id'], coll)
rec_find = {'db':db_id['id'], 'coll':coll_id['id']}
rec_set = {}
rec_set['complete'] = True
rec_set['running'] = False
inv_db['ops'].update_many(rec_find, {"$set":rec_set})
except Exception as e:
inv.log_dest.error("Error updating progress "+ str(e))
return False
def collate_orphans():
"""Fetch all orphans and return summary"""
try:
got_client = inv.setup_internal_client(editor=True)
assert(got_client == True)
inv_db = inv.int_client[inv.get_inv_db_name()]
except Exception as e:
inv.log_dest.error("Error getting Db connection "+ str(e))
return False
#All orphans records
record = {'orphans':{"$exists":True}}
records = inv_db['ops'].find(record)
orph_data = {}
for entry in records:
#print entry['uid']
db_name = idc.get_db_name(inv_db, entry['db'])['name']
orph_data[db_name] = {}
coll = idc.get_coll_name(inv_db, entry['coll'])['name']
orph_tmp = split_orphans(entry)
orph_data[db_name][coll] = orph_tmp
return orph_data
def update_fields(diff, storeRollback=True):
"""Update a record from a diff object.
diff -- should be a fully qualified difference, containing db, collection names and then a list of changes, each being a dict containing the item, the field and the new content. Item corresponds to an entry in an object, field to the piece of information this specifies (for example, type, description, example)
e.g. {"db":"curve_automorphisms","collection":"passports","diffs":[{"item":"total_label","field":"type","content":"string"}]}
If this is a record entry, then the 'item' field will be a record hash.
storeRollback -- determine whether to store the undiff and diff to allow rollback of the change
"""
try:
got_client = inv.setup_internal_client(editor=True)
assert (got_client == True)
db = inv.int_client[inv.get_inv_db_name()]
except Exception as e:
inv.log_dest.error("Error getting Db connection " + str(e))
return
try:
if diff['collection'] is not None:
inv.log_dest.info("Updating descriptions for " + diff["db"] + '.' +
diff["collection"])
else:
inv.log_dest.info("Updating descriptions for " + diff["db"])
_id = idc.get_db_id(db, diff["db"])
rollback = None
try:
for change in diff["diffs"]:
if ih.is_special_field(change["item"]):
if storeRollback:
rollback = capture_rollback(db, _id['id'], diff["db"],
diff["collection"], change)
change["item"] = change["item"][
2:
-2] #Trim special fields. TODO this should be done better somehow
updated = idc.update_coll_data(db, _id['id'],
diff["collection"],
change["item"],
change["field"],
change["content"])
elif ih.is_toplevel_field(change["item"]):
#Here we have item == "toplevel", field the relevant field, and change the new value
if storeRollback:
rollback = capture_rollback(db, _id['id'], diff["db"],
diff["collection"], change)
#Only nice_name is currently an option
if (change["field"] not in ['nice_name', 'status']):
updated = {'err': True}
else:
if (diff["collection"]):
if (change['field']) == 'nice_name':
new_nice = change['content']
new_stat = None
else:
new_nice = None
new_stat = ih.status_to_code(change['content'])
c_id = idc.get_coll_id(db, _id['id'],
diff['collection'])
updated = idc.update_coll(db,
c_id['id'],
nice_name=new_nice,
status=new_stat)
else:
#Is database nice_name
updated = idc.update_db(
db, _id['id'], nice_name=change["content"])
else:
_c_id = idc.get_coll_id(db, _id['id'], diff["collection"])
if storeRollback:
rollback = capture_rollback(db,
_id['id'],
diff["db"],
diff["collection"],
change,
coll_id=_c_id['id'])
succeeded = False
#if it looks like a record, try treating as one
#If this fails try it as a field
if ih.is_probable_record_hash(change['item']):
updated = idc.update_record_description(
db, _c_id['id'], {
'hash': change["item"],
change["field"]: change["content"]
})
if updated['err'] == False:
succeeded = True
if not succeeded:
updated = idc.update_field(db,
_c_id['id'],
change["item"],
change["field"],
change["content"],
type="human")
if updated['err']:
raise KeyError("Cannot update, item not present")
else:
if storeRollback:
store_rollback(db, rollback)
except Exception as e:
inv.log_dest.error("Error applying diff " + str(change) + ' ' +
str(e))
raise UpdateFailed(str(e))
except Exception as e:
inv.log_dest.error("Error updating fields " + str(e))
"""Raise for errors not otherwise specified"""
errcode = 32
def __init__(self, message):
mess = "Unknown error "+message
super(RuntimeError, self).__init__(mess)
err_registry = {1:DiffKeyError(" "), 2:DiffBadType(""), 4:DiffDecodeError(""), 8:DiffCollideError(""), 16:EditLockError(""), 32:DiffUnknownError("")}
# End custom exceptions for diff validation
#End functions to deal with edit submissions -------------------------------------------------------
if __name__ == "__main__":
inv.setup_internal_client()
db = inv.int_client[inv.ALL_STRUC.name]
listing = get_edit_list()
for item in listing:
print(item)
coll_listing = gen_retrieve_db_listing(db, item)
print(" " + str(coll_listing))
#print json.dumps({'4': 5, '6': 7}, sort_keys=True,
#... indent=4, separators=(',', ': '))
import random
random.seed()
item = random.choice(listing)
coll_listing = gen_retrieve_db_listing(db, item)
def update_fields(diff, storeRollback=True):
"""Update a record from a diff object.
diff -- should be a fully qualified difference, containing db, collection names and then a list of changes, each being a dict containing the item, the field and the new content. Item corresponds to an entry in an object, field to the piece of information this specifies (for example, type, description, example)
e.g. {"db":"curve_automorphisms","collection":"passports","diffs":[{"item":"total_label","field":"type","content":"string"}]}
If this is a record entry, then the 'item' field will be a record hash.
storeRollback -- determine whether to store the undiff and diff to allow rollback of the change
"""
try:
got_client = inv.setup_internal_client(editor=True)
assert(got_client == True)
db = inv.int_client[inv.get_inv_db_name()]
except Exception as e:
inv.log_dest.error("Error getting Db connection "+ str(e))
return
try:
if diff['collection'] is not None:
inv.log_dest.info("Updating descriptions for " + diff["db"]+'.'+diff["collection"])
else:
inv.log_dest.info("Updating descriptions for " + diff["db"])
_id = idc.get_db_id(db, diff["db"])
rollback = None
try:
for change in diff["diffs"]:
if ih.is_special_field(change["item"]):
if storeRollback:
rollback = capture_rollback(db, _id['id'], diff["db"], diff["collection"], change)
change["item"] = change["item"][2:-2] #Trim special fields. TODO this should be done better somehow
updated = idc.update_coll_data(db, _id['id'], diff["collection"], change["item"], change["field"], change["content"])
elif ih.is_toplevel_field(change["item"]):
#Here we have item == "toplevel", field the relevant field, and change the new value
if storeRollback:
rollback = capture_rollback(db, _id['id'], diff["db"], diff["collection"], change)
#Only nice_name is currently an option
if(change["field"] not in ['nice_name', 'status']):
updated = {'err':True}
else:
if(diff["collection"]):
if(change['field']) == 'nice_name':
new_nice = change['content']
new_stat = None
else:
new_nice = None
new_stat = ih.status_to_code(change['content'])
c_id = idc.get_coll_id(db, _id['id'], diff['collection'])
updated = idc.update_coll(db, c_id['id'], nice_name=new_nice, status=new_stat)
else:
#Is database nice_name
updated = idc.update_db(db, _id['id'], nice_name=change["content"])
else:
_c_id = idc.get_coll_id(db, _id['id'], diff["collection"])
if storeRollback:
rollback = capture_rollback(db, _id['id'], diff["db"], diff["collection"], change, coll_id = _c_id['id'])
succeeded = False
#if it looks like a record, try treating as one
#If this fails try it as a field
if ih.is_probable_record_hash(change['item']):
updated = idc.update_record_description(db, _c_id['id'], {'hash':change["item"], change["field"]:change["content"]})
if updated['err'] == False:
succeeded = True;
if not succeeded:
updated = idc.update_field(db, _c_id['id'], change["item"], change["field"], change["content"], type="human")
if updated['err']:
raise KeyError("Cannot update, item not present")
else:
if storeRollback:
store_rollback(db, rollback)
except Exception as e:
inv.log_dest.error("Error applying diff "+ str(change)+' '+str(e))
raise UpdateFailed(str(e))
except Exception as e:
inv.log_dest.error("Error updating fields "+ str(e))
err_registry = {
1: DiffKeyError(" "),
2: DiffBadType(""),
4: DiffDecodeError(""),
8: DiffCollideError(""),
16: DiffUnknownError("")
}
# End custom exceptions for diff validation
#End functions to deal with edit submissions -------------------------------------------------------
if __name__ == "__main__":
inv.setup_internal_client()
db = inv.int_client[inv.ALL_STRUC.name]
listing = get_edit_list()
for item in listing:
print(item)
coll_listing = gen_retrieve_db_listing(db, item)
print(" " + str(coll_listing))
#print json.dumps({'4': 5, '6': 7}, sort_keys=True,
#... indent=4, separators=(',', ': '))
import random
random.seed()
item = random.choice(listing)
coll_listing = gen_retrieve_db_listing(db, item)
