def mark_all_gone():
"""Set status of all removed collections to gone"""
dbs = iv.gen_retrieve_db_listing()
all_colls = get_db_lists()
gone_code = ih.status_to_code('gone')
for db in dbs:
colls = iv.gen_retrieve_db_listing(db[0])
db_id = idc.get_db_id(db[0])
for coll in colls:
gone = not (coll[0] in all_colls[db[0]])
#Only mark if isn't already
mark = gone and coll[3] != 'gone'
if mark:
coll_id = idc.get_coll_id(db_id['id'], coll[0])
idc.update_coll(coll_id['id'], status=gone_code)
def update_gone_list():
"""Set status of all removed tables to gone"""
dbs = iv.retrieve_db_listing()
all_tables = get_db_lists()
gone_code = ih.status_to_code('gone')
for db_rec in dbs:
db_name = db_rec[0]
tables = iv.retrieve_db_listing(db_name)
#db_id = idc.get_db_id(db_name)
for table in tables:
table_name = table[0]
gone = not (table_name in all_tables[db_name])
#Only mark if isn't already
if gone and table[3] != 'gone':
table_id = idc.get_table_id(table_name)
idc.update_table(table_id, status=gone_code)
inv.log_dest.info(str(table_name) + ' is now gone')
def mark_all_gone(main_db):
"""Set status of all removed collections to gone"""
inv_db = main_db[inv.get_inv_db_name()]
dbs = iv.gen_retrieve_db_listing(inv_db)
all_colls = get_db_lists()
gone_code = ih.status_to_code('gone')
for db in dbs:
colls = iv.gen_retrieve_db_listing(inv_db, db[0])
db_id = idc.get_db_id(inv_db, db[0])
for coll in colls:
gone = not (coll[0] in all_colls[db[0]])
#Only mark if isn't already
mark = gone and coll[3] != 'gone'
if mark:
coll_id = idc.get_coll_id(inv_db, db_id['id'], coll[0])
idc.update_coll(inv_db, coll_id['id'], status=gone_code)
inv.log_dest.info(str(db) +'.'+str(coll) +' is now gone')
def update_gone_list():
"""Set status of all removed tables to gone"""
dbs = iv.retrieve_db_listing()
all_tables = get_db_lists()
gone_code = ih.status_to_code('gone')
for db_rec in dbs:
db_name = db_rec[0]
tables = iv.retrieve_db_listing(db_name)
#db_id = idc.get_db_id(db_name)
for table in tables:
table_name = table[0]
gone = not (table_name in all_tables[db_name])
#Only mark if isn't already
if gone and table[3] != 'gone':
table_id = idc.get_table_id(table_name)
idc.update_table(table_id, status=gone_code)
inv.log_dest.info(str(table_name) +' is now gone')
def update_fields(diff, storeRollback=True):
"""Update a record from a diff object.
diff -- should be a fully qualified difference, containing db, table names and then a list of changes, each being a dict containing the item, the field and the new content. Item corresponds to an entry in an object, field to the piece of information this specifies (for example, type, description, example)
e.g. {"db":"hgcwa","table":"hgcwa_passports","diffs":[{"item":"total_label","field":"type","content":"string"}]}
If this is a record entry, then the 'item' field will be a record hash.
storeRollback -- determine whether to store the undiff and diff to allow rollback of the change
"""
try:
if diff['table'] is not None:
inv.log_dest.info("Updating descriptions for " + diff["table"])
else:
inv.log_dest.info("Updating descriptions for " + diff["db"])
db_id = idc.get_db_id(diff["db"])
rollback = None
try:
for change in diff["diffs"]:
if ih.is_special_field(change["item"]):
if storeRollback:
rollback = capture_rollback(db_id, diff["db"], diff["table"], change)
change["item"] = change["item"][2:-2] #Trim special fields. TODO this should be done better somehow
updated = idc.update_table_data(db_id, diff["table"], change["item"], change["field"], change["content"])
elif ih.is_toplevel_field(change["item"]):
#Here we have item == "toplevel", field the relevant field, and change the new value
if storeRollback:
rollback = capture_rollback(db_id, diff["db"], diff["table"], change)
#Only nice_name is currently an option
if(change["field"] not in ['nice_name', 'status']):
updated = {'err':True}
else:
if(diff["table"]):
if(change['field']) == 'nice_name':
new_nice = change['content']
new_stat = None
else:
new_nice = None
new_stat = ih.status_to_code(change['content'])
table_id = idc.get_table_id(diff['table'])
updated = idc.update_table(table_id, nice_name=new_nice, status=new_stat)
else:
#Is database nice_name
updated = idc.update_db(db_id, nice_name=change["content"])
else:
table_id = idc.get_table_id(diff["table"])
if storeRollback:
rollback = capture_rollback(db_id, diff["db"], diff["table"], change, table_id = table_id)
updated = idc.update_field(table_id, change["item"], change["field"], change["content"], type="human")
if updated['err']:
raise KeyError("Cannot update, item not present")
else:
if storeRollback:
store_rollback(rollback)
except Exception as e:
inv.log_dest.error("Error applying diff "+ str(change)+' '+str(e))
raise UpdateFailed(str(e))
except Exception as e:
inv.log_dest.error("Error updating fields "+ str(e))
def update_fields(diff, storeRollback=True):
"""Update a record from a diff object.
diff -- should be a fully qualified difference, containing db, table names and then a list of changes, each being a dict containing the item, the field and the new content. Item corresponds to an entry in an object, field to the piece of information this specifies (for example, type, description, example)
e.g. {"db":"hgcwa","table":"hgcwa_passports","diffs":[{"item":"total_label","field":"type","content":"string"}]}
If this is a record entry, then the 'item' field will be a record hash.
storeRollback -- determine whether to store the undiff and diff to allow rollback of the change
"""
try:
if diff['table'] is not None:
inv.log_dest.info("Updating descriptions for " + diff["table"])
else:
inv.log_dest.info("Updating descriptions for " + diff["db"])
db_id = idc.get_db_id(diff["db"])
rollback = None
try:
for change in diff["diffs"]:
if ih.is_special_field(change["item"]):
if storeRollback:
rollback = capture_rollback(db_id, diff["db"],
diff["table"], change)
change["item"] = change["item"][
2:
-2] #Trim special fields. TODO this should be done better somehow
updated = idc.update_table_data(db_id, diff["table"],
change["item"],
change["field"],
change["content"])
elif ih.is_toplevel_field(change["item"]):
#Here we have item == "toplevel", field the relevant field, and change the new value
if storeRollback:
rollback = capture_rollback(db_id, diff["db"],
diff["table"], change)
#Only nice_name is currently an option
if (change["field"] not in ['nice_name', 'status']):
updated = {'err': True}
else:
if (diff["table"]):
if (change['field']) == 'nice_name':
new_nice = change['content']
new_stat = None
else:
new_nice = None
new_stat = ih.status_to_code(change['content'])
table_id = idc.get_table_id(diff['table'])
updated = idc.update_table(table_id,
nice_name=new_nice,
status=new_stat)
else:
#Is database nice_name
updated = idc.update_db(
db_id, nice_name=change["content"])
else:
table_id = idc.get_table_id(diff["table"])
if storeRollback:
rollback = capture_rollback(db_id,
diff["db"],
diff["table"],
change,
table_id=table_id)
updated = idc.update_field(table_id,
change["item"],
change["field"],
change["content"],
type="human")
if updated['err']:
raise KeyError("Cannot update, item not present")
else:
if storeRollback:
store_rollback(rollback)
except Exception as e:
inv.log_dest.error("Error applying diff " + str(change) + ' ' +
str(e))
raise UpdateFailed(str(e))
except Exception as e:
inv.log_dest.error("Error updating fields " + str(e))
def update_fields(diff, storeRollback=True):
"""Update a record from a diff object.
diff -- should be a fully qualified difference, containing db, collection names and then a list of changes, each being a dict containing the item, the field and the new content. Item corresponds to an entry in an object, field to the piece of information this specifies (for example, type, description, example)
e.g. {"db":"curve_automorphisms","collection":"passports","diffs":[{"item":"total_label","field":"type","content":"string"}]}
If this is a record entry, then the 'item' field will be a record hash.
storeRollback -- determine whether to store the undiff and diff to allow rollback of the change
"""
try:
_id = idc.get_db_id(diff["db"])
rollback = None
storeRollback = False
try:
for change in diff["diffs"]:
if ih.is_special_field(change["item"]):
if storeRollback:
rollback = capture_rollback(_id['id'], diff["db"], diff["collection"], change)
change["item"] = change["item"][2:-2] #Trim special fields. TODO this should be done better somehow
updated = idc.update_coll_data(_id['id'], diff["collection"], change["item"], change["field"], change["content"])
elif ih.is_toplevel_field(change["item"]):
#Here we have item == "toplevel", field the relevant field, and change the new value
if storeRollback:
rollback = capture_rollback(_id['id'], diff["db"], diff["collection"], change)
#Only nice_name is currently an option
print(change['field'])
if(change["field"] not in ['nice_name', 'status']):
updated = {'err':True}
else:
if(diff["collection"]):
if(change['field']) == 'nice_name':
new_nice = change['content']
new_stat = None
else:
new_nice = None
new_stat = ih.status_to_code(change['content'])
c_id = idc.get_coll_id(_id['id'], diff['collection'])
updated = idc.update_coll(c_id['id'], nice_name=new_nice, status=new_stat)
else:
#Is database nice_name
print(_id)
updated = idc.update_db(_id['id'], nice_name=change["content"])
else:
_c_id = idc.get_coll_id(_id['id'], diff["collection"])
if storeRollback:
rollback = capture_rollback(_id['id'], diff["db"], diff["collection"], change, coll_id = _c_id['id'])
succeeded = False
#if it looks like a record, try treating as one
#If this fails try it as a field
if ih.is_probable_record_hash(change['item']):
updated = idc.update_record_description(_c_id['id'], {'hash':change["item"], change["field"]:change["content"]})
if updated['err'] == False:
succeeded = True;
if not succeeded:
updated = idc.update_field(_c_id['id'], change["item"], change["field"], change["content"], type="human")
if updated['err']:
raise KeyError("Cannot update, item not present")
else:
if storeRollback:
store_rollback(rollback)
except Exception as e:
raise UpdateFailed(str(e))
except Exception as e:
#inv.log_dest.error("Error updating fields "+ str(e))
pass
def report_gone(tables):
gone_stat = ih.status_to_code('gone')
gone_tables = [item for item in tables if item['status'] == gone_stat]
return {'tables_gone' :{'num':len(gone_tables), 'items': gone_tables}}
def report_gone(colls):
gone_stat = ih.status_to_code('gone')
gone_colls = [item for item in colls if item['status'] == gone_stat]
return {'colls_gone': {'num': len(gone_colls), 'items': gone_colls}}
def update_fields(diff, storeRollback=True):
"""Update a record from a diff object.
diff -- should be a fully qualified difference, containing db, collection names and then a list of changes, each being a dict containing the item, the field and the new content. Item corresponds to an entry in an object, field to the piece of information this specifies (for example, type, description, example)
e.g. {"db":"curve_automorphisms","collection":"passports","diffs":[{"item":"total_label","field":"type","content":"string"}]}
If this is a record entry, then the 'item' field will be a record hash.
storeRollback -- determine whether to store the undiff and diff to allow rollback of the change
"""
try:
got_client = inv.setup_internal_client(editor=True)
assert(got_client == True)
db = inv.int_client[inv.get_inv_db_name()]
except Exception as e:
inv.log_dest.error("Error getting Db connection "+ str(e))
return
try:
if diff['collection'] is not None:
inv.log_dest.info("Updating descriptions for " + diff["db"]+'.'+diff["collection"])
else:
inv.log_dest.info("Updating descriptions for " + diff["db"])
_id = idc.get_db_id(db, diff["db"])
rollback = None
try:
for change in diff["diffs"]:
if ih.is_special_field(change["item"]):
if storeRollback:
rollback = capture_rollback(db, _id['id'], diff["db"], diff["collection"], change)
change["item"] = change["item"][2:-2] #Trim special fields. TODO this should be done better somehow
updated = idc.update_coll_data(db, _id['id'], diff["collection"], change["item"], change["field"], change["content"])
elif ih.is_toplevel_field(change["item"]):
#Here we have item == "toplevel", field the relevant field, and change the new value
if storeRollback:
rollback = capture_rollback(db, _id['id'], diff["db"], diff["collection"], change)
#Only nice_name is currently an option
if(change["field"] not in ['nice_name', 'status']):
updated = {'err':True}
else:
if(diff["collection"]):
if(change['field']) == 'nice_name':
new_nice = change['content']
new_stat = None
else:
new_nice = None
new_stat = ih.status_to_code(change['content'])
c_id = idc.get_coll_id(db, _id['id'], diff['collection'])
updated = idc.update_coll(db, c_id['id'], nice_name=new_nice, status=new_stat)
else:
#Is database nice_name
updated = idc.update_db(db, _id['id'], nice_name=change["content"])
else:
_c_id = idc.get_coll_id(db, _id['id'], diff["collection"])
if storeRollback:
rollback = capture_rollback(db, _id['id'], diff["db"], diff["collection"], change, coll_id = _c_id['id'])
succeeded = False
#if it looks like a record, try treating as one
#If this fails try it as a field
if ih.is_probable_record_hash(change['item']):
updated = idc.update_record_description(db, _c_id['id'], {'hash':change["item"], change["field"]:change["content"]})
if updated['err'] == False:
succeeded = True;
if not succeeded:
updated = idc.update_field(db, _c_id['id'], change["item"], change["field"], change["content"], type="human")
if updated['err']:
raise KeyError("Cannot update, item not present")
else:
if storeRollback:
store_rollback(db, rollback)
except Exception as e:
inv.log_dest.error("Error applying diff "+ str(change)+' '+str(e))
raise UpdateFailed(str(e))
except Exception as e:
inv.log_dest.error("Error updating fields "+ str(e))
def report_gone(colls):
gone_stat = ih.status_to_code('gone')
gone_colls = [item for item in colls if item['status'] == gone_stat]
return {'colls_gone' :{'num':len(gone_colls), 'items': gone_colls}}
