def prune(self, mode=None, doit=False):
# removes all unreferenced nodes
assert mode in [None, u'mon', u'device', u'location', u'config']
removed = True
# set of already deleted leafs
_deleted = set()
while removed:
removed = False
del_nodes = []
for cur_leaf in self.__node_dict.itervalues():
if cur_leaf not in _deleted:
if mode is None or cur_leaf.full_name.startswith(
"/{}".format(mode)):
if not cur_leaf.immutable:
# count related models (with m2m)
if not get_related_models(
cur_leaf, m2m=True, ignore_objs=_deleted):
del_nodes.append(cur_leaf)
for del_node in del_nodes:
# store idx in an extra field
del_node.saved_pk = del_node.idx
_deleted.add(del_node)
if doit:
del self[del_node.parent_id]._sub_tree[del_node.name]
del self.__node_dict[del_node.pk]
del_node.delete()
removed = len(del_nodes) > 0
return _deleted
def _check_special_commands(self):
from initat.md_config_server.special_commands import SPECIAL_DICT
pks_found = set()
mccs_dict = {}
for _name, _entry in SPECIAL_DICT.iteritems():
_inst = _entry(self.log)
if "special_{}".format(_inst.Meta.name) != _name:
self.log(
"special {} has illegal name {}".format(
_name, _inst.Meta.name),
logging_tools.LOG_LEVEL_CRITICAL)
else:
self.log("found special {}".format(_name))
cur_mccs = self._check_mccs(_inst.Meta)
mccs_dict[cur_mccs.name] = cur_mccs
pks_found.add(cur_mccs.pk)
if cur_mccs.meta:
for _sub_com in _inst.get_commands():
sub_mccs = self._check_mccs(_sub_com.Meta,
parent=cur_mccs)
mccs_dict[sub_mccs.name] = sub_mccs
pks_found.add(sub_mccs.pk)
# delete stale
del_mccs = mon_check_command_special.objects.exclude(pk__in=pks_found)
if del_mccs:
for _del_mcc in del_mccs:
self.log("trying to removing stale {}...".format(
unicode(_del_mcc), ))
_refs = get_related_models(_del_mcc)
if _refs:
self.log(
" unable to remove because referenced {}".format(
logging_tools.get_plural("time", _refs)),
logging_tools.LOG_LEVEL_ERROR)
else:
_del_mcc.delete()
self.log(" ...done")
# rewrite
for to_rewrite in mon_check_command.objects.filter(
Q(name__startswith="@")):
self.log("rewriting {} to new format... ".format(
unicode(to_rewrite)))
_key = to_rewrite.name.split("@")[1].lower()
if _key in mccs_dict:
to_rewrite.name = to_rewrite.name.split("@")[2]
to_rewrite.mon_check_command_special = mccs_dict[_key]
to_rewrite.save()
else:
self.log("key {} not found in dict".format(_key),
logging_tools.LOG_LEVEL_ERROR)
def post(self, request):
# returns:
# - related_objs: {obj_ok : [related_obj_info] } for objects which have related objects
# - deletable_objects: [obj_pk]
_post = request.POST
obj_pks = json.loads(_post.get("obj_pks"))
model = getattr(initat.cluster.backbone.models, _post.get("model"))
objs_to_delete = model.objects.filter(pk__in=obj_pks)
if len(objs_to_delete) < len(obj_pks):
request.xml_response.error("Could not find all objects to delete.")
logger.warn("To delete: {}; found only: {}".format(
obj_pks, [o.pk for o in objs_to_delete]))
related_objects_info = {}
deletable_objects = []
# import time
for obj_to_delete in objs_to_delete:
# a = time.time()
can_delete_answer = can_delete_obj(obj_to_delete, logger)
# print 'can del took ', time.time() - a, bool(can_delete_answer), len(can_delete_answer.related_objects)
if can_delete_answer:
deletable_objects.append(obj_to_delete.pk)
else:
info = []
# there are django related objects, which describe the fields that are related
# and there are referenced objects, which are the actual db objects having these fields
for related_object in can_delete_answer.related_objects:
referenced_objects_list = []
refs_of_refs = set()
for referenced_object in related_object.ref_list:
referenced_objects_list.append({
k: v
for (k,
v) in referenced_object.__dict__.iteritems()
if k != '_state'
})
refs_of_refs.update(
get_related_models(referenced_object, detail=True))
info.append({
'model':
related_object.model._meta.object_name,
'model_verbose_name':
related_object.model._meta.verbose_name.capitalize(),
'field_name':
related_object.field.name,
'field_verbose_name':
related_object.field.verbose_name.capitalize(),
'null':
related_object.field.null,
'objects': {
'num_refs_of_refs': len(refs_of_refs),
'list': referenced_objects_list,
},
})
related_objects_info[obj_to_delete.pk] = info
# print 'build 2nd level rel list', time.time() - a
# print 'obj', obj_pk, ' took ', time.time() - a
# json can't deal with datetime, django formatter doesn't have nice dates
def formatter(x):
if isinstance(x, datetime.datetime):
return x.strftime("%Y-%m-%d %H:%M")
elif isinstance(x, datetime.date):
# NOTE: datetime is instance of date, so check datetime first
return x.isoformat()
else:
return x
request.xml_response['related_objects'] = json.dumps(
related_objects_info, default=formatter)
request.xml_response['deletable_objects'] = json.dumps(
deletable_objects)
def post(self, request):
from initat.cluster.backbone.models.functions import can_delete_obj, get_related_models
# returns:
# - related_objs: {obj_ok : [related_obj_info] } for objects which have related objects
# - deletable_objects: [obj_pk]
_post = request.POST
_start_time = time.time()
obj_pks = json.loads(_post.get("obj_pks"))
model = apps.get_model("backbone", _post.get("model_name"))
objs_to_delete = model.objects.filter(pk__in=obj_pks)
if len(objs_to_delete) < len(obj_pks):
request.xml_response.error("Could not find all objects to delete.")
logger.warning(
"To delete: {}; found only: {}".format(
obj_pks,
[o.pk for o in objs_to_delete]
)
)
related_objects_info = {}
deletable_objects = []
# import time
for obj_to_delete in objs_to_delete:
# a = time.time()
can_delete_answer = can_delete_obj(obj_to_delete, logger)
# print 'can del took ', time.time() - a, bool(can_delete_answer), len(can_delete_answer.related_objects)
if can_delete_answer: # or True:
deletable_objects.append(obj_to_delete.pk)
else:
info = []
# there are django related objects, which describe the fields that are related
# and there are referenced objects, which are the actual db objects having these fields
for related_object in can_delete_answer.related_objects:
referenced_objects_list = []
refs_of_refs = set()
for referenced_object in related_object.ref_list:
# add simple representations of referenced objects
# ignore all fields starting with underscore '_' (for example '_state')
referenced_objects_list.append(
{
k: v for (k, v) in referenced_object.__dict__.items() if not k.startswith("_")
}
)
refs_of_refs.update(get_related_models(referenced_object, simple_tuples=True))
# print("*", refs_of_refs)
info.append(
{
'model': related_object.model._meta.object_name,
'model_verbose_name': related_object.model._meta.verbose_name.capitalize(),
'related_model': related_object.related_model._meta.object_name,
'related_model_verbose_name': related_object.related_model._meta.verbose_name.capitalize(),
'field_name': related_object.field.name,
'field_verbose_name': related_object.field.verbose_name.capitalize(),
'null': related_object.field.null,
'objects': {
'num_refs_of_refs': len(refs_of_refs),
'list': referenced_objects_list,
},
}
)
related_objects_info[obj_to_delete.pk] = {
"list": info,
}
# print 'build 2nd level rel list', time.time() - a
# print 'obj', obj_pk, ' took ', time.time() - a
_end_time = time.time()
# json can't deal with datetime, django formatter doesn't have nice dates
def formatter(val):
if isinstance(val, datetime.datetime):
return val.strftime("%Y-%m-%d %H:%M")
elif isinstance(val, datetime.date):
# NOTE: datetime is instance of date, so check datetime first
return val.isoformat()
else:
return val
request.xml_response['related_objects'] = json.dumps(
related_objects_info, default=formatter
)
request.xml_response['deletable_objects'] = json.dumps(
deletable_objects
)
request.xml_response["delete_info"] = json.dumps(
{
"runtime": _end_time - _start_time,
}
)
def check_special_commands(log_com):
from initat.cluster.backbone.models import mon_check_command
def _check_db_for_mccs(log_com, meta, parent=None):
try:
cur_mccs = mon_check_command.objects.get(Q(uuid=meta.uuid))
except mon_check_command.DoesNotExist:
try:
cur_mccs = mon_check_command.objects.get(
Q(uuid=meta.database_name))
except:
# not found
cur_mccs = None
if cur_mccs is None:
log_com("Creating new special mon_check_command")
cur_mccs = mon_check_command(
name=meta.database_name,
is_special_command=True,
uuid=meta.uuid,
)
# print("*", cur_mccs)
# except mon_check_command_special.DoesNotExist:
#
# cur_mccs = mon_check_command_special(name=meta.database_name)
# also used in snmp/struct.py and generic_net_handler.py
for attr_name in {
"command_line", "info", "description", "is_active", "meta",
"identifier"
}:
setattr(cur_mccs, attr_name, getattr(meta, attr_name, ""))
cur_mccs.group = getattr(meta, "group", SpecialGroupsEnum.unspec).value
cur_mccs.parent = parent
cur_mccs.save()
return cur_mccs
pks_found = set()
mccs_dict = {}
for _name, _inst in dynamic_checks.valid_class_dict(log_com).items():
# print(_name, _entry)
if True: # False:
cur_mccs = _check_db_for_mccs(log_com, _inst.Meta)
mccs_dict[cur_mccs.name] = cur_mccs
pks_found.add(cur_mccs.pk)
if cur_mccs.meta:
# iterate over subcommnds
for _sub_com in _inst.get_commands():
if not hasattr(_sub_com.Meta, "database_name"):
# for instance commands from SNMP checks
# set database_name attribute
_sub_com.Meta.database_name = _sub_com.Meta.name
# create database entry (if needed) and set parent
sub_mccs = _check_db_for_mccs(log_com,
_sub_com.Meta,
parent=cur_mccs)
mccs_dict[sub_mccs.name] = sub_mccs
pks_found.add(sub_mccs.pk)
# print(mccs_dict)
# delete stale
if True: # False:
del_mccs = mon_check_command.objects.exclude(
Q(pk__in=pks_found)).filter(Q(is_special_command=True))
if del_mccs.count():
for _del_mcc in del_mccs:
log_com("trying to remove stale {}...".format(str(_del_mcc), ))
_refs = get_related_models(_del_mcc)
if _refs:
log_com(
" unable to remove because referenced {}".format(
logging_tools.get_plural("time", _refs)),
logging_tools.LOG_LEVEL_ERROR)
else:
_del_mcc.delete()
log_com(" ...done")