def init_cache():
db.session.remove()
if current_app.config.get("USE_ES"):
from api.extensions import es
from api.models.cmdb import Attribute
from api.lib.cmdb.utils import ValueTypeMap
attributes = Attribute.get_by(to_dict=False)
for attr in attributes:
other = dict()
other['index'] = True if attr.is_index else False
if attr.value_type == ValueTypeEnum.TEXT:
other['analyzer'] = 'ik_max_word'
other['search_analyzer'] = 'ik_smart'
if attr.is_index:
other["fields"] = {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
try:
es.update_mapping(attr.name,
ValueTypeMap.es_type[attr.value_type], other)
except Exception as e:
print(e)
cis = CI.get_by(to_dict=False)
for ci in cis:
if current_app.config.get("USE_ES"):
res = es.get_index_id(ci.id)
if res:
continue
else:
res = rd.get([ci.id], REDIS_PREFIX_CI)
if res and list(filter(lambda x: x, res)):
continue
m = api.lib.cmdb.ci.CIManager()
ci_dict = m.get_ci_by_id_from_db(ci.id,
need_children=False,
use_master=False)
if current_app.config.get("USE_ES"):
es.create(ci_dict)
else:
rd.create_or_update({ci.id: json.dumps(ci_dict)}, REDIS_PREFIX_CI)
ci_relations = CIRelation.get_by(to_dict=False)
relations = dict()
for cr in ci_relations:
relations.setdefault(cr.first_ci_id, {}).update(
{cr.second_ci_id: cr.second_ci.type_id})
for i in relations:
relations[i] = json.dumps(relations[i])
if relations:
rd.create_or_update(relations, REDIS_PREFIX_CI_RELATION)
db.session.remove()
def search(self):
ids = [self.root_id
] if not isinstance(self.root_id, list) else self.root_id
cis = [
CI.get_by_id(_id)
or abort(404, "CI <{0}> does not exist".format(_id)) for _id in ids
]
merge_ids = []
for level in self.level:
ids = [self.root_id
] if not isinstance(self.root_id, list) else self.root_id
for _ in range(0, level):
_tmp = list(
map(
lambda x: list(json.loads(x).keys()),
filter(lambda x: x is not None,
rd.get(ids, REDIS_PREFIX_CI_RELATION) or [])))
ids = [j for i in _tmp for j in i]
merge_ids.extend(ids)
if not self.orig_query or ("_type:" not in self.orig_query
and "type_id:" not in self.orig_query
and "ci_type:" not in self.orig_query):
type_ids = []
for level in self.level:
for ci in cis:
type_ids.extend(
CITypeRelationManager.get_child_type_ids(
ci.type_id, level))
type_ids = list(set(type_ids))
if self.orig_query:
self.orig_query = "_type:({0}),{1}".format(
";".join(list(map(str, type_ids))), self.orig_query)
else:
self.orig_query = "_type:({0})".format(";".join(
list(map(str, type_ids))))
if not merge_ids:
# cis, counter, total, self.page, numfound, facet_
return [], {}, 0, self.page, 0, {}
if current_app.config.get("USE_ES"):
return SearchFromES(self.orig_query,
fl=self.fl,
facet_field=self.facet_field,
page=self.page,
count=self.count,
sort=self.sort,
ci_ids=merge_ids).search()
else:
return SearchFromDB(self.orig_query,
fl=self.fl,
facet_field=self.facet_field,
page=self.page,
count=self.count,
sort=self.sort,
ci_ids=merge_ids).search()
def ci_relation_delete(parent_id, child_id):
children = rd.get([parent_id], REDIS_PREFIX_CI_RELATION)[0]
children = json.loads(children) if children is not None else {}
if str(child_id) in children:
children.pop(str(child_id))
rd.create_or_update({parent_id: json.dumps(children)},
REDIS_PREFIX_CI_RELATION)
current_app.logger.info("DELETE ci relation cache: {0} -> {1}".format(
parent_id, child_id))
def statistics(self, type_ids):
_tmp = []
ids = [self.root_id
] if not isinstance(self.root_id, list) else self.root_id
for l in range(0, int(self.level)):
if not l:
_tmp = list(
map(lambda x: list(json.loads(x).keys()), [
i or '{}'
for i in rd.get(ids, REDIS_PREFIX_CI_RELATION) or []
]))
else:
for idx, i in enumerate(_tmp):
if i:
if type_ids and l == self.level - 1:
__tmp = list(
map(
lambda x: list({
_id: 1
for _id, type_id in json.loads(x).
items() if type_id in type_ids
}.keys()),
filter(
lambda x: x is not None,
rd.get(i, REDIS_PREFIX_CI_RELATION)
or [])))
else:
__tmp = list(
map(
lambda x: list(json.loads(x).keys()),
filter(
lambda x: x is not None,
rd.get(i, REDIS_PREFIX_CI_RELATION)
or [])))
_tmp[idx] = [j for i in __tmp for j in i]
else:
_tmp[idx] = []
return {_id: len(_tmp[idx]) for idx, _id in enumerate(ids)}
def _get_cis_from_cache(ci_ids, ret_key=RetKey.NAME, fields=None):
res = rd.get(ci_ids)
if res is not None and None not in res and ret_key == RetKey.NAME:
res = list(map(json.loads, res))
if not fields:
return res
else:
_res = []
for d in res:
_d = dict()
_d["_id"], _d["_type"] = d.get("_id"), d.get("_type")
_d["ci_type"] = d.get("ci_type")
for field in fields:
_d[field] = d.get(field)
_res.append(_d)
return _res
def ci_relation_cache(parent_id, child_id):
children = rd.get([parent_id], REDIS_PREFIX_CI_RELATION)[0]
children = json.loads(children) if children is not None else {}
cr = CIRelation.get_by(first_ci_id=parent_id,
second_ci_id=child_id,
first=True,
to_dict=False)
if str(child_id) not in children:
children[str(child_id)] = cr.second_ci.type_id
rd.create_or_update({parent_id: json.dumps(children)},
REDIS_PREFIX_CI_RELATION)
current_app.logger.info("ADD ci relation cache: {0} -> {1}".format(
parent_id, child_id))