def update(self, _id, **kwargs):
attr = Attribute.get_by_id(_id) or abort(404, "Attribute <{0}> does not exist".format(_id))
if kwargs.get("name"):
other = Attribute.get_by(name=kwargs['name'], first=True, to_dict=False)
if other and other.id != attr.id:
return abort(400, "Attribute name <{0}> cannot be duplicate!".format(kwargs['name']))
if kwargs.get("alias"):
other = Attribute.get_by(alias=kwargs['alias'], first=True, to_dict=False)
if other and other.id != attr.id:
return abort(400, "Attribute alias <{0}> cannot be duplicate!".format(kwargs['alias']))
choice_value = kwargs.pop("choice_value", False)
is_choice = True if choice_value else False
kwargs['is_choice'] = is_choice
attr.update(flush=True, **kwargs)
if is_choice:
self._add_choice_values(attr.id, attr.value_type, choice_value)
else:
self._del_choice_values(attr.id, attr.value_type)
try:
db.session.commit()
except Exception as e:
db.session.rollback()
current_app.logger.error("update attribute error, {0}".format(str(e)))
return abort(400, "update attribute <{0}> failed".format(_id))
AttributeCache.clean(attr)
return attr.id
def add(cls, **kwargs):
choice_value = kwargs.pop("choice_value", [])
kwargs.pop("is_choice", None)
is_choice = True if choice_value else False
name = kwargs.pop("name")
alias = kwargs.pop("alias", "")
alias = name if not alias else alias
Attribute.get_by(name=name, first=True) and abort(400, "attribute {0} is already existed".format(name))
attr = Attribute.create(flush=True,
name=name,
alias=alias,
is_choice=is_choice,
**kwargs)
if choice_value:
cls._add_choice_values(attr.id, attr.value_type, choice_value)
try:
db.session.commit()
except Exception as e:
db.session.rollback()
current_app.logger.error("add attribute error, {0}".format(str(e)))
return abort(400, "add attribute <{0}> failed".format(name))
AttributeCache.clean(attr)
return attr.id
def _check(type_id, attr_ids):
CITypeManager.check_is_existed(type_id)
if not attr_ids or not isinstance(attr_ids, list):
return abort(400, "Attributes are required")
for attr_id in attr_ids:
AttributeCache.get(attr_id) or abort(404, "Attribute <{0}> is not existed".format(attr_id))
def get_tree_view():
res = PreferenceTreeView.get_by(uid=g.user.uid, to_dict=True)
for item in res:
if item["levels"]:
item.update(CITypeCache.get(item['type_id']).to_dict())
item.update(dict(levels=[AttributeCache.get(l).to_dict()
for l in item["levels"].split(",") if AttributeCache.get(l)]))
return res
def add(cls, **kwargs):
choice_value = kwargs.pop("choice_value", [])
kwargs.pop("is_choice", None)
is_choice = True if choice_value else False
name = kwargs.pop("name")
alias = kwargs.pop("alias", "")
alias = name if not alias else alias
Attribute.get_by(name=name, first=True) and abort(
400, "attribute name <{0}> is duplicated".format(name))
Attribute.get_by(alias=alias, first=True) and abort(
400, "attribute alias <{0}> is duplicated".format(name))
attr = Attribute.create(flush=True,
name=name,
alias=alias,
is_choice=is_choice,
**kwargs)
if choice_value:
cls._add_choice_values(attr.id, attr.value_type, choice_value)
try:
db.session.commit()
except Exception as e:
db.session.rollback()
current_app.logger.error("add attribute error, {0}".format(str(e)))
return abort(400, "add attribute <{0}> failed".format(name))
AttributeCache.clean(attr)
if current_app.config.get("USE_ES"):
from api.extensions import es
other = dict()
other['index'] = True if attr.is_index else False
if attr.value_type == ValueTypeEnum.TEXT:
other['analyzer'] = 'ik_max_word'
other['search_analyzer'] = 'ik_smart'
if attr.is_index:
other["fields"] = {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
es.update_mapping(name, ValueTypeMap.es_type[attr.value_type],
other)
return attr.id
def get_ci_types(type_name=None):
ci_types = CIType.get_by() if type_name is None else CIType.get_by_like(name=type_name)
res = list()
for type_dict in ci_types:
type_dict["unique_key"] = AttributeCache.get(type_dict["unique_id"]).name
res.append(type_dict)
return res
def get_record_detail(record_id):
from api.lib.cmdb.ci import CIManager
record = OperationRecord.get_by_id(record_id) or abort(
404, "Record <{0}> is not found".format(record_id))
username = UserCache.get(record.uid).nickname or UserCache.get(
record.uid).username
timestamp = record.created_at.strftime("%Y-%m-%d %H:%M:%S")
attr_history = AttributeHistory.get_By(record_id=record_id,
to_dict=False)
rel_history = CIRelationHistory.get_by(record_id=record_id,
to_dict=False)
attr_dict, rel_dict = dict(), {"add": [], "delete": []}
for attr_h in attr_history:
attr_dict[AttributeCache.get(attr_h.attr_id).alias] = dict(
old=attr_h.old,
new=attr_h.new,
operate_type=attr_h.operate_type)
for rel_h in rel_history:
first = CIManager.get_ci_by_id(rel_h.first_ci_id)
second = CIManager.get_ci_by_id(rel_h.second_ci_id)
rel_dict[rel_h.operate_type].append(
(first, RelationTypeCache.get(rel_h.relation_type_id).name,
second))
return username, timestamp, attr_dict, rel_dict
def add(cls,
ci_type_name,
exist_policy=ExistPolicy.REPLACE,
_no_attribute_policy=ExistPolicy.IGNORE,
**ci_dict):
"""
:param ci_type_name:
:param exist_policy: replace or reject or need
:param _no_attribute_policy: ignore or reject
:param ci_dict:
:return:
"""
ci_type = CITypeManager.check_is_existed(ci_type_name)
unique_key = AttributeCache.get(ci_type.unique_id) or abort(
400, 'illegality unique attribute')
unique_value = ci_dict.get(unique_key.name)
unique_value = unique_value or ci_dict.get(unique_key.alias)
unique_value = unique_value or ci_dict.get(unique_key.id)
unique_value = unique_value or abort(
400, '{0} missing'.format(unique_key.name))
existed = cls.ci_is_exist(unique_key, unique_value)
if existed is not None:
if exist_policy == ExistPolicy.REJECT:
return abort(400, 'CI is already existed')
if existed.type_id != ci_type.id:
existed.update(type_id=ci_type.id)
ci = existed
else:
if exist_policy == ExistPolicy.NEED:
return abort(404,
'CI <{0}> does not exist'.format(unique_value))
ci = CI.create(type_id=ci_type.id)
ci_type_attrs_name = [
attr["name"] for attr in
CITypeAttributeManager().get_attributes_by_type_id(ci_type.id)
]
value_manager = AttributeValueManager()
for p, v in ci_dict.items():
if p not in ci_type_attrs_name:
current_app.logger.warning(
'ci_type: {0} not has attribute {1}, please check!'.format(
ci_type_name, p))
continue
try:
value_manager.create_or_update_attr_value(
p, v, ci, _no_attribute_policy)
except BadRequest as e:
if existed is None:
cls.delete(ci.id)
raise e
ci_cache.apply_async([ci.id], queue=CMDB_QUEUE)
return ci.id
def _sort_build(self):
fields = list(filter(lambda x: x != "", (self.sort or "").split(",")))
sorts = []
for field in fields:
sort_type = "asc"
if field.startswith("+"):
field = field[1:]
elif field.startswith("-"):
field = field[1:]
sort_type = "desc"
else:
field = field
if field == "ci_id":
sorts.append({field: {"order": sort_type}})
continue
attr = AttributeCache.get(field)
if not attr:
raise SearchError("Sort by <{0}> does not exist".format(field))
sort_by = "{0}.keyword".format(field) \
if attr.value_type not in (ValueTypeEnum.INT, ValueTypeEnum.FLOAT) else field
sorts.append({sort_by: {"order": sort_type}})
self.query.update(dict(sort=sorts))
def _query_wrap_for_device(self, query_sql, **kwargs):
_type = kwargs.pop("_type", False) or kwargs.pop(
"type", False) or kwargs.pop("ci_type", False)
if _type:
ci_type = CITypeCache.get(_type)
if ci_type is None:
return
query_sql = query_sql.filter(CI.type_id == ci_type.id)
for k, v in kwargs.items():
attr = AttributeCache.get(k)
if attr is None:
continue
value_table = TableMap(attr_name=k).table
ci_table = query_sql.subquery()
query_sql = db.session.query(ci_table.c.id).join(
value_table, value_table.ci_id == ci_table.c.id).filter(
value_table.attr_id == attr.id).filter(
ci_table.deleted.is_(False)).filter(
value_table.value.ilike(v.replace("*", "%")))
# current_app.logger.debug(query_sql)
sort_by = kwargs.pop("sort", "")
if sort_by:
query_sql = self._sort_handler(sort_by, query_sql)
return query_sql
def add(cls, **kwargs):
unique_key = kwargs.pop("unique_key", None)
unique_key = AttributeCache.get(unique_key) or abort(
404, "Unique key is not defined")
kwargs["alias"] = kwargs["name"] if not kwargs.get(
"alias") else kwargs["alias"]
cls._validate_unique(name=kwargs['name'])
cls._validate_unique(alias=kwargs['alias'])
kwargs["unique_id"] = unique_key.id
ci_type = CIType.create(**kwargs)
CITypeAttributeManager.add(ci_type.id, [unique_key.id],
is_required=True)
CITypeCache.clean(ci_type.name)
if current_app.config.get("USE_ACL"):
from api.lib.perm.acl.acl import ACLManager
from api.lib.cmdb.const import ResourceTypeEnum, RoleEnum, PermEnum
ACLManager().add_resource(ci_type.name, ResourceTypeEnum.CI)
ACLManager().grant_resource_to_role(ci_type.name,
RoleEnum.CMDB_READ_ALL,
ResourceTypeEnum.CI,
permissions=[PermEnum.READ])
return ci_type.id
def get_attribute(self, key):
attr = AttributeCache.get(key).to_dict()
if attr and attr["is_choice"]:
attr.update(
dict(choice_value=self.get_choice_values(
attr["id"], attr["value_type"])))
return attr
def batch_update(cls, ci_ids, parents):
"""
only for many to one
:param ci_ids:
:param parents:
:return:
"""
from api.lib.cmdb.utils import TableMap
if parents is not None and isinstance(parents, dict):
for attr_name in parents:
if parents[attr_name]:
attr = AttributeCache.get(attr_name)
value_table = TableMap(attr_name=attr.name).table
parent = value_table.get_by(attr_id=attr.id,
value=parents[attr_name],
first=True,
to_dict=False)
if not parent:
return abort(
404, "{0}: {1} is not found".format(
attr_name, parents[attr_name]))
parent_id = parent.ci_id
for ci_id in ci_ids:
cls.add(parent_id, ci_id, many_to_one=True)
def __sort_by_field(self, field, sort_type, query_sql):
attr = AttributeCache.get(field)
attr_id = attr.id
table_name = TableMap(attr_name=attr.name).table_name
_v_query_sql = """SELECT {0}.ci_id, {1}.value
FROM ({2}) AS {0} INNER JOIN {1} ON {1}.ci_id = {0}.ci_id
WHERE {1}.attr_id = {3}""".format("ALIAS", table_name, query_sql, attr_id)
new_table = _v_query_sql
if self.only_type_query or not self.type_id_list:
return "SELECT SQL_CALC_FOUND_ROWS DISTINCT C.ci_id, C.value " \
"FROM ({0}) AS C " \
"ORDER BY C.value {2} " \
"LIMIT {1:d}, {3};".format(new_table, (self.page - 1) * self.count, sort_type, self.count)
elif self.type_id_list:
self.query_sql = """SELECT C.ci_id
FROM ({0}) AS C
INNER JOIN c_cis on c_cis.id=C.ci_id
WHERE c_cis.type_id IN ({1})""".format(new_table, ",".join(self.type_id_list))
return """SELECT SQL_CALC_FOUND_ROWS DISTINCT C.ci_id, C.value
FROM ({0}) AS C
INNER JOIN c_cis on c_cis.id=C.ci_id
WHERE c_cis.type_id IN ({4})
ORDER BY C.value {2}
LIMIT {1:d}, {3};""".format(new_table,
(self.page - 1) * self.count,
sort_type, self.count,
",".join(self.type_id_list))
def delete(ci_id):
ci = CI.get_by_id(ci_id) or abort(
404, "CI <{0}> is not found".format(ci_id))
attrs = CITypeAttribute.get_by(type_id=ci.type_id, to_dict=False)
attr_names = set(
[AttributeCache.get(attr.attr_id).name for attr in attrs])
for attr_name in attr_names:
value_table = TableMap(attr_name=attr_name).table
for item in value_table.get_by(ci_id=ci_id, to_dict=False):
item.delete()
for item in CIRelation.get_by(first_ci_id=ci_id, to_dict=False):
item.delete()
for item in CIRelation.get_by(second_ci_id=ci_id, to_dict=False):
item.delete()
ci.delete() # TODO: soft delete
AttributeHistoryManger.add(ci_id,
[(None, OperateType.DELETE, None, None)])
ci_delete.apply_async([ci.id], queue=CMDB_QUEUE)
return ci_id
def _sort_handler(sort_by, query_sql):
if sort_by.startswith("+"):
sort_type = "asc"
sort_by = sort_by[1:]
elif sort_by.startswith("-"):
sort_type = "desc"
sort_by = sort_by[1:]
else:
sort_type = "asc"
attr = AttributeCache.get(sort_by)
if attr is None:
return query_sql
attr_id = attr.id
value_table = TableMap(attr_name=sort_by).table
ci_table = query_sql.subquery()
query_sql = db.session.query(ci_table.c.id, value_table.value).join(
value_table, value_table.ci_id == ci_table.c.id).filter(
value_table.attr_id == attr_id).filter(
ci_table.deleted.is_(False)).order_by(
getattr(value_table.value, sort_type)())
return query_sql
def get_ci_by_id_from_db(ci_id, ret_key=RetKey.NAME, fields=None, need_children=True, use_master=False):
"""
:param ci_id:
:param ret_key: name, id or alias
:param fields: list
:param need_children:
:param use_master: whether to use master db
:return:
"""
ci = CI.get_by_id(ci_id) or abort(404, "CI <{0}> is not existed".format(ci_id))
res = dict()
if need_children:
children = CIRelationManager.get_children(ci_id, ret_key=ret_key) # one floor
res.update(children)
ci_type = CITypeCache.get(ci.type_id)
res["ci_type"] = ci_type.name
fields = CITypeAttributeManager.get_attr_names_by_type_id(ci.type_id) if not fields else fields
unique_key = AttributeCache.get(ci_type.unique_id)
_res = AttributeValueManager().get_attr_values(fields,
ci_id,
ret_key=ret_key,
unique_key=unique_key,
use_master=use_master)
res.update(_res)
res['type_id'] = ci_type.id
res['ci_id'] = ci_id
return res
def delete_attr_value(attr_id, ci_id):
attr = AttributeCache.get(attr_id)
if attr is not None:
value_table = TableMap(attr_name=attr.name).table
for item in value_table.get_by(attr_id=attr.id,
ci_id=ci_id,
to_dict=False):
item.delete()
def get(self, type_id=None, type_name=None):
t = CITypeCache.get(type_id) or CITypeCache.get(type_name) or abort(404, "CIType does not exist")
type_id = t.id
unique_id = t.unique_id
unique = AttributeCache.get(unique_id).name
return self.jsonify(attributes=CITypeAttributeManager.get_attributes_by_type_id(type_id),
type_id=type_id,
unique_id=unique_id,
unique=unique)
def _get_cis_from_db(ci_ids,
ret_key=RetKey.NAME,
fields=None,
value_tables=None):
if not fields:
filter_fields_sql = ""
else:
_fields = list()
for field in fields:
attr = AttributeCache.get(field)
if attr is not None:
_fields.append(str(attr.id))
filter_fields_sql = "WHERE A.attr_id in ({0})".format(
",".join(_fields))
ci_ids = ",".join(ci_ids)
if value_tables is None:
value_tables = type_map["table_name"].values()
value_sql = " UNION ".join([
QUERY_CIS_BY_VALUE_TABLE.format(value_table, ci_ids)
for value_table in value_tables
])
query_sql = QUERY_CIS_BY_IDS.format(filter_fields_sql, value_sql)
# current_app.logger.debug(query_sql)
cis = db.session.execute(query_sql).fetchall()
ci_set = set()
res = list()
ci_dict = dict()
for ci_id, type_id, attr_id, attr_name, attr_alias, value, value_type, is_list in cis:
if ci_id not in ci_set:
ci_dict = dict()
ci_type = CITypeCache.get(type_id)
ci_dict["_id"] = ci_id
ci_dict["_type"] = type_id
ci_dict["ci_type"] = ci_type.name
ci_dict["ci_type_alias"] = ci_type.alias
ci_set.add(ci_id)
res.append(ci_dict)
if ret_key == RetKey.NAME:
attr_key = attr_name
elif ret_key == RetKey.ALIAS:
attr_key = attr_alias
elif ret_key == RetKey.ID:
attr_key = attr_id
else:
return abort(400, "invalid ret key")
value = type_map["serialize2"][value_type](value)
if is_list:
ci_dict.setdefault(attr_key, []).append(value)
else:
ci_dict[attr_key] = value
return res
def delete(_id):
attr = Attribute.get_by_id(_id) or abort(404, "Attribute <{0}> does not exist".format(_id))
name = attr.name
if attr.is_choice:
choice_table = type_map["choice"].get(attr.value_type)
db.session.query(choice_table).filter(choice_table.attr_id == _id).delete() # FIXME: session conflict
db.session.flush()
AttributeCache.clean(attr)
attr.soft_delete()
for i in CITypeAttribute.get_by(attr_id=_id, to_dict=False):
i.soft_delete()
for i in PreferenceShowAttributes.get_by(attr_id=_id, to_dict=False):
i.soft_delete()
return name
def update(self, _id, **kwargs):
attr = Attribute.get_by_id(_id) or abort(404, "Attribute <{0}> does not exist".format(_id))
choice_value = kwargs.pop("choice_value", False)
is_choice = True if choice_value else False
attr.update(flush=True, **kwargs)
if is_choice:
self._add_choice_values(attr.id, attr.value_type, choice_value)
try:
db.session.commit()
except Exception as e:
db.session.rollback()
current_app.logger.error("update attribute error, {0}".format(str(e)))
return abort(400, "update attribute <{0}> failed".format(_id))
AttributeCache.clean(attr)
return attr.id
def _attr_name_proc(self, key):
operator, key = self._operator_proc(key)
if key in ('ci_type', 'type', '_type'):
return '_type', ValueTypeEnum.TEXT, operator, None
if key in ('id', 'ci_id', '_id'):
return '_id', ValueTypeEnum.TEXT, operator, None
attr = AttributeCache.get(key)
if attr:
return attr.name, attr.value_type, operator, attr
else:
raise SearchError("{0} is not existed".format(key))
def add_heartbeat(ci_type, unique_value):
ci_type = CITypeManager().check_is_existed(ci_type)
unique_key = AttributeCache.get(ci_type.unique_id)
value_table = TableMap(attr_name=unique_key.name).table
v = value_table.get_by(attr_id=unique_key.id,
value=unique_value,
to_dict=False,
first=True) \
or abort(404, "not found")
ci = CI.get_by_id(v.ci_id) or abort(404, "CI <{0}> is not found".format(v.ci_id))
ci.update(heartbeat=datetime.datetime.now())
def get_records(start, end, username, page, page_size):
records = db.session.query(OperationRecord).filter(
OperationRecord.deleted.is_(False))
numfound = db.session.query(db.func.count(OperationRecord.id)).filter(
OperationRecord.deleted.is_(False))
if start:
records = records.filter(OperationRecord.created_at >= start)
numfound = numfound.filter(OperationRecord.created_at >= start)
if end:
records = records.filter(OperationRecord.created_at <= end)
numfound = records.filter(OperationRecord.created_at <= end)
if username:
user = UserCache.get(username)
if user:
records = records.filter(OperationRecord.uid == user.uid)
else:
return abort(404, "User <{0}> is not found".format(username))
records = records.order_by(-OperationRecord.id).offset(
page_size * (page - 1)).limit(page_size).all()
total = len(records)
numfound = numfound.first()[0]
res = []
for record in records:
_res = record.to_dict()
_res["user"] = UserCache.get(
_res.get("uid")).nickname or UserCache.get(
_res.get("uid")).username
attr_history = AttributeHistory.get_by(record_id=_res.get("id"),
to_dict=False)
_res["attr_history"] = [
AttributeCache.get(h.attr_id).attr_alias for h in attr_history
]
rel_history = CIRelationHistory.get_by(record_id=_res.get("id"),
to_dict=False)
rel_statis = {}
for rel in rel_history:
if rel.operate_type not in rel_statis:
rel_statis[rel.operate_type] = 1
else:
rel_statis[rel.operate_type] += 1
_res["rel_history"] = rel_statis
res.append(_res)
return numfound, total, res
def _facet_build(self):
aggregations = dict(aggs={})
for field in self.facet_field:
attr = AttributeCache.get(field)
if not attr:
raise SearchError("Facet by <{0}> does not exist".format(field))
aggregations['aggs'].update({
field: {
"terms": {
"field": "{0}.keyword".format(field)
if attr.value_type not in (ValueTypeEnum.INT, ValueTypeEnum.FLOAT) else field
}
}
})
if aggregations['aggs']:
self.query.update(aggregations)
def create_or_update_tree_view(type_id, levels):
attrs = CITypeAttributesCache.get(type_id)
for idx, i in enumerate(levels):
for attr in attrs:
attr = AttributeCache.get(attr.attr_id)
if i == attr.id or i == attr.name or i == attr.alias:
levels[idx] = str(attr.id)
levels = ",".join(levels)
existed = PreferenceTreeView.get_by(uid=g.user.uid, type_id=type_id, to_dict=False, first=True)
if existed is not None:
if not levels:
existed.soft_delete()
return existed
return existed.update(levels=levels)
elif levels:
return PreferenceTreeView.create(levels=levels, type_id=type_id, uid=g.user.uid)
def get_by_type_id(type_id, need_other=None):
groups = CITypeAttributeGroup.get_by(type_id=type_id)
groups = sorted(groups, key=lambda x: x["order"])
grouped = list()
for group in groups:
items = CITypeAttributeGroupItem.get_by(group_id=group["id"], to_dict=False)
items = sorted(items, key=lambda x: x.order)
group["attributes"] = [AttributeCache.get(i.attr_id).to_dict() for i in items]
grouped.extend([i.attr_id for i in items])
if need_other is not None:
grouped = set(grouped)
attributes = CITypeAttributeManager.get_attributes_by_type_id(type_id)
other_attributes = [attr for attr in attributes if attr["id"] not in grouped]
groups.append(dict(attributes=other_attributes))
return groups
def _facet_build(self):
facet = {}
for f in self.facet_field:
k, field_type, _, attr = self._attr_name_proc(f)
if k:
table_name = TableMap(attr_name=k).table_name
query_sql = FACET_QUERY.format(table_name, self.query_sql, attr.id)
current_app.logger.debug(query_sql)
result = db.session.execute(query_sql).fetchall()
facet[k] = result
facet_result = dict()
for k, v in facet.items():
if not k.startswith('_'):
a = getattr(AttributeCache.get(k), self.ret_key)
facet_result[a] = [(f[0], f[1], a) for f in v]
return facet_result
def update(cls, type_id, **kwargs):
ci_type = cls.check_is_existed(type_id)
unique_key = kwargs.pop("unique_key", None)
unique_key = AttributeCache.get(unique_key)
if unique_key is not None:
kwargs["unique_id"] = unique_key.id
type_attr = CITypeAttribute.get_by(type_id=type_id,
attr_id=unique_key.id,
first=True,
to_dict=False)
if type_attr is None:
CITypeAttributeManager.add(type_id, [unique_key.id], is_required=True)
ci_type.update(**kwargs)
CITypeCache.clean(type_id)
return type_id