def update(self, _id, **kwargs): attr = Attribute.get_by_id(_id) or abort(404, "Attribute <{0}> does not exist".format(_id)) if kwargs.get("name"): other = Attribute.get_by(name=kwargs['name'], first=True, to_dict=False) if other and other.id != attr.id: return abort(400, "Attribute name <{0}> cannot be duplicate!".format(kwargs['name'])) if kwargs.get("alias"): other = Attribute.get_by(alias=kwargs['alias'], first=True, to_dict=False) if other and other.id != attr.id: return abort(400, "Attribute alias <{0}> cannot be duplicate!".format(kwargs['alias'])) choice_value = kwargs.pop("choice_value", False) is_choice = True if choice_value else False kwargs['is_choice'] = is_choice attr.update(flush=True, **kwargs) if is_choice: self._add_choice_values(attr.id, attr.value_type, choice_value) else: self._del_choice_values(attr.id, attr.value_type) try: db.session.commit() except Exception as e: db.session.rollback() current_app.logger.error("update attribute error, {0}".format(str(e))) return abort(400, "update attribute <{0}> failed".format(_id)) AttributeCache.clean(attr) return attr.id
def add(cls, **kwargs): choice_value = kwargs.pop("choice_value", []) kwargs.pop("is_choice", None) is_choice = True if choice_value else False name = kwargs.pop("name") alias = kwargs.pop("alias", "") alias = name if not alias else alias Attribute.get_by(name=name, first=True) and abort(400, "attribute {0} is already existed".format(name)) attr = Attribute.create(flush=True, name=name, alias=alias, is_choice=is_choice, **kwargs) if choice_value: cls._add_choice_values(attr.id, attr.value_type, choice_value) try: db.session.commit() except Exception as e: db.session.rollback() current_app.logger.error("add attribute error, {0}".format(str(e))) return abort(400, "add attribute <{0}> failed".format(name)) AttributeCache.clean(attr) return attr.id
def _check(type_id, attr_ids): CITypeManager.check_is_existed(type_id) if not attr_ids or not isinstance(attr_ids, list): return abort(400, "Attributes are required") for attr_id in attr_ids: AttributeCache.get(attr_id) or abort(404, "Attribute <{0}> is not existed".format(attr_id))
def get_tree_view(): res = PreferenceTreeView.get_by(uid=g.user.uid, to_dict=True) for item in res: if item["levels"]: item.update(CITypeCache.get(item['type_id']).to_dict()) item.update(dict(levels=[AttributeCache.get(l).to_dict() for l in item["levels"].split(",") if AttributeCache.get(l)])) return res
def add(cls, **kwargs): choice_value = kwargs.pop("choice_value", []) kwargs.pop("is_choice", None) is_choice = True if choice_value else False name = kwargs.pop("name") alias = kwargs.pop("alias", "") alias = name if not alias else alias Attribute.get_by(name=name, first=True) and abort( 400, "attribute name <{0}> is duplicated".format(name)) Attribute.get_by(alias=alias, first=True) and abort( 400, "attribute alias <{0}> is duplicated".format(name)) attr = Attribute.create(flush=True, name=name, alias=alias, is_choice=is_choice, **kwargs) if choice_value: cls._add_choice_values(attr.id, attr.value_type, choice_value) try: db.session.commit() except Exception as e: db.session.rollback() current_app.logger.error("add attribute error, {0}".format(str(e))) return abort(400, "add attribute <{0}> failed".format(name)) AttributeCache.clean(attr) if current_app.config.get("USE_ES"): from api.extensions import es other = dict() other['index'] = True if attr.is_index else False if attr.value_type == ValueTypeEnum.TEXT: other['analyzer'] = 'ik_max_word' other['search_analyzer'] = 'ik_smart' if attr.is_index: other["fields"] = { "keyword": { "type": "keyword", "ignore_above": 256 } } es.update_mapping(name, ValueTypeMap.es_type[attr.value_type], other) return attr.id
def get_ci_types(type_name=None): ci_types = CIType.get_by() if type_name is None else CIType.get_by_like(name=type_name) res = list() for type_dict in ci_types: type_dict["unique_key"] = AttributeCache.get(type_dict["unique_id"]).name res.append(type_dict) return res
def get_record_detail(record_id): from api.lib.cmdb.ci import CIManager record = OperationRecord.get_by_id(record_id) or abort( 404, "Record <{0}> is not found".format(record_id)) username = UserCache.get(record.uid).nickname or UserCache.get( record.uid).username timestamp = record.created_at.strftime("%Y-%m-%d %H:%M:%S") attr_history = AttributeHistory.get_By(record_id=record_id, to_dict=False) rel_history = CIRelationHistory.get_by(record_id=record_id, to_dict=False) attr_dict, rel_dict = dict(), {"add": [], "delete": []} for attr_h in attr_history: attr_dict[AttributeCache.get(attr_h.attr_id).alias] = dict( old=attr_h.old, new=attr_h.new, operate_type=attr_h.operate_type) for rel_h in rel_history: first = CIManager.get_ci_by_id(rel_h.first_ci_id) second = CIManager.get_ci_by_id(rel_h.second_ci_id) rel_dict[rel_h.operate_type].append( (first, RelationTypeCache.get(rel_h.relation_type_id).name, second)) return username, timestamp, attr_dict, rel_dict
def add(cls, ci_type_name, exist_policy=ExistPolicy.REPLACE, _no_attribute_policy=ExistPolicy.IGNORE, **ci_dict): """ :param ci_type_name: :param exist_policy: replace or reject or need :param _no_attribute_policy: ignore or reject :param ci_dict: :return: """ ci_type = CITypeManager.check_is_existed(ci_type_name) unique_key = AttributeCache.get(ci_type.unique_id) or abort( 400, 'illegality unique attribute') unique_value = ci_dict.get(unique_key.name) unique_value = unique_value or ci_dict.get(unique_key.alias) unique_value = unique_value or ci_dict.get(unique_key.id) unique_value = unique_value or abort( 400, '{0} missing'.format(unique_key.name)) existed = cls.ci_is_exist(unique_key, unique_value) if existed is not None: if exist_policy == ExistPolicy.REJECT: return abort(400, 'CI is already existed') if existed.type_id != ci_type.id: existed.update(type_id=ci_type.id) ci = existed else: if exist_policy == ExistPolicy.NEED: return abort(404, 'CI <{0}> does not exist'.format(unique_value)) ci = CI.create(type_id=ci_type.id) ci_type_attrs_name = [ attr["name"] for attr in CITypeAttributeManager().get_attributes_by_type_id(ci_type.id) ] value_manager = AttributeValueManager() for p, v in ci_dict.items(): if p not in ci_type_attrs_name: current_app.logger.warning( 'ci_type: {0} not has attribute {1}, please check!'.format( ci_type_name, p)) continue try: value_manager.create_or_update_attr_value( p, v, ci, _no_attribute_policy) except BadRequest as e: if existed is None: cls.delete(ci.id) raise e ci_cache.apply_async([ci.id], queue=CMDB_QUEUE) return ci.id
def _sort_build(self): fields = list(filter(lambda x: x != "", (self.sort or "").split(","))) sorts = [] for field in fields: sort_type = "asc" if field.startswith("+"): field = field[1:] elif field.startswith("-"): field = field[1:] sort_type = "desc" else: field = field if field == "ci_id": sorts.append({field: {"order": sort_type}}) continue attr = AttributeCache.get(field) if not attr: raise SearchError("Sort by <{0}> does not exist".format(field)) sort_by = "{0}.keyword".format(field) \ if attr.value_type not in (ValueTypeEnum.INT, ValueTypeEnum.FLOAT) else field sorts.append({sort_by: {"order": sort_type}}) self.query.update(dict(sort=sorts))
def _query_wrap_for_device(self, query_sql, **kwargs): _type = kwargs.pop("_type", False) or kwargs.pop( "type", False) or kwargs.pop("ci_type", False) if _type: ci_type = CITypeCache.get(_type) if ci_type is None: return query_sql = query_sql.filter(CI.type_id == ci_type.id) for k, v in kwargs.items(): attr = AttributeCache.get(k) if attr is None: continue value_table = TableMap(attr_name=k).table ci_table = query_sql.subquery() query_sql = db.session.query(ci_table.c.id).join( value_table, value_table.ci_id == ci_table.c.id).filter( value_table.attr_id == attr.id).filter( ci_table.deleted.is_(False)).filter( value_table.value.ilike(v.replace("*", "%"))) # current_app.logger.debug(query_sql) sort_by = kwargs.pop("sort", "") if sort_by: query_sql = self._sort_handler(sort_by, query_sql) return query_sql
def add(cls, **kwargs): unique_key = kwargs.pop("unique_key", None) unique_key = AttributeCache.get(unique_key) or abort( 404, "Unique key is not defined") kwargs["alias"] = kwargs["name"] if not kwargs.get( "alias") else kwargs["alias"] cls._validate_unique(name=kwargs['name']) cls._validate_unique(alias=kwargs['alias']) kwargs["unique_id"] = unique_key.id ci_type = CIType.create(**kwargs) CITypeAttributeManager.add(ci_type.id, [unique_key.id], is_required=True) CITypeCache.clean(ci_type.name) if current_app.config.get("USE_ACL"): from api.lib.perm.acl.acl import ACLManager from api.lib.cmdb.const import ResourceTypeEnum, RoleEnum, PermEnum ACLManager().add_resource(ci_type.name, ResourceTypeEnum.CI) ACLManager().grant_resource_to_role(ci_type.name, RoleEnum.CMDB_READ_ALL, ResourceTypeEnum.CI, permissions=[PermEnum.READ]) return ci_type.id
def get_attribute(self, key): attr = AttributeCache.get(key).to_dict() if attr and attr["is_choice"]: attr.update( dict(choice_value=self.get_choice_values( attr["id"], attr["value_type"]))) return attr
def batch_update(cls, ci_ids, parents): """ only for many to one :param ci_ids: :param parents: :return: """ from api.lib.cmdb.utils import TableMap if parents is not None and isinstance(parents, dict): for attr_name in parents: if parents[attr_name]: attr = AttributeCache.get(attr_name) value_table = TableMap(attr_name=attr.name).table parent = value_table.get_by(attr_id=attr.id, value=parents[attr_name], first=True, to_dict=False) if not parent: return abort( 404, "{0}: {1} is not found".format( attr_name, parents[attr_name])) parent_id = parent.ci_id for ci_id in ci_ids: cls.add(parent_id, ci_id, many_to_one=True)
def __sort_by_field(self, field, sort_type, query_sql): attr = AttributeCache.get(field) attr_id = attr.id table_name = TableMap(attr_name=attr.name).table_name _v_query_sql = """SELECT {0}.ci_id, {1}.value FROM ({2}) AS {0} INNER JOIN {1} ON {1}.ci_id = {0}.ci_id WHERE {1}.attr_id = {3}""".format("ALIAS", table_name, query_sql, attr_id) new_table = _v_query_sql if self.only_type_query or not self.type_id_list: return "SELECT SQL_CALC_FOUND_ROWS DISTINCT C.ci_id, C.value " \ "FROM ({0}) AS C " \ "ORDER BY C.value {2} " \ "LIMIT {1:d}, {3};".format(new_table, (self.page - 1) * self.count, sort_type, self.count) elif self.type_id_list: self.query_sql = """SELECT C.ci_id FROM ({0}) AS C INNER JOIN c_cis on c_cis.id=C.ci_id WHERE c_cis.type_id IN ({1})""".format(new_table, ",".join(self.type_id_list)) return """SELECT SQL_CALC_FOUND_ROWS DISTINCT C.ci_id, C.value FROM ({0}) AS C INNER JOIN c_cis on c_cis.id=C.ci_id WHERE c_cis.type_id IN ({4}) ORDER BY C.value {2} LIMIT {1:d}, {3};""".format(new_table, (self.page - 1) * self.count, sort_type, self.count, ",".join(self.type_id_list))
def delete(ci_id): ci = CI.get_by_id(ci_id) or abort( 404, "CI <{0}> is not found".format(ci_id)) attrs = CITypeAttribute.get_by(type_id=ci.type_id, to_dict=False) attr_names = set( [AttributeCache.get(attr.attr_id).name for attr in attrs]) for attr_name in attr_names: value_table = TableMap(attr_name=attr_name).table for item in value_table.get_by(ci_id=ci_id, to_dict=False): item.delete() for item in CIRelation.get_by(first_ci_id=ci_id, to_dict=False): item.delete() for item in CIRelation.get_by(second_ci_id=ci_id, to_dict=False): item.delete() ci.delete() # TODO: soft delete AttributeHistoryManger.add(ci_id, [(None, OperateType.DELETE, None, None)]) ci_delete.apply_async([ci.id], queue=CMDB_QUEUE) return ci_id
def _sort_handler(sort_by, query_sql): if sort_by.startswith("+"): sort_type = "asc" sort_by = sort_by[1:] elif sort_by.startswith("-"): sort_type = "desc" sort_by = sort_by[1:] else: sort_type = "asc" attr = AttributeCache.get(sort_by) if attr is None: return query_sql attr_id = attr.id value_table = TableMap(attr_name=sort_by).table ci_table = query_sql.subquery() query_sql = db.session.query(ci_table.c.id, value_table.value).join( value_table, value_table.ci_id == ci_table.c.id).filter( value_table.attr_id == attr_id).filter( ci_table.deleted.is_(False)).order_by( getattr(value_table.value, sort_type)()) return query_sql
def get_ci_by_id_from_db(ci_id, ret_key=RetKey.NAME, fields=None, need_children=True, use_master=False): """ :param ci_id: :param ret_key: name, id or alias :param fields: list :param need_children: :param use_master: whether to use master db :return: """ ci = CI.get_by_id(ci_id) or abort(404, "CI <{0}> is not existed".format(ci_id)) res = dict() if need_children: children = CIRelationManager.get_children(ci_id, ret_key=ret_key) # one floor res.update(children) ci_type = CITypeCache.get(ci.type_id) res["ci_type"] = ci_type.name fields = CITypeAttributeManager.get_attr_names_by_type_id(ci.type_id) if not fields else fields unique_key = AttributeCache.get(ci_type.unique_id) _res = AttributeValueManager().get_attr_values(fields, ci_id, ret_key=ret_key, unique_key=unique_key, use_master=use_master) res.update(_res) res['type_id'] = ci_type.id res['ci_id'] = ci_id return res
def delete_attr_value(attr_id, ci_id): attr = AttributeCache.get(attr_id) if attr is not None: value_table = TableMap(attr_name=attr.name).table for item in value_table.get_by(attr_id=attr.id, ci_id=ci_id, to_dict=False): item.delete()
def get(self, type_id=None, type_name=None): t = CITypeCache.get(type_id) or CITypeCache.get(type_name) or abort(404, "CIType does not exist") type_id = t.id unique_id = t.unique_id unique = AttributeCache.get(unique_id).name return self.jsonify(attributes=CITypeAttributeManager.get_attributes_by_type_id(type_id), type_id=type_id, unique_id=unique_id, unique=unique)
def _get_cis_from_db(ci_ids, ret_key=RetKey.NAME, fields=None, value_tables=None): if not fields: filter_fields_sql = "" else: _fields = list() for field in fields: attr = AttributeCache.get(field) if attr is not None: _fields.append(str(attr.id)) filter_fields_sql = "WHERE A.attr_id in ({0})".format( ",".join(_fields)) ci_ids = ",".join(ci_ids) if value_tables is None: value_tables = type_map["table_name"].values() value_sql = " UNION ".join([ QUERY_CIS_BY_VALUE_TABLE.format(value_table, ci_ids) for value_table in value_tables ]) query_sql = QUERY_CIS_BY_IDS.format(filter_fields_sql, value_sql) # current_app.logger.debug(query_sql) cis = db.session.execute(query_sql).fetchall() ci_set = set() res = list() ci_dict = dict() for ci_id, type_id, attr_id, attr_name, attr_alias, value, value_type, is_list in cis: if ci_id not in ci_set: ci_dict = dict() ci_type = CITypeCache.get(type_id) ci_dict["_id"] = ci_id ci_dict["_type"] = type_id ci_dict["ci_type"] = ci_type.name ci_dict["ci_type_alias"] = ci_type.alias ci_set.add(ci_id) res.append(ci_dict) if ret_key == RetKey.NAME: attr_key = attr_name elif ret_key == RetKey.ALIAS: attr_key = attr_alias elif ret_key == RetKey.ID: attr_key = attr_id else: return abort(400, "invalid ret key") value = type_map["serialize2"][value_type](value) if is_list: ci_dict.setdefault(attr_key, []).append(value) else: ci_dict[attr_key] = value return res
def delete(_id): attr = Attribute.get_by_id(_id) or abort(404, "Attribute <{0}> does not exist".format(_id)) name = attr.name if attr.is_choice: choice_table = type_map["choice"].get(attr.value_type) db.session.query(choice_table).filter(choice_table.attr_id == _id).delete() # FIXME: session conflict db.session.flush() AttributeCache.clean(attr) attr.soft_delete() for i in CITypeAttribute.get_by(attr_id=_id, to_dict=False): i.soft_delete() for i in PreferenceShowAttributes.get_by(attr_id=_id, to_dict=False): i.soft_delete() return name
def update(self, _id, **kwargs): attr = Attribute.get_by_id(_id) or abort(404, "Attribute <{0}> does not exist".format(_id)) choice_value = kwargs.pop("choice_value", False) is_choice = True if choice_value else False attr.update(flush=True, **kwargs) if is_choice: self._add_choice_values(attr.id, attr.value_type, choice_value) try: db.session.commit() except Exception as e: db.session.rollback() current_app.logger.error("update attribute error, {0}".format(str(e))) return abort(400, "update attribute <{0}> failed".format(_id)) AttributeCache.clean(attr) return attr.id
def _attr_name_proc(self, key): operator, key = self._operator_proc(key) if key in ('ci_type', 'type', '_type'): return '_type', ValueTypeEnum.TEXT, operator, None if key in ('id', 'ci_id', '_id'): return '_id', ValueTypeEnum.TEXT, operator, None attr = AttributeCache.get(key) if attr: return attr.name, attr.value_type, operator, attr else: raise SearchError("{0} is not existed".format(key))
def add_heartbeat(ci_type, unique_value): ci_type = CITypeManager().check_is_existed(ci_type) unique_key = AttributeCache.get(ci_type.unique_id) value_table = TableMap(attr_name=unique_key.name).table v = value_table.get_by(attr_id=unique_key.id, value=unique_value, to_dict=False, first=True) \ or abort(404, "not found") ci = CI.get_by_id(v.ci_id) or abort(404, "CI <{0}> is not found".format(v.ci_id)) ci.update(heartbeat=datetime.datetime.now())
def get_records(start, end, username, page, page_size): records = db.session.query(OperationRecord).filter( OperationRecord.deleted.is_(False)) numfound = db.session.query(db.func.count(OperationRecord.id)).filter( OperationRecord.deleted.is_(False)) if start: records = records.filter(OperationRecord.created_at >= start) numfound = numfound.filter(OperationRecord.created_at >= start) if end: records = records.filter(OperationRecord.created_at <= end) numfound = records.filter(OperationRecord.created_at <= end) if username: user = UserCache.get(username) if user: records = records.filter(OperationRecord.uid == user.uid) else: return abort(404, "User <{0}> is not found".format(username)) records = records.order_by(-OperationRecord.id).offset( page_size * (page - 1)).limit(page_size).all() total = len(records) numfound = numfound.first()[0] res = [] for record in records: _res = record.to_dict() _res["user"] = UserCache.get( _res.get("uid")).nickname or UserCache.get( _res.get("uid")).username attr_history = AttributeHistory.get_by(record_id=_res.get("id"), to_dict=False) _res["attr_history"] = [ AttributeCache.get(h.attr_id).attr_alias for h in attr_history ] rel_history = CIRelationHistory.get_by(record_id=_res.get("id"), to_dict=False) rel_statis = {} for rel in rel_history: if rel.operate_type not in rel_statis: rel_statis[rel.operate_type] = 1 else: rel_statis[rel.operate_type] += 1 _res["rel_history"] = rel_statis res.append(_res) return numfound, total, res
def _facet_build(self): aggregations = dict(aggs={}) for field in self.facet_field: attr = AttributeCache.get(field) if not attr: raise SearchError("Facet by <{0}> does not exist".format(field)) aggregations['aggs'].update({ field: { "terms": { "field": "{0}.keyword".format(field) if attr.value_type not in (ValueTypeEnum.INT, ValueTypeEnum.FLOAT) else field } } }) if aggregations['aggs']: self.query.update(aggregations)
def create_or_update_tree_view(type_id, levels): attrs = CITypeAttributesCache.get(type_id) for idx, i in enumerate(levels): for attr in attrs: attr = AttributeCache.get(attr.attr_id) if i == attr.id or i == attr.name or i == attr.alias: levels[idx] = str(attr.id) levels = ",".join(levels) existed = PreferenceTreeView.get_by(uid=g.user.uid, type_id=type_id, to_dict=False, first=True) if existed is not None: if not levels: existed.soft_delete() return existed return existed.update(levels=levels) elif levels: return PreferenceTreeView.create(levels=levels, type_id=type_id, uid=g.user.uid)
def get_by_type_id(type_id, need_other=None): groups = CITypeAttributeGroup.get_by(type_id=type_id) groups = sorted(groups, key=lambda x: x["order"]) grouped = list() for group in groups: items = CITypeAttributeGroupItem.get_by(group_id=group["id"], to_dict=False) items = sorted(items, key=lambda x: x.order) group["attributes"] = [AttributeCache.get(i.attr_id).to_dict() for i in items] grouped.extend([i.attr_id for i in items]) if need_other is not None: grouped = set(grouped) attributes = CITypeAttributeManager.get_attributes_by_type_id(type_id) other_attributes = [attr for attr in attributes if attr["id"] not in grouped] groups.append(dict(attributes=other_attributes)) return groups
def _facet_build(self): facet = {} for f in self.facet_field: k, field_type, _, attr = self._attr_name_proc(f) if k: table_name = TableMap(attr_name=k).table_name query_sql = FACET_QUERY.format(table_name, self.query_sql, attr.id) current_app.logger.debug(query_sql) result = db.session.execute(query_sql).fetchall() facet[k] = result facet_result = dict() for k, v in facet.items(): if not k.startswith('_'): a = getattr(AttributeCache.get(k), self.ret_key) facet_result[a] = [(f[0], f[1], a) for f in v] return facet_result
def update(cls, type_id, **kwargs): ci_type = cls.check_is_existed(type_id) unique_key = kwargs.pop("unique_key", None) unique_key = AttributeCache.get(unique_key) if unique_key is not None: kwargs["unique_id"] = unique_key.id type_attr = CITypeAttribute.get_by(type_id=type_id, attr_id=unique_key.id, first=True, to_dict=False) if type_attr is None: CITypeAttributeManager.add(type_id, [unique_key.id], is_required=True) ci_type.update(**kwargs) CITypeCache.clean(type_id) return type_id