def add(self, parent, child, relation_type="contain"):
p = CITypeCache.get(parent)
if p is None:
return abort(404, "parent {0} is not existed".format(parent))
c = CITypeCache.get(child)
if c is None:
return abort(404, "child {0} is not existed".format(child))
existed = db.session.query(CITypeRelation.ctr_id).filter_by(
parent_id=parent).filter_by(child_id=child).first()
if existed is not None:
return True, existed.ctr_id
ctr = CITypeRelation()
ctr.parent_id = parent
ctr.child_id = child
ctr.relation_type = relation_type
db.session.add(ctr)
try:
db.session.commit()
except Exception as e:
db.session.rollback()
current_app.logger.error(
"add CITypeRelation is error, {0}".format(str(e)))
return abort(
500, "add CITypeRelation is error, {0}".format(str(e)))
return ctr.ctr_id
def get_ci_by_id(self, ci_id, ret_key="name",
fields=None, need_children=True, use_master=False):
"""@params: `ret_key` is one of 'name', 'id', 'alias'
`fields` is list of attribute name/alias/id
"""
ci = CI.query.get(ci_id) or \
abort(404, "CI {0} is not existed".format(ci_id))
res = dict()
if need_children:
children = self.get_children(ci_id, ret_key=ret_key) # one floor
res.update(children)
ci_type = CITypeCache.get(ci.type_id)
res["ci_type"] = ci_type.type_name
uniq_key = CIAttributeCache.get(ci_type.uniq_id)
if not fields: # fields are all attributes
attr_ids = db.session.query(CITypeAttribute.attr_id).filter_by(
type_id=ci.type_id)
fields = [CIAttributeCache.get(_.attr_id).attr_name
for _ in attr_ids]
if uniq_key.attr_name not in fields:
fields.append(uniq_key.attr_name)
if fields:
value_manager = AttributeValueManager()
_res = value_manager._get_attr_values(
fields, ci_id,
ret_key=ret_key, uniq_key=uniq_key, use_master=use_master)
res.update(_res)
res['_type'] = ci_type.type_id
res['_id'] = ci_id
return res
def _query_wrap_for_device(self, query_sql, **kwargs):
_type = kwargs.pop("_type", False) or kwargs.pop("type", False) \
or kwargs.pop("ci_type", False)
if _type:
ci_type = CITypeCache.get(_type)
if ci_type is None:
return
query_sql = query_sql.filter(CI.type_id == ci_type.type_id)
for k, v in kwargs.iteritems():
attr = CIAttributeCache.get(k)
if attr is None:
continue
Table = TableMap(attr_name=k).table
CI_table = query_sql.subquery()
query_sql = db.session.query(CI_table.c.ci_id).join(
Table, Table.ci_id == CI_table.c.ci_id).filter(
Table.attr_id == attr.attr_id).filter(
Table.value.ilike(v.replace("*", "%")))
current_app.logger.debug(query_sql)
sort_by = kwargs.pop("sort", False)
if sort_by:
query_sql = self._sort_handler(sort_by, query_sql)
return query_sql
def update(self, type_id, type_name, type_alias, _id=None, unique=None,
icon_url="", enabled=None):
citype = CITypeCache.get(type_id)
if citype is None:
return False, "CIType {0} is not existed".format(type_name)
uniq_key = CIAttributeCache.get(_id) or CIAttributeCache.get(unique)
if uniq_key is not None:
citype.uniq_id = uniq_key.attr_id
citype_attr = db.session.query(CITypeAttribute).filter(
CITypeAttribute.type_id == type_id).filter(
CITypeAttribute.attr_id == uniq_key.attr_id).first()
if citype_attr is None:
citype_attr = CITypeAttribute()
citype_attr.attr_id = uniq_key.attr_id
citype_attr.type_id = type_id
citype_attr.is_required = True
db.session.add(citype_attr)
if type_name:
citype.type_name = type_name
if type_alias:
citype.type_alias = type_alias
if icon_url:
citype.icon_url = icon_url
if enabled is not None:
citype.enabled = enabled
db.session.add(citype)
try:
db.session.commit()
except Exception as e:
db.session.rollback()
current_app.logger.error("add CIType is error, {0}".format(str(e)))
return False, str(e)
CITypeCache.clean(type_id)
return True, type_id
def delete(self, type_id, attr_ids=None):
"""
delete attributes at CIType, attr_ids are list
"""
if not attr_ids or not isinstance(attr_ids, list):
return abort(
500, "delete attribute of CIType, attr_ids must be required")
ci_type = CITypeCache.get(type_id)
if ci_type is None:
return abort(
404, "CIType ID({0}) is not existed".format(type_id))
for attr_id in attr_ids:
attr = CIAttributeCache.get(attr_id)
if attr is None:
return abort(
404, "attribute id {0} is not existed".format(attr_id))
db.session.query(CITypeAttribute).filter_by(
type_id=type_id).filter_by(attr_id=attr_id).delete()
try:
db.session.commit()
except Exception as e:
db.session.rollback()
current_app.logger.error(
"delete attributes of CIType is error, {0}".format(str(e)))
return abort(500, "delete attributes of CIType is error")
CITypeAttributeCache.clean(type_id)
return True
def add(self, type_name, type_alias, _id=None, unique=None,
icon_url="", enabled=True):
uniq_key = CIAttributeCache.get(_id) or CIAttributeCache.get(unique)
if uniq_key is None:
return False, "uniq_key is not existed"
citype = CITypeCache.get(type_name)
if citype:
return False, "this CIType {0} is existed".format(type_name)
_citype = CIType()
_citype.type_name = type_name
_citype.type_alias = type_alias
_citype.uniq_id = uniq_key.attr_id
_citype.enabled = enabled
_citype.icon_url = icon_url
db.session.add(_citype)
db.session.flush()
_citype_attr = CITypeAttribute()
_citype_attr.attr_id = uniq_key.attr_id
_citype_attr.type_id = _citype.type_id
_citype_attr.is_required = True
db.session.add(_citype_attr)
try:
db.session.commit()
except Exception as e:
db.session.rollback()
current_app.logger.error("add CIType is error, {0}".format(str(e)))
return False, str(e)
CITypeCache.clean(type_name)
return True, _citype.type_id
def add(self, type_id, attr_ids=None, is_required=False):
"""
add attributes to CIType, attr_ids are list
"""
if not attr_ids or not isinstance(attr_ids, list):
return abort(500, "attr_ids must be required")
ci_type = CITypeCache.get(type_id)
if ci_type is None:
return abort(404, "CIType ID({0}) is not existed".format(type_id))
for attr_id in attr_ids:
attr = CIAttributeCache.get(attr_id)
if attr is None:
return abort(404,
"attribute id {0} is not existed".format(attr_id))
existed = db.session.query(CITypeAttribute.attr_id).filter_by(
type_id=type_id).filter_by(attr_id=attr_id).first()
if existed is not None:
continue
current_app.logger.debug(attr_id)
db.session.add(CITypeAttribute(
type_id=type_id, attr_id=attr_id, is_required=is_required))
try:
db.session.commit()
except Exception as e:
db.session.rollback()
current_app.logger.error(
"add attribute to CIType is error, {0}".format(str(e)))
return abort(
500, "add attribute to CIType is error, maybe duplicate entry")
CITypeAttributeCache.clean(type_id)
return True
def get_attributes_by_type(type_id=None, type_name=None):
manager = CITypeAttributeManager()
from models.attribute import CIAttributeCache
from models.ci_type import CITypeCache
from models.ci_type import CITypeAttributeCache
t = CITypeCache.get(type_id)
if not t:
t = CITypeCache.get(type_name)
if not t:
return abort(400, "CIType {0} is not existed".format(type_id))
type_id = t.type_id
uniq_id = t.uniq_id
CITypeAttributeCache.clean(type_id)
unique = CIAttributeCache.get(uniq_id).attr_name
return jsonify(attributes=manager.get_attributes_by_type_id(type_id),
type_id=type_id, uniq_id=uniq_id, unique=unique)
def set_enabled(self, type_id, enabled=True):
citype = CITypeCache.get(type_id)
if citype is None:
return abort(404, "CIType[{0}] is not existed".format(type_id))
citype.enabled = enabled
db.session.add(citype)
try:
db.session.commit()
except Exception as e:
db.session.rollback()
current_app.logger.error(
"set CIType enabled is error, {0}".format(str(e)))
return abort(500, str(e))
return type_id
def get_heartbeat():
page = get_page(request.values.get("page", 1))
ci_type = request.values.get("ci_type", "").strip()
try:
ci_type = CITypeCache.get(ci_type).type_id
except:
return jsonify(numfound=0, result=[])
agent_status = request.values.get("agent_status", None)
if agent_status:
agent_status = int(agent_status)
numfound, result = CIManager().get_heartbeat(page,
ci_type,
agent_status=agent_status)
return jsonify(numfound=numfound, result=result)
def get_children(self, parent_id):
children = db.session.query(CITypeRelation).filter(
CITypeRelation.parent_id == parent_id).all()
result = []
for child in children:
ctr_id = child.ctr_id
citype = CITypeCache.get(child.child_id)
citype_dict = row2dict(citype)
citype_dict["ctr_id"] = ctr_id
manager = CITypeAttributeManager()
citype_dict["attributes"] = manager.get_attributes_by_type_id(
citype.type_id)
citype_dict["relation_type"] = child.relation_type
result.append(citype_dict)
return result
def get_parents(self, child_id):
parents = db.session.query(CITypeRelation).filter(
CITypeRelation.child_id == child_id).all()
result = []
for parent in parents:
ctr_id = parent.ctr_id
citype = CITypeCache.get(parent.parent_id)
citype_dict = row2dict(citype)
citype_dict["ctr_id"] = ctr_id
manager = CITypeAttributeManager()
citype_dict["attributes"] = manager.get_attributes_by_type_id(
citype.type_id)
citype_dict["relation_type"] = parent.relation_type
result.append(citype_dict)
return result
def type_query_handler(self, v, only_type_query):
new_v = [v]
if v.startswith("(") and v.endswith(")"):
new_v = v[1:-1].split(";")
for _v in new_v:
ci_type = CITypeCache.get(_v)
if ci_type is not None:
self.type_id_list.append(str(ci_type.type_id))
if self.type_id_list:
type_ids = ",".join(self.type_id_list)
_query_sql = QUERY_CI_BY_TYPE.format(type_ids)
if only_type_query:
return _query_sql
else:
return ""
return ""
def get_hosts_by_product(self, product_id_list=None):
res = {}
if not product_id_list:
product = CITypeCache.get("product")
products = db.session.query(CI.ci_id).filter(
CI.type_id == product.type_id).all()
product_id_list = (product.ci_id for product in products)
product_id_list = map(str, product_id_list)
product_ids = ",".join(product_id_list)
nums = db.session.execute(QUERY_HOSTS_NUM_BY_PRODUCT.format(
"".join(["(", product_ids, ")"]))).fetchall()
if nums:
for ci_id in product_id_list:
res[int(ci_id)] = 0
for ci_id, num in nums:
res[ci_id] = num
return res
def add_heartbeat(self, ci_type, unique):
ci_type = CITypeCache.get(ci_type)
if not ci_type:
return 'error'
uniq_key = CIAttributeCache.get(ci_type.uniq_id)
Table = TableMap(attr_name=uniq_key.attr_name).table
ci_id = db.session.query(
Table.ci_id).filter(Table.attr_id == uniq_key.attr_id).filter(
Table.value == unique).first()
if ci_id is None:
return 'error'
ci = db.session.query(CI).filter(CI.ci_id == ci_id.ci_id).first()
if ci is None:
return 'error'
ci.heartbeat = datetime.datetime.now()
db.session.add(ci)
db.session.commit()
return "ok"
def get_children(self, ci_id, ret_key='name', relation_type="contain"):
second_cis = db.session.query(CIRelation.second_ci_id).filter(
CIRelation.first_ci_id == ci_id).filter(
or_(CIRelation.relation_type == relation_type,
CIRelation.relation_type == "deploy"))
second_ci_ids = (second_ci.second_ci_id for second_ci in second_cis)
ci_types = {}
for ci_id in second_ci_ids:
type_id = db.session.query(
CI.type_id).filter(CI.ci_id == ci_id).first().type_id
if type_id not in ci_types:
ci_types[type_id] = [ci_id]
else:
ci_types[type_id].append(ci_id)
res = {}
for type_id in ci_types:
ci_type = CITypeCache.get(type_id)
children = get_cis_by_ids(map(str, ci_types.get(type_id)),
ret_key=ret_key)
res[ci_type.type_name] = children
return res
def get_hosts_by_bu(self, bu_id_list=None):
res = {}
if not bu_id_list:
bu = CITypeCache.get("bu")
bus = db.session.query(
CI.ci_id).filter(CI.type_id == bu.type_id).all()
bu_id_list = (bu.ci_id for bu in bus)
bu_id_list = map(str, bu_id_list)
bu_ids = ",".join(bu_id_list)
current_app.logger.debug(
QUERY_HOSTS_NUM_BY_BU.format("".join(["(", bu_ids, ")"])))
if not bu_ids:
return res
nums = db.session.execute(
QUERY_HOSTS_NUM_BY_BU.format("".join(["(", bu_ids,
")"]))).fetchall()
if nums:
for ci_id in bu_id_list:
res[int(ci_id)] = 0
for ci_id, num in nums:
res[ci_id] = num
return res
def add(self,
ci_type_name,
exist_policy="replace",
_no_attribute_policy="ignore",
**ci_dict):
ci_existed = False
ci_type = CITypeCache.get(ci_type_name) or \
abort(404, "CIType {0} is not existed".format(ci_type_name))
unique_key = CIAttributeCache.get(ci_type.uniq_id) \
or abort(500, 'illegality unique attribute')
unique = ci_dict.get(unique_key.attr_name) \
or abort(500, '{0} missing'.format(unique_key.attr_name))
old_ci = self.ci_is_exist(ci_type, unique_key, unique)
if old_ci is not None:
ci_existed = True
if exist_policy == 'reject':
return abort(500, 'CI is existed')
if old_ci.type_id != ci_type.type_id: # update ci_type
old_ci.type_id = ci_type.type_id
db.session.add(old_ci)
db.session.flush()
ci = old_ci
else:
if exist_policy == 'need':
return abort(404, 'CI {0} not exist'.format(unique))
ci = CI()
ci.type_id = ci_type.type_id
_uuid = uuid.uuid4().hex
ci.uuid = _uuid
ci.created_time = datetime.datetime.now()
db.session.add(ci)
try:
db.session.commit()
except Exception as e:
db.session.rollback()
current_app.logger.error('add CI error: {0}'.format(str(e)))
return abort(500, 'add CI error')
value_manager = AttributeValueManager()
histories = list()
for p, v in ci_dict.items():
ret, res = value_manager.add_attr_value(
p,
v,
ci.ci_id,
ci_type,
_no_attribute_policy=_no_attribute_policy,
ci_existed=ci_existed)
if not ret:
db.session.rollback()
if not ci_existed:
self.delete(ci.ci_id)
current_app.logger.info(res)
return abort(500, res)
if res is not None:
histories.append(res)
try:
db.session.commit()
except Exception as e:
current_app.logger.error(str(e))
db.session.rollback()
if not ci_existed: # only add
self.delete(ci.ci_id)
return abort(500, "add CI error")
his_manager = CIAttributeHistoryManger()
his_manager.add(ci.ci_id, histories)
ci_cache.apply_async([ci.ci_id], queue="cmdb_async")
return ci.ci_id
def query(self, _type):
citype = CITypeCache.get(_type)
if citype:
return row2dict(citype)
return abort(404, "citype is not found")