def __fetch_objects(self):
query = []
for source in self.__job.get_sources():
condition = []
for con in source["condition"]:
if con["operator"] == "!=":
operator = {"$ne": con["value"]}
else:
operator = con["value"]
if operator in ['True', 'true']:
operator = True
elif operator in ['False', 'false']:
operator = False
else:
operator = operator
condition.append({'fields': {"$elemMatch": {"name": con["name"], "value": operator}}})
condition.append({'type_id': source["type_id"]})
condition.append({'active': {'$eq': True}})
query.append({"$and": condition})
if not source["condition"]:
query.append({'type_id': source["type_id"], 'active': {'$eq': True}})
current_objects = self.__obm.get_objects_by(sort="public_id", **{'$or': query})
result = (RenderList(current_objects, None).render_result_list())
return result
def get_dt_objects_by_type(type_id, request_user: User):
"""Return all objects by type_id"""
try:
table_config = request.args
filter_state = {'type_id': type_id}
if _fetch_only_active_objs():
filter_state['active'] = {"$eq": True}
start_at = int(table_config.get('start'))
site_length = int(table_config.get('length'))
order_column = table_config.get('order') if table_config.get('order') else 'type_id'
order_direction = 1 if table_config.get('direction') == 'asc' else -1
if order_column in ['active', 'public_id', 'type_id', 'author_id', 'creation_time']:
object_list = object_manager.get_objects_by(sort=order_column, direction=order_direction, **filter_state)
else:
object_list = object_manager.sort_objects_by_field_value(value=order_column, order=order_direction, match=filter_state)
totals = len(object_list)
object_list = object_list[start_at:start_at + site_length]
except CMDBError:
return abort(400)
rendered_list = RenderList(object_list, request_user, dt_render=True).render_result_list()
table_response = {
'data': rendered_list,
'recordsTotal': totals,
'recordsFiltered': totals
}
return make_response(table_response)
def get_objects_by_type(public_id, request_user: UserModel):
"""Return all objects by type_id"""
try:
filter_state = {'type_id': public_id}
if _fetch_only_active_objs():
filter_state['active'] = {"$eq": True}
object_list = object_manager.get_objects_by(sort="type_id",
**filter_state)
except CMDBError:
return abort(400)
if request.args.get('start') is not None:
start = int(request.args.get('start'))
length = int(request.args.get('length'))
object_list = object_list[start:start + length]
if len(object_list) < 1:
return make_response(object_list, 204)
rendered_list = RenderList(
object_list,
request_user,
database_manager=current_app.database_manager,
object_manager=object_manager).render_result_list()
resp = make_response(rendered_list)
return resp
def get_object_list(request_user: User):
"""
get all objects in database
Args:
request_user: auto inserted user who made the request.
Returns:
list of rendered objects
"""
try:
filter_state = {'sort': 'public_id'}
if _fetch_only_active_objs():
filter_state['active'] = {"$eq": True}
object_list = object_manager.get_objects_by(**filter_state)
if request.args.get('start') is not None:
start = int(request.args.get('start'))
length = int(request.args.get('length'))
object_list = object_list[start:start + length]
except ObjectManagerGetError as err:
LOGGER.error(err.message)
return abort(404)
if len(object_list) < 1:
return make_response(object_list, 204)
rendered_list = RenderList(object_list, request_user).render_result_list()
return make_response(rendered_list)
def from_database(self, database_manager, user: UserModel,
permission: AccessControlPermission):
"""Get all objects from the collection"""
manager = ObjectManager(database_manager=database_manager)
dep_object_manager = CmdbObjectManager(
database_manager=database_manager)
try:
_params = self.export_config.parameters
_result: List[CmdbObject] = manager.iterate(
filter=_params.filter,
sort=_params.sort,
order=_params.order,
limit=0,
skip=0,
user=user,
permission=permission).results
self.data = RenderList(
object_list=_result,
request_user=user,
database_manager=database_manager,
object_manager=dep_object_manager,
ref_render=True).render_result_list(raw=False)
except CMDBError as e:
return abort(400, e)
def get_objects(params: CollectionParameters, request_user: UserModel):
manager = ObjectManager(database_manager=current_app.database_manager)
view = params.optional.get('view', 'native')
if _fetch_only_active_objs():
if isinstance(params.filter, dict):
filter_ = params.filter
params.filter = [{'$match': filter_}]
params.filter.append({'$match': {'active': {"$eq": True}}})
elif isinstance(params.filter, list):
params.filter.append({'$match': {'active': {"$eq": True}}})
try:
iteration_result: IterationResult[CmdbObject] = manager.iterate(
filter=params.filter,
limit=params.limit,
skip=params.skip,
sort=params.sort,
order=params.order,
user=request_user,
permission=AccessControlPermission.READ)
if view == 'native':
object_list: List[dict] = [
object_.__dict__ for object_ in iteration_result.results
]
api_response = GetMultiResponse(object_list,
total=iteration_result.total,
params=params,
url=request.url,
model=CmdbObject.MODEL,
body=request.method == 'HEAD')
elif view == 'render':
rendered_list = RenderList(
object_list=iteration_result.results,
request_user=request_user,
database_manager=current_app.database_manager,
object_manager=object_manager,
ref_render=True).render_result_list(raw=True)
api_response = GetMultiResponse(rendered_list,
total=iteration_result.total,
params=params,
url=request.url,
model=Model('RenderResult'),
body=request.method == 'HEAD')
else:
return abort(401, 'No possible view parameter')
except ManagerIterationError as err:
return abort(400, err.message)
except ManagerGetError as err:
return abort(404, err.message)
return api_response.make_response()
def get_objects_by_user(public_id: int, request_user: User):
try:
object_list = object_manager.get_objects_by(sort="type_id", author_id=public_id)
except ObjectManagerGetError as err:
LOGGER.error(err)
return abort(400)
if len(object_list) < 1:
return make_response(object_list, 204)
rendered_list = RenderList(object_list, request_user).render_result_list()
return make_response(rendered_list)
def get_objects_by_reference(public_id: int, request_user: User):
try:
active_flag = None
if _fetch_only_active_objs():
active_flag = True
reference_list: list = object_manager.get_object_references(public_id=public_id, active_flag=active_flag)
rendered_reference_list = RenderList(reference_list, request_user).render_result_list()
except ObjectManagerGetError as err:
LOGGER.error(err)
return abort(404)
if len(reference_list) < 1:
return make_response(rendered_reference_list, 204)
return make_response(rendered_reference_list)
def get_changed_objects_since(timestamp: int, request_user: User):
request_date = datetime.fromtimestamp(timestamp, pytz.utc)
query = {'last_edit_time': {'$gt': request_date, '$ne': None}}
try:
object_list = object_manager.get_objects_by(sort="creation_time",
**query)
except ObjectManagerGetError as err:
LOGGER.error(err)
return abort(400)
if len(object_list) < 1:
return make_response(object_list, 204)
rendered_list = RenderList(object_list, request_user).render_result_list()
return make_response(rendered_list)
def get_objects_by_public_id(public_ids, request_user: User):
"""Return all objects by public_ids"""
try:
filter_state = {'public_id': public_ids}
query = _build_query(filter_state, q_operator='$or')
all_objects_list = object_manager.get_objects_by(sort="public_id",
**query)
rendered_list = RenderList(all_objects_list,
request_user).render_result_list()
except CMDBError:
return abort(400)
resp = make_response(rendered_list)
return resp
def get_latest(request_user: User):
"""
get object with newest last edit time
Args:
request_user: auto inserted user who made the request.
Returns:
list of rendered objects
"""
last_objects_list = object_manager.get_objects_by(sort='last_edit_time',
limit=25,
active={"$eq": True},
last_edit_time={'$ne': None})
rendered_list = RenderList(last_objects_list, request_user).render_result_list()
if len(rendered_list) < 1:
return make_response(rendered_list, 204)
return make_response(rendered_list)
def get_objects_by_types(type_ids, request_user: User):
"""Return all objects by type_id"""
try:
filter_state = {'type_id': type_ids}
if _fetch_only_active_objs():
filter_state['active'] = {"$eq": True}
query = _build_query(filter_state, q_operator='$or')
all_objects_list = object_manager.get_objects_by(sort="type_id", **query)
rendered_list = RenderList(all_objects_list, request_user).render_result_list()
except CMDBError:
return abort(400)
resp = make_response(rendered_list)
return resp
def get_new_objects_since(timestamp: int, request_user: UserModel):
request_date = datetime.fromtimestamp(timestamp, pytz.utc)
query = {'creation_time': {'$gt': request_date}}
try:
object_list = object_manager.get_objects_by(sort="creation_time",
**query)
except ObjectManagerGetError as err:
LOGGER.error(err)
return abort(400)
if len(object_list) < 1:
return make_response(object_list, 204)
rendered_list = RenderList(
object_list,
request_user,
database_manager=current_app.database_manager).render_result_list()
return make_response(rendered_list)
def get_newest(request_user: UserModel):
"""
get object with newest creation time
Args:
request_user: auto inserted user who made the request.
Returns:
list of rendered objects
"""
newest_objects_list = object_manager.get_objects_by(
sort='creation_time',
limit=10,
active={"$eq": True},
creation_time={'$ne': None})
rendered_list = RenderList(newest_objects_list,
request_user).render_result_list()
if len(rendered_list) < 1:
return make_response(rendered_list, 204)
return make_response(rendered_list)
def get_objects_by_types(type_ids, request_user: User):
"""Return all objects by type_id"""
try:
in_types = {'type_id': {'$in': list(map(int, type_ids.split(',')))}}
is_active = {'active': {"$eq": False}}
if _fetch_only_active_objs:
is_active = {'active': {"$eq": True}}
query = {'$and': [in_types, is_active]}
all_objects_list = object_manager.get_objects_by(sort="type_id",
**query)
rendered_list = RenderList(all_objects_list,
request_user).render_result_list()
except CMDBError:
return abort(400)
resp = make_response(rendered_list)
return resp
def get_latest(request_user: UserModel):
"""
get object with newest last edit time
Args:
request_user: auto inserted user who made the request.
Returns:
list of rendered objects
"""
last_objects_list = object_manager.get_objects_by(
sort='last_edit_time',
limit=10,
active={"$eq": True},
last_edit_time={'$ne': None},
user=request_user,
permission=AccessControlPermission.READ)
rendered_list = RenderList(
last_objects_list,
request_user,
database_manager=current_app.database_manager).render_result_list()
if len(rendered_list) < 1:
return make_response(rendered_list, 204)
return make_response(rendered_list)
def get_objects_by_reference(public_id: int, request_user: UserModel):
try:
active_flag = None
if _fetch_only_active_objs():
active_flag = True
reference_list: list = object_manager.get_object_references(
public_id=public_id,
active_flag=active_flag,
user=request_user,
permission=AccessControlPermission.READ)
rendered_reference_list = RenderList(
object_list=reference_list,
request_user=request_user,
ref_render=True,
object_manager=object_manager).render_result_list()
except ObjectManagerGetError as err:
return abort(404, err.message)
except AccessDeniedError as err:
return abort(403, err.message)
if len(reference_list) < 1:
return make_response(rendered_reference_list, 204)
return make_response(rendered_reference_list)
def aggregate(self,
pipeline: Pipeline,
request_user: UserModel = None,
limit: int = Search.DEFAULT_LIMIT,
skip: int = Search.DEFAULT_SKIP,
**kwargs) -> SearchResult[RenderResult]:
"""
Use mongodb aggregation system with pipeline queries
Args:
pipeline (Pipeline): list of requirement pipes
request_user (UserModel): user who started this search
limit (int): max number of documents to return
skip (int): number of documents to be skipped
**kwargs:
Returns:
SearchResult with generic list of RenderResults
"""
# Insert skip and limit
plb = PipelineBuilder(pipeline)
# define search output
stages: dict = {}
active = kwargs.get('active', True)
if kwargs.get('resolve', False):
plb.add_pipe(
plb.lookup_sub_(from_='framework.objects',
let_={'ref_id': '$public_id'},
pipeline_=[
plb.match_({
'$expr': {
'$in':
['$$ref_id', '$fields.value']
}
})
],
as_='refs'))
if active:
active_pipe = [{
'$match': {
'active': {
"$eq": True
}
}
}, {
'$match': {
'$expr': {
'$in': ['$$ref_id', '$fields.value']
}
}
}]
else:
active_pipe = [{
'$match': {
'$expr': {
'$in': ['$$ref_id', '$fields.value']
}
}
}]
plb.add_pipe(
plb.facet_({
'root': [{
'$replaceRoot': {
'newRoot': {
'$mergeObjects': ['$$ROOT']
}
}
}],
'references': [{
'$lookup': {
'from': 'framework.objects',
'let': {
'ref_id': '$public_id'
},
'pipeline': active_pipe,
'as': 'refs'
}
}, {
'$unwind': '$refs'
}, {
'$replaceRoot': {
'newRoot': '$refs'
}
}]
}))
plb.add_pipe(
plb.project_(specification={
'complete': {
'$concatArrays': ['$root', '$references']
}
}))
plb.add_pipe(plb.unwind_(path='$complete'))
plb.add_pipe({'$replaceRoot': {'newRoot': '$complete'}})
stages.update({'metadata': [PipelineBuilder.count_('total')]})
stages.update({
'data':
[PipelineBuilder.skip_(skip),
PipelineBuilder.limit_(limit)]
})
group_stage: dict = {
'group': [
PipelineBuilder.lookup_(TypeModel.COLLECTION, 'type_id',
'public_id', 'lookup_data'),
PipelineBuilder.unwind_('$lookup_data'),
PipelineBuilder.project_({
'_id': 0,
'type_id': 1,
'label': '$lookup_data.label'
}),
PipelineBuilder.group_('$$ROOT.type_id', {
'types': {
'$first': '$$ROOT'
},
'total': {
'$sum': 1
}
}),
PipelineBuilder.project_({
'_id': 0,
'searchText': '$types.label',
'searchForm': 'type',
'searchLabel': '$types.label',
'settings': {
'types': ['$types.type_id']
},
'total': 1
}),
PipelineBuilder.sort_('total', -1)
]
}
stages.update(group_stage)
plb.add_pipe(PipelineBuilder.facet_(stages))
raw_search_result = self.manager.aggregate(
collection=CmdbObject.COLLECTION, pipeline=plb.pipeline)
raw_search_result_list = list(raw_search_result)
try:
matches_regex = plb.get_regex_pipes_values()
except Exception as err:
LOGGER.error(f'Extract regex pipes: {err}')
matches_regex = []
if len(raw_search_result_list[0]['data']) > 0:
raw_search_result_list_entry = raw_search_result_list[0]
# parse result list
pre_rendered_result_list = [
CmdbObject(**raw_result)
for raw_result in raw_search_result_list_entry['data']
]
rendered_result_list = RenderList(
pre_rendered_result_list,
request_user,
object_manager=self.manager).render_result_list()
total_results = raw_search_result_list_entry['metadata'][0].get(
'total', 0)
group_result_list = raw_search_result_list[0]['group']
else:
rendered_result_list = []
group_result_list = []
total_results = 0
# generate output
search_result = SearchResult[RenderResult](
results=rendered_result_list,
total_results=total_results,
groups=group_result_list,
alive=raw_search_result.alive,
matches_regex=matches_regex,
limit=limit,
skip=skip)
return search_result
def get_dt_filter_objects_by_type(type_id, request_user: User):
"""Return all objects by type_id"""
try:
table_config = request.args
filter_ids = table_config.getlist('idList')
start_at = int(table_config.get('start', 0, int))
site_length = int(table_config.get('length', 10, int))
search_for = table_config.get('search', '', str)
order_column = table_config.get('order', 'type_id', str)
order_direction = 1 if table_config.get('direction') == 'asc' else -1
dt_render = False if table_config.get('dtRender') == 'false' else True
# Prepare search term
if search_for in ['true', 'True']:
search_for = True
elif search_for in ['false', 'False']:
search_for = False
elif search_for.isdigit():
search_for = int(search_for)
# Filter Objects by IDS
filter_arg = []
if filter_ids:
filter_arg.append(
{'public_id': {
'$in': list(map(int, filter_ids))
}})
# Search default values
filter_arg.append({'type_id': type_id})
if _fetch_only_active_objs():
filter_arg.append({'active': {"$eq": True}})
# Search search term over entire object
or_conditions = []
if not isinstance(search_for, bool):
search_term = {'$regex': str(search_for), '$options': 'i'}
else:
search_term = search_for
or_conditions.append(
{'fields': {
'$elemMatch': {
'value': search_term
}
}})
# ToDo: Find - convert string to date
# or_conditions.append({'creation_time': {'$toDate': str(search_for)}})
if isinstance(search_for, int) and not isinstance(search_for, bool):
if order_column in ['public_id', 'author_id']:
or_conditions.append({
'$where':
"this.public_id.toString().includes(%s)" % search_for
})
or_conditions.append({
'$where':
"this.author_id.toString().includes(%s)" % search_for
})
else:
or_conditions.append({'public_id': search_for})
or_conditions.append({'author_id': search_for})
# Linking queries
filter_arg.append({'$or': or_conditions})
filter_state = {'$and': filter_arg}
if order_column in [
'active', 'public_id', 'type_id', 'author_id', 'creation_time'
]:
object_list = object_manager.get_objects_by(
sort=order_column, direction=order_direction, **filter_state)
else:
object_list = object_manager.sort_objects_by_field_value(
value=order_column, order=order_direction, match=filter_state)
totals = len(object_list)
object_list = object_list[start_at:start_at + site_length]
except CMDBError:
return abort(400)
rendered_list = RenderList(object_list, request_user,
dt_render=dt_render).render_result_list()
table_response = {
'data': rendered_list,
'recordsTotal': totals,
'recordsFiltered': totals
}
return make_response(table_response)
def aggregate(self,
pipeline: Pipeline,
request_user: UserModel = None,
permission: AccessControlPermission = None,
limit: int = Search.DEFAULT_LIMIT,
skip: int = Search.DEFAULT_SKIP,
**kwargs) -> SearchResult[RenderResult]:
"""
Use mongodb aggregation system with pipeline queries
Args:
pipeline (Pipeline): list of requirement pipes
request_user (UserModel): user who started this search
permission (AccessControlPermission) : Permission enum for possible ACL operations..
limit (int): max number of documents to return
skip (int): number of documents to be skipped
**kwargs:
Returns:
SearchResult with generic list of RenderResults
"""
# Insert skip and limit
plb = SearchPipelineBuilder(pipeline)
# define search output
stages: dict = {}
stages.update({'metadata': [SearchPipelineBuilder.count_('total')]})
stages.update({
'data': [
SearchPipelineBuilder.skip_(skip),
SearchPipelineBuilder.limit_(limit)
]
})
group_stage: dict = {
'group': [
SearchPipelineBuilder.lookup_(TypeModel.COLLECTION, 'type_id',
'public_id', 'lookup_data'),
SearchPipelineBuilder.unwind_('$lookup_data'),
SearchPipelineBuilder.project_({
'_id': 0,
'type_id': 1,
'label': '$lookup_data.label'
}),
SearchPipelineBuilder.group_('$$ROOT.type_id', {
'types': {
'$first': '$$ROOT'
},
'total': {
'$sum': 1
}
}),
SearchPipelineBuilder.project_({
'_id': 0,
'searchText': '$types.label',
'searchForm': 'type',
'searchLabel': '$types.label',
'settings': {
'types': ['$types.type_id']
},
'total': 1
}),
SearchPipelineBuilder.sort_('total', -1)
]
}
stages.update(group_stage)
plb.add_pipe(SearchPipelineBuilder.facet_(stages))
raw_search_result = self.manager.aggregate(
collection=CmdbObject.COLLECTION, pipeline=plb.pipeline)
raw_search_result_list = list(raw_search_result)
try:
matches_regex = plb.get_regex_pipes_values()
except Exception as err:
LOGGER.error(f'Extract regex pipes: {err}')
matches_regex = []
if len(raw_search_result_list[0]['data']) > 0:
raw_search_result_list_entry = raw_search_result_list[0]
# parse result list
pre_rendered_result_list = [
CmdbObject(**raw_result)
for raw_result in raw_search_result_list_entry['data']
]
rendered_result_list = RenderList(
pre_rendered_result_list,
request_user,
database_manager=self.manager.dbm,
object_manager=self.manager).render_result_list()
total_results = raw_search_result_list_entry['metadata'][0].get(
'total', 0)
group_result_list = raw_search_result_list[0]['group']
else:
rendered_result_list = []
group_result_list = []
total_results = 0
# generate output
search_result = SearchResult[RenderResult](
results=rendered_result_list,
total_results=total_results,
groups=group_result_list,
alive=raw_search_result.alive,
matches_regex=matches_regex,
limit=limit,
skip=skip)
return search_result
def aggregate(self,
pipeline: Pipeline,
request_user: User = None,
limit: int = Search.DEFAULT_LIMIT,
skip: int = Search.DEFAULT_SKIP,
**kwargs) -> SearchResult[RenderResult]:
"""
Use mongodb aggregation system with pipeline queries
Args:
pipeline (Pipeline): list of requirement pipes
request_user (User): user who started this search
matches_regex (List): list of regex match values
limit (int): max number of documents to return
skip (int): number of documents to be skipped
**kwargs:
Returns:
SearchResult with generic list of RenderResults
"""
# Insert skip and limit
plb = PipelineBuilder(pipeline)
# define search output
stages: dict = {
'metadata': [PipelineBuilder.count_('total')],
'data':
[PipelineBuilder.skip_(skip),
PipelineBuilder.limit_(limit)]
}
plb.add_pipe(PipelineBuilder.facet_(stages))
raw_search_result = self.manager.aggregate(
collection=CmdbObject.COLLECTION, pipeline=plb.pipeline)
raw_search_result_list = list(raw_search_result)
try:
matches_regex = plb.get_regex_pipes_values()
except Exception as err:
LOGGER.error(f'Extract regex pipes: {err}')
matches_regex = []
if len(raw_search_result_list[0]['data']) > 0:
raw_search_result_list_entry = raw_search_result_list[0]
# parse result list
pre_rendered_result_list = [
CmdbObject(**raw_result)
for raw_result in raw_search_result_list_entry['data']
]
rendered_result_list = RenderList(
pre_rendered_result_list,
request_user,
object_manager=self.manager).render_result_list()
total_results = raw_search_result_list_entry['metadata'][0].get(
'total', 0)
else:
rendered_result_list = []
total_results = 0
# generate output
search_result = SearchResult[RenderResult](
results=rendered_result_list,
total_results=total_results,
alive=raw_search_result.alive,
matches_regex=matches_regex,
limit=limit,
skip=skip)
return search_result
def __fetch_objects(self):
query = []
result = []
condition = []
subset: bool = self.__job.scheduling['event'].get('subset', False)
if subset and self.event.get_param('event') in ['delete']:
from cmdb.framework.cmdb_render import RenderResult
deleted = RenderResult()
deleted.object_information['object_id'] = self.event.get_param(
'id')
deleted.type_information['type_id'] = self.event.get_param(
'type_id')
result.append(deleted)
else:
if subset and self.event.get_param('event') in [
'insert', 'update'
]:
condition.append({'public_id': self.event.get_param('id')})
for source in self.__job.get_sources():
temp = []
for con in source["condition"]:
operator = con["operator"]
value = con["value"]
regex = {
"$ne": con["value"]
} if operator == "!=" else {
"$regex": value,
"$options": "si"
}
regex = True if value in [
'True', 'true'
] else False if value in ['False', 'false'] else regex
temp.append({
'fields': {
"$elemMatch": {
"name": con["name"],
"value": regex
}
}
})
temp.append({'type_id': source["type_id"]})
temp.append({'active': {'$eq': True}})
query.append({"$and": [*condition, *temp]})
if not source["condition"]:
query.append({
'type_id': source["type_id"],
'active': {
'$eq': True
}
})
current_objects = self.__obm.get_objects_by(sort="public_id",
**{'$or': query})
result = (RenderList(
current_objects,
None,
database_manager=self.__obm.dbm,
object_manager=self.__obm).render_result_list())
return result
