def __fetch_objects(self): query = [] for source in self.__job.get_sources(): condition = [] for con in source["condition"]: if con["operator"] == "!=": operator = {"$ne": con["value"]} else: operator = con["value"] if operator in ['True', 'true']: operator = True elif operator in ['False', 'false']: operator = False else: operator = operator condition.append({'fields': {"$elemMatch": {"name": con["name"], "value": operator}}}) condition.append({'type_id': source["type_id"]}) condition.append({'active': {'$eq': True}}) query.append({"$and": condition}) if not source["condition"]: query.append({'type_id': source["type_id"], 'active': {'$eq': True}}) current_objects = self.__obm.get_objects_by(sort="public_id", **{'$or': query}) result = (RenderList(current_objects, None).render_result_list()) return result
def get_dt_objects_by_type(type_id, request_user: User): """Return all objects by type_id""" try: table_config = request.args filter_state = {'type_id': type_id} if _fetch_only_active_objs(): filter_state['active'] = {"$eq": True} start_at = int(table_config.get('start')) site_length = int(table_config.get('length')) order_column = table_config.get('order') if table_config.get('order') else 'type_id' order_direction = 1 if table_config.get('direction') == 'asc' else -1 if order_column in ['active', 'public_id', 'type_id', 'author_id', 'creation_time']: object_list = object_manager.get_objects_by(sort=order_column, direction=order_direction, **filter_state) else: object_list = object_manager.sort_objects_by_field_value(value=order_column, order=order_direction, match=filter_state) totals = len(object_list) object_list = object_list[start_at:start_at + site_length] except CMDBError: return abort(400) rendered_list = RenderList(object_list, request_user, dt_render=True).render_result_list() table_response = { 'data': rendered_list, 'recordsTotal': totals, 'recordsFiltered': totals } return make_response(table_response)
def get_objects_by_type(public_id, request_user: UserModel): """Return all objects by type_id""" try: filter_state = {'type_id': public_id} if _fetch_only_active_objs(): filter_state['active'] = {"$eq": True} object_list = object_manager.get_objects_by(sort="type_id", **filter_state) except CMDBError: return abort(400) if request.args.get('start') is not None: start = int(request.args.get('start')) length = int(request.args.get('length')) object_list = object_list[start:start + length] if len(object_list) < 1: return make_response(object_list, 204) rendered_list = RenderList( object_list, request_user, database_manager=current_app.database_manager, object_manager=object_manager).render_result_list() resp = make_response(rendered_list) return resp
def get_object_list(request_user: User): """ get all objects in database Args: request_user: auto inserted user who made the request. Returns: list of rendered objects """ try: filter_state = {'sort': 'public_id'} if _fetch_only_active_objs(): filter_state['active'] = {"$eq": True} object_list = object_manager.get_objects_by(**filter_state) if request.args.get('start') is not None: start = int(request.args.get('start')) length = int(request.args.get('length')) object_list = object_list[start:start + length] except ObjectManagerGetError as err: LOGGER.error(err.message) return abort(404) if len(object_list) < 1: return make_response(object_list, 204) rendered_list = RenderList(object_list, request_user).render_result_list() return make_response(rendered_list)
def from_database(self, database_manager, user: UserModel, permission: AccessControlPermission): """Get all objects from the collection""" manager = ObjectManager(database_manager=database_manager) dep_object_manager = CmdbObjectManager( database_manager=database_manager) try: _params = self.export_config.parameters _result: List[CmdbObject] = manager.iterate( filter=_params.filter, sort=_params.sort, order=_params.order, limit=0, skip=0, user=user, permission=permission).results self.data = RenderList( object_list=_result, request_user=user, database_manager=database_manager, object_manager=dep_object_manager, ref_render=True).render_result_list(raw=False) except CMDBError as e: return abort(400, e)
def get_objects(params: CollectionParameters, request_user: UserModel): manager = ObjectManager(database_manager=current_app.database_manager) view = params.optional.get('view', 'native') if _fetch_only_active_objs(): if isinstance(params.filter, dict): filter_ = params.filter params.filter = [{'$match': filter_}] params.filter.append({'$match': {'active': {"$eq": True}}}) elif isinstance(params.filter, list): params.filter.append({'$match': {'active': {"$eq": True}}}) try: iteration_result: IterationResult[CmdbObject] = manager.iterate( filter=params.filter, limit=params.limit, skip=params.skip, sort=params.sort, order=params.order, user=request_user, permission=AccessControlPermission.READ) if view == 'native': object_list: List[dict] = [ object_.__dict__ for object_ in iteration_result.results ] api_response = GetMultiResponse(object_list, total=iteration_result.total, params=params, url=request.url, model=CmdbObject.MODEL, body=request.method == 'HEAD') elif view == 'render': rendered_list = RenderList( object_list=iteration_result.results, request_user=request_user, database_manager=current_app.database_manager, object_manager=object_manager, ref_render=True).render_result_list(raw=True) api_response = GetMultiResponse(rendered_list, total=iteration_result.total, params=params, url=request.url, model=Model('RenderResult'), body=request.method == 'HEAD') else: return abort(401, 'No possible view parameter') except ManagerIterationError as err: return abort(400, err.message) except ManagerGetError as err: return abort(404, err.message) return api_response.make_response()
def get_objects_by_user(public_id: int, request_user: User): try: object_list = object_manager.get_objects_by(sort="type_id", author_id=public_id) except ObjectManagerGetError as err: LOGGER.error(err) return abort(400) if len(object_list) < 1: return make_response(object_list, 204) rendered_list = RenderList(object_list, request_user).render_result_list() return make_response(rendered_list)
def get_objects_by_reference(public_id: int, request_user: User): try: active_flag = None if _fetch_only_active_objs(): active_flag = True reference_list: list = object_manager.get_object_references(public_id=public_id, active_flag=active_flag) rendered_reference_list = RenderList(reference_list, request_user).render_result_list() except ObjectManagerGetError as err: LOGGER.error(err) return abort(404) if len(reference_list) < 1: return make_response(rendered_reference_list, 204) return make_response(rendered_reference_list)
def get_changed_objects_since(timestamp: int, request_user: User): request_date = datetime.fromtimestamp(timestamp, pytz.utc) query = {'last_edit_time': {'$gt': request_date, '$ne': None}} try: object_list = object_manager.get_objects_by(sort="creation_time", **query) except ObjectManagerGetError as err: LOGGER.error(err) return abort(400) if len(object_list) < 1: return make_response(object_list, 204) rendered_list = RenderList(object_list, request_user).render_result_list() return make_response(rendered_list)
def get_objects_by_public_id(public_ids, request_user: User): """Return all objects by public_ids""" try: filter_state = {'public_id': public_ids} query = _build_query(filter_state, q_operator='$or') all_objects_list = object_manager.get_objects_by(sort="public_id", **query) rendered_list = RenderList(all_objects_list, request_user).render_result_list() except CMDBError: return abort(400) resp = make_response(rendered_list) return resp
def get_latest(request_user: User): """ get object with newest last edit time Args: request_user: auto inserted user who made the request. Returns: list of rendered objects """ last_objects_list = object_manager.get_objects_by(sort='last_edit_time', limit=25, active={"$eq": True}, last_edit_time={'$ne': None}) rendered_list = RenderList(last_objects_list, request_user).render_result_list() if len(rendered_list) < 1: return make_response(rendered_list, 204) return make_response(rendered_list)
def get_objects_by_types(type_ids, request_user: User): """Return all objects by type_id""" try: filter_state = {'type_id': type_ids} if _fetch_only_active_objs(): filter_state['active'] = {"$eq": True} query = _build_query(filter_state, q_operator='$or') all_objects_list = object_manager.get_objects_by(sort="type_id", **query) rendered_list = RenderList(all_objects_list, request_user).render_result_list() except CMDBError: return abort(400) resp = make_response(rendered_list) return resp
def get_new_objects_since(timestamp: int, request_user: UserModel): request_date = datetime.fromtimestamp(timestamp, pytz.utc) query = {'creation_time': {'$gt': request_date}} try: object_list = object_manager.get_objects_by(sort="creation_time", **query) except ObjectManagerGetError as err: LOGGER.error(err) return abort(400) if len(object_list) < 1: return make_response(object_list, 204) rendered_list = RenderList( object_list, request_user, database_manager=current_app.database_manager).render_result_list() return make_response(rendered_list)
def get_newest(request_user: UserModel): """ get object with newest creation time Args: request_user: auto inserted user who made the request. Returns: list of rendered objects """ newest_objects_list = object_manager.get_objects_by( sort='creation_time', limit=10, active={"$eq": True}, creation_time={'$ne': None}) rendered_list = RenderList(newest_objects_list, request_user).render_result_list() if len(rendered_list) < 1: return make_response(rendered_list, 204) return make_response(rendered_list)
def get_objects_by_types(type_ids, request_user: User): """Return all objects by type_id""" try: in_types = {'type_id': {'$in': list(map(int, type_ids.split(',')))}} is_active = {'active': {"$eq": False}} if _fetch_only_active_objs: is_active = {'active': {"$eq": True}} query = {'$and': [in_types, is_active]} all_objects_list = object_manager.get_objects_by(sort="type_id", **query) rendered_list = RenderList(all_objects_list, request_user).render_result_list() except CMDBError: return abort(400) resp = make_response(rendered_list) return resp
def get_latest(request_user: UserModel): """ get object with newest last edit time Args: request_user: auto inserted user who made the request. Returns: list of rendered objects """ last_objects_list = object_manager.get_objects_by( sort='last_edit_time', limit=10, active={"$eq": True}, last_edit_time={'$ne': None}, user=request_user, permission=AccessControlPermission.READ) rendered_list = RenderList( last_objects_list, request_user, database_manager=current_app.database_manager).render_result_list() if len(rendered_list) < 1: return make_response(rendered_list, 204) return make_response(rendered_list)
def get_objects_by_reference(public_id: int, request_user: UserModel): try: active_flag = None if _fetch_only_active_objs(): active_flag = True reference_list: list = object_manager.get_object_references( public_id=public_id, active_flag=active_flag, user=request_user, permission=AccessControlPermission.READ) rendered_reference_list = RenderList( object_list=reference_list, request_user=request_user, ref_render=True, object_manager=object_manager).render_result_list() except ObjectManagerGetError as err: return abort(404, err.message) except AccessDeniedError as err: return abort(403, err.message) if len(reference_list) < 1: return make_response(rendered_reference_list, 204) return make_response(rendered_reference_list)
def aggregate(self, pipeline: Pipeline, request_user: UserModel = None, limit: int = Search.DEFAULT_LIMIT, skip: int = Search.DEFAULT_SKIP, **kwargs) -> SearchResult[RenderResult]: """ Use mongodb aggregation system with pipeline queries Args: pipeline (Pipeline): list of requirement pipes request_user (UserModel): user who started this search limit (int): max number of documents to return skip (int): number of documents to be skipped **kwargs: Returns: SearchResult with generic list of RenderResults """ # Insert skip and limit plb = PipelineBuilder(pipeline) # define search output stages: dict = {} active = kwargs.get('active', True) if kwargs.get('resolve', False): plb.add_pipe( plb.lookup_sub_(from_='framework.objects', let_={'ref_id': '$public_id'}, pipeline_=[ plb.match_({ '$expr': { '$in': ['$$ref_id', '$fields.value'] } }) ], as_='refs')) if active: active_pipe = [{ '$match': { 'active': { "$eq": True } } }, { '$match': { '$expr': { '$in': ['$$ref_id', '$fields.value'] } } }] else: active_pipe = [{ '$match': { '$expr': { '$in': ['$$ref_id', '$fields.value'] } } }] plb.add_pipe( plb.facet_({ 'root': [{ '$replaceRoot': { 'newRoot': { '$mergeObjects': ['$$ROOT'] } } }], 'references': [{ '$lookup': { 'from': 'framework.objects', 'let': { 'ref_id': '$public_id' }, 'pipeline': active_pipe, 'as': 'refs' } }, { '$unwind': '$refs' }, { '$replaceRoot': { 'newRoot': '$refs' } }] })) plb.add_pipe( plb.project_(specification={ 'complete': { '$concatArrays': ['$root', '$references'] } })) plb.add_pipe(plb.unwind_(path='$complete')) plb.add_pipe({'$replaceRoot': {'newRoot': '$complete'}}) stages.update({'metadata': [PipelineBuilder.count_('total')]}) stages.update({ 'data': [PipelineBuilder.skip_(skip), PipelineBuilder.limit_(limit)] }) group_stage: dict = { 'group': [ PipelineBuilder.lookup_(TypeModel.COLLECTION, 'type_id', 'public_id', 'lookup_data'), PipelineBuilder.unwind_('$lookup_data'), PipelineBuilder.project_({ '_id': 0, 'type_id': 1, 'label': '$lookup_data.label' }), PipelineBuilder.group_('$$ROOT.type_id', { 'types': { '$first': '$$ROOT' }, 'total': { '$sum': 1 } }), PipelineBuilder.project_({ '_id': 0, 'searchText': '$types.label', 'searchForm': 'type', 'searchLabel': '$types.label', 'settings': { 'types': ['$types.type_id'] }, 'total': 1 }), PipelineBuilder.sort_('total', -1) ] } stages.update(group_stage) plb.add_pipe(PipelineBuilder.facet_(stages)) raw_search_result = self.manager.aggregate( collection=CmdbObject.COLLECTION, pipeline=plb.pipeline) raw_search_result_list = list(raw_search_result) try: matches_regex = plb.get_regex_pipes_values() except Exception as err: LOGGER.error(f'Extract regex pipes: {err}') matches_regex = [] if len(raw_search_result_list[0]['data']) > 0: raw_search_result_list_entry = raw_search_result_list[0] # parse result list pre_rendered_result_list = [ CmdbObject(**raw_result) for raw_result in raw_search_result_list_entry['data'] ] rendered_result_list = RenderList( pre_rendered_result_list, request_user, object_manager=self.manager).render_result_list() total_results = raw_search_result_list_entry['metadata'][0].get( 'total', 0) group_result_list = raw_search_result_list[0]['group'] else: rendered_result_list = [] group_result_list = [] total_results = 0 # generate output search_result = SearchResult[RenderResult]( results=rendered_result_list, total_results=total_results, groups=group_result_list, alive=raw_search_result.alive, matches_regex=matches_regex, limit=limit, skip=skip) return search_result
def get_dt_filter_objects_by_type(type_id, request_user: User): """Return all objects by type_id""" try: table_config = request.args filter_ids = table_config.getlist('idList') start_at = int(table_config.get('start', 0, int)) site_length = int(table_config.get('length', 10, int)) search_for = table_config.get('search', '', str) order_column = table_config.get('order', 'type_id', str) order_direction = 1 if table_config.get('direction') == 'asc' else -1 dt_render = False if table_config.get('dtRender') == 'false' else True # Prepare search term if search_for in ['true', 'True']: search_for = True elif search_for in ['false', 'False']: search_for = False elif search_for.isdigit(): search_for = int(search_for) # Filter Objects by IDS filter_arg = [] if filter_ids: filter_arg.append( {'public_id': { '$in': list(map(int, filter_ids)) }}) # Search default values filter_arg.append({'type_id': type_id}) if _fetch_only_active_objs(): filter_arg.append({'active': {"$eq": True}}) # Search search term over entire object or_conditions = [] if not isinstance(search_for, bool): search_term = {'$regex': str(search_for), '$options': 'i'} else: search_term = search_for or_conditions.append( {'fields': { '$elemMatch': { 'value': search_term } }}) # ToDo: Find - convert string to date # or_conditions.append({'creation_time': {'$toDate': str(search_for)}}) if isinstance(search_for, int) and not isinstance(search_for, bool): if order_column in ['public_id', 'author_id']: or_conditions.append({ '$where': "this.public_id.toString().includes(%s)" % search_for }) or_conditions.append({ '$where': "this.author_id.toString().includes(%s)" % search_for }) else: or_conditions.append({'public_id': search_for}) or_conditions.append({'author_id': search_for}) # Linking queries filter_arg.append({'$or': or_conditions}) filter_state = {'$and': filter_arg} if order_column in [ 'active', 'public_id', 'type_id', 'author_id', 'creation_time' ]: object_list = object_manager.get_objects_by( sort=order_column, direction=order_direction, **filter_state) else: object_list = object_manager.sort_objects_by_field_value( value=order_column, order=order_direction, match=filter_state) totals = len(object_list) object_list = object_list[start_at:start_at + site_length] except CMDBError: return abort(400) rendered_list = RenderList(object_list, request_user, dt_render=dt_render).render_result_list() table_response = { 'data': rendered_list, 'recordsTotal': totals, 'recordsFiltered': totals } return make_response(table_response)
def aggregate(self, pipeline: Pipeline, request_user: UserModel = None, permission: AccessControlPermission = None, limit: int = Search.DEFAULT_LIMIT, skip: int = Search.DEFAULT_SKIP, **kwargs) -> SearchResult[RenderResult]: """ Use mongodb aggregation system with pipeline queries Args: pipeline (Pipeline): list of requirement pipes request_user (UserModel): user who started this search permission (AccessControlPermission) : Permission enum for possible ACL operations.. limit (int): max number of documents to return skip (int): number of documents to be skipped **kwargs: Returns: SearchResult with generic list of RenderResults """ # Insert skip and limit plb = SearchPipelineBuilder(pipeline) # define search output stages: dict = {} stages.update({'metadata': [SearchPipelineBuilder.count_('total')]}) stages.update({ 'data': [ SearchPipelineBuilder.skip_(skip), SearchPipelineBuilder.limit_(limit) ] }) group_stage: dict = { 'group': [ SearchPipelineBuilder.lookup_(TypeModel.COLLECTION, 'type_id', 'public_id', 'lookup_data'), SearchPipelineBuilder.unwind_('$lookup_data'), SearchPipelineBuilder.project_({ '_id': 0, 'type_id': 1, 'label': '$lookup_data.label' }), SearchPipelineBuilder.group_('$$ROOT.type_id', { 'types': { '$first': '$$ROOT' }, 'total': { '$sum': 1 } }), SearchPipelineBuilder.project_({ '_id': 0, 'searchText': '$types.label', 'searchForm': 'type', 'searchLabel': '$types.label', 'settings': { 'types': ['$types.type_id'] }, 'total': 1 }), SearchPipelineBuilder.sort_('total', -1) ] } stages.update(group_stage) plb.add_pipe(SearchPipelineBuilder.facet_(stages)) raw_search_result = self.manager.aggregate( collection=CmdbObject.COLLECTION, pipeline=plb.pipeline) raw_search_result_list = list(raw_search_result) try: matches_regex = plb.get_regex_pipes_values() except Exception as err: LOGGER.error(f'Extract regex pipes: {err}') matches_regex = [] if len(raw_search_result_list[0]['data']) > 0: raw_search_result_list_entry = raw_search_result_list[0] # parse result list pre_rendered_result_list = [ CmdbObject(**raw_result) for raw_result in raw_search_result_list_entry['data'] ] rendered_result_list = RenderList( pre_rendered_result_list, request_user, database_manager=self.manager.dbm, object_manager=self.manager).render_result_list() total_results = raw_search_result_list_entry['metadata'][0].get( 'total', 0) group_result_list = raw_search_result_list[0]['group'] else: rendered_result_list = [] group_result_list = [] total_results = 0 # generate output search_result = SearchResult[RenderResult]( results=rendered_result_list, total_results=total_results, groups=group_result_list, alive=raw_search_result.alive, matches_regex=matches_regex, limit=limit, skip=skip) return search_result
def aggregate(self, pipeline: Pipeline, request_user: User = None, limit: int = Search.DEFAULT_LIMIT, skip: int = Search.DEFAULT_SKIP, **kwargs) -> SearchResult[RenderResult]: """ Use mongodb aggregation system with pipeline queries Args: pipeline (Pipeline): list of requirement pipes request_user (User): user who started this search matches_regex (List): list of regex match values limit (int): max number of documents to return skip (int): number of documents to be skipped **kwargs: Returns: SearchResult with generic list of RenderResults """ # Insert skip and limit plb = PipelineBuilder(pipeline) # define search output stages: dict = { 'metadata': [PipelineBuilder.count_('total')], 'data': [PipelineBuilder.skip_(skip), PipelineBuilder.limit_(limit)] } plb.add_pipe(PipelineBuilder.facet_(stages)) raw_search_result = self.manager.aggregate( collection=CmdbObject.COLLECTION, pipeline=plb.pipeline) raw_search_result_list = list(raw_search_result) try: matches_regex = plb.get_regex_pipes_values() except Exception as err: LOGGER.error(f'Extract regex pipes: {err}') matches_regex = [] if len(raw_search_result_list[0]['data']) > 0: raw_search_result_list_entry = raw_search_result_list[0] # parse result list pre_rendered_result_list = [ CmdbObject(**raw_result) for raw_result in raw_search_result_list_entry['data'] ] rendered_result_list = RenderList( pre_rendered_result_list, request_user, object_manager=self.manager).render_result_list() total_results = raw_search_result_list_entry['metadata'][0].get( 'total', 0) else: rendered_result_list = [] total_results = 0 # generate output search_result = SearchResult[RenderResult]( results=rendered_result_list, total_results=total_results, alive=raw_search_result.alive, matches_regex=matches_regex, limit=limit, skip=skip) return search_result
def __fetch_objects(self): query = [] result = [] condition = [] subset: bool = self.__job.scheduling['event'].get('subset', False) if subset and self.event.get_param('event') in ['delete']: from cmdb.framework.cmdb_render import RenderResult deleted = RenderResult() deleted.object_information['object_id'] = self.event.get_param( 'id') deleted.type_information['type_id'] = self.event.get_param( 'type_id') result.append(deleted) else: if subset and self.event.get_param('event') in [ 'insert', 'update' ]: condition.append({'public_id': self.event.get_param('id')}) for source in self.__job.get_sources(): temp = [] for con in source["condition"]: operator = con["operator"] value = con["value"] regex = { "$ne": con["value"] } if operator == "!=" else { "$regex": value, "$options": "si" } regex = True if value in [ 'True', 'true' ] else False if value in ['False', 'false'] else regex temp.append({ 'fields': { "$elemMatch": { "name": con["name"], "value": regex } } }) temp.append({'type_id': source["type_id"]}) temp.append({'active': {'$eq': True}}) query.append({"$and": [*condition, *temp]}) if not source["condition"]: query.append({ 'type_id': source["type_id"], 'active': { '$eq': True } }) current_objects = self.__obm.get_objects_by(sort="public_id", **{'$or': query}) result = (RenderList( current_objects, None, database_manager=self.__obm.dbm, object_manager=self.__obm).render_result_list()) return result