def test_register_entries(self): ret = Entry.search_entries(self.user, [self.entity.id]) self.assertEqual(ret['ret_count'], 0) self.assertEqual(ret['ret_values'], []) register_documents(self._es, self._es._index) ret = Entry.search_entries(self.user, [self.entity.id]) self.assertEqual(ret['ret_count'], 3) self.assertTrue( all([ x['entity']['id'] == self.entity.id for x in ret['ret_values'] ])) self.assertTrue( all([ x['entry']['id'] in [y.id for y in self.entries] for x in ret['ret_values'] ]))
def test_register_entries(self): ret = Entry.search_entries(self.user, [self.entity.id]) self.assertEqual(ret["ret_count"], 0) self.assertEqual(ret["ret_values"], []) register_documents(self._es, self._es._index) ret = Entry.search_entries(self.user, [self.entity.id]) self.assertEqual(ret["ret_count"], 3) self.assertTrue( all([ x["entity"]["id"] == self.entity.id for x in ret["ret_values"] ])) self.assertTrue( all([ x["entry"]["id"] in [y.id for y in self.entries] for x in ret["ret_values"] ]))
def advanced_search_result(request): user = User.objects.get(id=request.user.id) recv_entity = request.GET.getlist('entity[]') recv_attr = request.GET.getlist('attr[]') is_all_entities = request.GET.get('is_all_entities') == 'true' has_referral = request.GET.get('has_referral') == 'true' attrinfo = request.GET.get('attrinfo') entry_name = request.GET.get('entry_name') # check entity params if not is_all_entities: if not recv_entity: return HttpResponse("The entity[] parameters are required", status=400) if not all( [Entity.objects.filter(id=x, is_active=True).exists() for x in recv_entity]): return HttpResponse("Invalid entity ID is specified", status=400) # check attribute params if not recv_attr and not attrinfo: return HttpResponse("The attr[] or attrinfo parameters is required", status=400) # build hint attrs from JSON encoded params, # or attr[] the older param to keep backward compatibility # TODO deprecate attr[] hint_attrs = [{'name': x} for x in recv_attr] if attrinfo: try: hint_attrs = json.loads(attrinfo) except json.JSONDecodeError: return HttpResponse("The attrinfo parameter is not JSON", status=400) attr_names = [x['name'] for x in hint_attrs] if is_all_entities: attrs = sum( [list(EntityAttr.objects.filter(name=x, is_active=True)) for x in attr_names], []) entities = list(set([x.parent_entity.id for x in attrs if x])) else: entities = recv_entity return render(request, 'advanced_search_result.html', { 'hint_attrs': hint_attrs, 'results': Entry.search_entries(user, entities, hint_attrs, CONFIG.MAXIMUM_SEARCH_RESULTS, entry_name, hint_referral=has_referral), 'max_num': CONFIG.MAXIMUM_SEARCH_RESULTS, 'entities': ','.join([str(x) for x in entities]), 'has_referral': has_referral, 'is_all_entities': is_all_entities, 'entry_name': entry_name, })
def test_delete_entry(self): register_documents(self._es, self._es._index) # delete entry-0 entry = self.entries[0] Entry.objects.get(id=entry.id).delete() delete_unnecessary_documents(self._es, self._es._index) ret = Entry.search_entries(self.user, [self.entity.id]) self.assertEqual(ret["ret_count"], 2) self.assertFalse( any(x["entry"]["id"] == entry.id for x in ret["ret_values"]))
def test_delete_entry(self): register_documents(self._es, self._es._index) # delete entry-0 entry = self.entries[0] Entry.objects.filter(id=entry.id).delete() delete_unnecessary_documents(self._es, self._es._index) ret = Entry.search_entries(self.user, [self.entity.id]) self.assertEqual(ret['ret_count'], 2) self.assertFalse( any(x['entry']['id'] == entry.id for x in ret['ret_values']))
def advanced_search_result(request): user = User.objects.get(id=request.user.id) recv_entity = request.GET.getlist('entity[]') recv_attr = request.GET.getlist('attr[]') is_all_entities = request.GET.get('is_all_entities') == 'true' has_referral = request.GET.get('has_referral') == 'true' if not is_all_entities and (not recv_entity or not recv_attr): return HttpResponse("The attr[] and entity[] parameters are required", status=400) elif is_all_entities and not recv_attr: return HttpResponse("The attr[] parameters are required", status=400) if not is_all_entities and not all([ Entity.objects.filter(id=x, is_active=True).exists() for x in recv_entity ]): return HttpResponse("Invalid entity ID is specified", status=400) if is_all_entities: attrs = sum([ list(EntityAttr.objects.filter(name=x, is_active=True)) for x in recv_attr ], []) entities = list(set([x.parent_entity.id for x in attrs if x])) else: entities = recv_entity return render( request, 'advanced_search_result.html', { 'attrs': recv_attr, 'results': Entry.search_entries(user, entities, [{ 'name': x } for x in recv_attr], CONFIG.MAXIMUM_SEARCH_RESULTS, hint_referral=has_referral), 'max_num': CONFIG.MAXIMUM_SEARCH_RESULTS, 'entities': ','.join([str(x) for x in entities]), 'has_referral': has_referral, 'is_all_entities': is_all_entities, })
def post(self, request, format=None): user = User.objects.get(id=request.user.id) hint_entity = request.data.get('entities') hint_entry_name = request.data.get('entry_name', '') hint_attr = request.data.get('attrinfo') hint_referral = request.data.get('referral') entry_limit = request.data.get('entry_limit', CONFIG_ENTRY.MAX_LIST_ENTRIES) if (not isinstance(hint_entity, list) or not isinstance(hint_attr, list) or not isinstance(entry_limit, int)): return Response('The type of parameter is incorrect', status=status.HTTP_400_BAD_REQUEST) # forbid to input large size request if any( [len(str(x)) > CONFIG_ENTRY.MAX_QUERY_SIZE * 2 for x in hint_attr]): return Response("Sending parameter is too large", status=400) # convert hint_referral type to be eligible for search_entries method if hint_referral is None: hint_referral = False hint_entity_ids = [] for hint in hint_entity: try: if Entity.objects.filter(id=hint).exists(): hint_entity_ids.append(hint) except ValueError: # This may happen when a string value is specified in the entities parameter entity = Entity.objects.filter(name=hint).first() if entity: hint_entity_ids.append(entity.id) resp = Entry.search_entries( user, hint_entity_ids, hint_attr, entry_limit, **{ 'hint_referral': hint_referral, 'entry_name': hint_entry_name, }) return Response({'result': resp}, content_type='application/json; charset=UTF-8')
def export_search_result(self, job_id): job = Job.objects.get(id=job_id) if not job.proceed_if_ready(): return # set flag to indicate that this job starts processing job.update(Job.STATUS["PROCESSING"]) user = job.user recv_data = json.loads(job.params) has_referral = recv_data.get("has_referral", False) referral_name = recv_data.get("referral_name") entry_name = recv_data.get("entry_name") hint_referral = "" if has_referral else False if referral_name: hint_referral = referral_name resp = Entry.search_entries( user, recv_data["entities"], recv_data["attrinfo"], settings.ES_CONFIG["MAXIMUM_RESULTS_NUM"], entry_name, hint_referral, ) io_stream = None if recv_data["export_style"] == "yaml": io_stream = _yaml_export(job, resp["ret_values"], recv_data, has_referral) elif recv_data["export_style"] == "csv": io_stream = _csv_export(job, resp["ret_values"], recv_data, has_referral) if io_stream: job.set_cache(io_stream.getvalue()) # update job status and save it except for the case that target job is canceled. if not job.is_canceled(): job.update(Job.STATUS["DONE"])
def export_search_result(self, job_id): job = Job.objects.get(id=job_id) if not job.is_ready_to_process(): return # wait dependent job is finished job.wait_dependent_job() # set flag to indicate that this job starts processing job.set_status(Job.STATUS['PROCESSING']) user = job.user recv_data = json.loads(job.params) has_referral = False if 'has_referral' in recv_data: has_referral = recv_data['has_referral'] hint_entry_name = '' if 'entry_name' in recv_data and recv_data['entry_name']: hint_entry_name = recv_data['entry_name'] resp = Entry.search_entries(user, recv_data['entities'], recv_data['attrinfo'], settings.ES_CONFIG['MAXIMUM_RESULTS_NUM'], hint_referral=has_referral, entry_name=hint_entry_name) io_stream = None if recv_data['export_style'] == 'yaml': io_stream = _yaml_export(job, resp['ret_values'], recv_data, has_referral) elif recv_data['export_style'] == 'csv': io_stream = _csv_export(job, resp['ret_values'], recv_data, has_referral) if io_stream: job.set_cache(io_stream.getvalue()) # update job status and save it except for the case that target job is canceled. if not job.is_canceled(): job.set_status(Job.STATUS['DONE'])
def test_update_entry(self): register_documents(self._es, self._es._index) # update entry-0 entry = self.entries[0] entry.attrs.first().add_value(self.user, 'new-attr-value') entry.name = 'new-entry-name' entry.save() register_documents(self._es, self._es._index) ret = Entry.search_entries(self.user, [self.entity.id]) self.assertEqual(ret['ret_count'], 3) entry_info = [ x for x in ret['ret_values'] if x['entry']['id'] == entry.id ][0] self.assertEqual(entry_info['entry']['name'], 'new-entry-name') self.assertEqual(entry_info['attrs']['attr']['value'], 'new-attr-value')
def export_search_result(self, job_id): job = Job.objects.get(id=job_id) if not job.proceed_if_ready(): return # set flag to indicate that this job starts processing job.update(Job.STATUS['PROCESSING']) user = job.user recv_data = json.loads(job.params) has_referral = recv_data.get('has_referral', False) referral_name = recv_data.get('referral_name') entry_name = recv_data.get('entry_name') hint_referral = '' if has_referral else False if referral_name: hint_referral = referral_name resp = Entry.search_entries(user, recv_data['entities'], recv_data['attrinfo'], settings.ES_CONFIG['MAXIMUM_RESULTS_NUM'], entry_name, hint_referral) io_stream = None if recv_data['export_style'] == 'yaml': io_stream = _yaml_export(job, resp['ret_values'], recv_data, has_referral) elif recv_data['export_style'] == 'csv': io_stream = _csv_export(job, resp['ret_values'], recv_data, has_referral) if io_stream: job.set_cache(io_stream.getvalue()) # update job status and save it except for the case that target job is canceled. if not job.is_canceled(): job.update(Job.STATUS['DONE'])
def test_update_entry(self): register_documents(self._es, self._es._index) # update entry-0 entry = self.entries[0] entry.attrs.first().add_value(self.user, "new-attr-value") entry.name = "new-entry-name" entry.save() register_documents(self._es, self._es._index) ret = Entry.search_entries(self.user, [self.entity.id], [{ "name": "attr" }]) self.assertEqual(ret["ret_count"], 3) entry_info = [ x for x in ret["ret_values"] if x["entry"]["id"] == entry.id ][0] self.assertEqual(entry_info["entry"]["name"], "new-entry-name") self.assertEqual(entry_info["attrs"]["attr"]["value"], "new-attr-value")
def post(self, request, format=None): hint_entities = request.data.get("entities") hint_entry_name = request.data.get("entry_name", "") hint_attrs = request.data.get("attrinfo") hint_has_referral = request.data.get("has_referral", False) hint_referral_name = request.data.get("referral_name", "") is_output_all = request.data.get("is_output_all", True) entry_limit = request.data.get("entry_limit", self.MAX_LIST_ENTRIES) hint_referral = False if hint_has_referral: hint_referral = hint_referral_name if (not isinstance(hint_entities, list) or not isinstance(hint_entry_name, str) or not isinstance(hint_attrs, list) or not isinstance(is_output_all, bool) or not isinstance(hint_referral, (str, bool)) or not isinstance(entry_limit, int)): return Response("The type of parameter is incorrect", status=status.HTTP_400_BAD_REQUEST) # forbid to input large size request if len(hint_entry_name) > self.MAX_QUERY_SIZE: return Response("Sending parameter is too large", status=400) # check attribute params for hint_attr in hint_attrs: if "name" not in hint_attr: return Response( "The name key is required for attrinfo parameter", status=400) if not isinstance(hint_attr["name"], str): return Response("Invalid value for attrinfo parameter", status=400) if hint_attr.get("keyword"): if not isinstance(hint_attr["keyword"], str): return Response("Invalid value for attrinfo parameter", status=400) # forbid to input large size request if len(hint_attr["keyword"]) > self.MAX_QUERY_SIZE: return Response("Sending parameter is too large", status=400) # check entities params if not hint_entities: return Response("The entities parameters are required", status=400) hint_entity_ids = [] for hint_entity in hint_entities: entity = None if isinstance(hint_entity, int): entity = Entity.objects.filter(id=hint_entity, is_active=True).first() elif isinstance(hint_entity, str): if hint_entity.isnumeric(): entity = Entity.objects.filter( Q(id=hint_entity) | Q(name=hint_entity), Q(is_active=True)).first() else: entity = Entity.objects.filter(name=hint_entity, is_active=True).first() if entity and request.user.has_permission(entity, ACLType.Readable): hint_entity_ids.append(entity.id) resp = Entry.search_entries( request.user, hint_entity_ids, hint_attrs, entry_limit, hint_entry_name, hint_referral, is_output_all, ) # convert field values to fit entry retrieve API data type, as a workaround. # FIXME should be replaced with DRF serializer etc for entry in resp["ret_values"]: for name, attr in entry["attrs"].items(): def _get_typed_value(type: int) -> str: if type & AttrTypeValue["array"]: if type & AttrTypeValue["string"]: return "asArrayString" elif type & AttrTypeValue["named"]: return "asArrayNamedObject" elif type & AttrTypeValue["object"]: return "asArrayObject" elif type & AttrTypeValue["group"]: return "asArrayGroup" elif type & AttrTypeValue[ "string"] or type & AttrTypeValue["text"]: return "asString" elif type & AttrTypeValue["named"]: return "asNamedObject" elif type & AttrTypeValue["object"]: return "asObject" elif type & AttrTypeValue["boolean"]: return "asBoolean" elif type & AttrTypeValue["date"]: return "asString" elif type & AttrTypeValue["group"]: return "asGroup" raise ValidationError(f"unexpected type: {type}") entry["attrs"][name] = { "is_readble": attr["is_readble"], "type": attr["type"], "value": { _get_typed_value(attr["type"]): attr.get("value", ""), }, } return Response({"result": resp})
def post(self, request, format=None): hint_entities = request.data.get("entities") hint_entry_name = request.data.get("entry_name", "") hint_attrs = request.data.get("attrinfo") hint_referral = request.data.get("referral", False) is_output_all = request.data.get("is_output_all", True) entry_limit = request.data.get("entry_limit", CONFIG_ENTRY.MAX_LIST_ENTRIES) if ( not isinstance(hint_entities, list) or not isinstance(hint_entry_name, str) or not isinstance(hint_attrs, list) or not isinstance(is_output_all, bool) or not isinstance(hint_referral, (str, bool)) or not isinstance(entry_limit, int) ): return Response( "The type of parameter is incorrect", status=status.HTTP_400_BAD_REQUEST ) # forbid to input large size request if len(hint_entry_name) > CONFIG_ENTRY.MAX_QUERY_SIZE: return Response("Sending parameter is too large", status=400) # check attribute params for hint_attr in hint_attrs: if "name" not in hint_attr: return Response("The name key is required for attrinfo parameter", status=400) if not isinstance(hint_attr["name"], str): return Response("Invalid value for attrinfo parameter", status=400) if hint_attr.get("keyword"): if not isinstance(hint_attr["keyword"], str): return Response("Invalid value for attrinfo parameter", status=400) # forbid to input large size request if len(hint_attr["keyword"]) > CONFIG_ENTRY.MAX_QUERY_SIZE: return Response("Sending parameter is too large", status=400) # check entities params if not hint_entities: return Response("The entities parameters are required", status=400) hint_entity_ids = [] for hint_entity in hint_entities: entity = None if isinstance(hint_entity, int): entity = Entity.objects.filter(id=hint_entity, is_active=True).first() elif isinstance(hint_entity, str): if hint_entity.isnumeric(): entity = Entity.objects.filter( Q(id=hint_entity) | Q(name=hint_entity), Q(is_active=True) ).first() else: entity = Entity.objects.filter(name=hint_entity, is_active=True).first() if entity and request.user.has_permission(entity, ACLType.Readable): hint_entity_ids.append(entity.id) resp = Entry.search_entries( request.user, hint_entity_ids, hint_attrs, entry_limit, hint_entry_name, hint_referral, is_output_all, ) return Response({"result": resp})
def advanced_search_result(request): user = User.objects.get(id=request.user.id) recv_entity = request.GET.getlist('entity[]') recv_attr = request.GET.getlist('attr[]') is_all_entities = request.GET.get('is_all_entities') == 'true' has_referral = request.GET.get('has_referral') == 'true' referral_name = request.GET.get('referral_name') attrinfo = request.GET.get('attrinfo') entry_name = request.GET.get('entry_name', '') # forbid to input large size request if len(entry_name) > CONFIG_ENTRY.MAX_QUERY_SIZE: return HttpResponse("Sending parameter is too large", status=400) # check referral params # # process of converting older param for backward compatibility hint_referral = '' if has_referral else False if referral_name: hint_referral = referral_name # check attribute params # The "attr" parameter guarantees backward compatibility. # The "atterinfo" is another parameter, # that has same purpose that indicates which attributes to search, # And "attrinfo" is prioritize than "attr". # TODO deprecate attr[] hint_attrs = [{'name': x} for x in recv_attr] if attrinfo: try: # build hint attrs from JSON encoded params hint_attrs = json.loads(attrinfo) except json.JSONDecodeError: return HttpResponse("The attrinfo parameter is not JSON", status=400) for hint_attr in hint_attrs: if 'name' not in hint_attr: return HttpResponse( "The name key is required for attrinfo parameter", status=400) if not isinstance(hint_attr['name'], str): return HttpResponse("Invalid value for attrinfo parameter", status=400) if 'keyword' in hint_attr: if not isinstance(hint_attr['keyword'], str): return HttpResponse("Invalid value for attrinfo parameter", status=400) if len(hint_attr['keyword']) > CONFIG_ENTRY.MAX_QUERY_SIZE: return HttpResponse("Sending parameter is too large", status=400) # check entity params if is_all_entities: attr_names = [x['name'] for x in hint_attrs] recv_entity = list( EntityAttr.objects.filter( name__in=attr_names, is_active=True, parent_entity__is_active=True).order_by( 'parent_entity__name').values_list('parent_entity__id', flat=True).distinct()) if not recv_entity: return HttpResponse("Invalid value for attribute parameter", status=400) if not recv_entity: return HttpResponse("The entity[] parameters are required", status=400) hint_entity_ids = [] for entity_id in recv_entity: if not isinstance(entity_id, int) and not entity_id.isnumeric(): return HttpResponse("Invalid entity ID is specified", status=400) entity = Entity.objects.filter(id=entity_id, is_active=True).first() if not entity: return HttpResponse("Invalid entity ID is specified", status=400) if user.has_permission(entity, ACLType.Readable): hint_entity_ids.append(entity.id) return render( request, 'advanced_search_result.html', { 'hint_attrs': hint_attrs, 'results': Entry.search_entries(user, hint_entity_ids, hint_attrs, CONFIG.MAXIMUM_SEARCH_RESULTS, entry_name, hint_referral), 'max_num': CONFIG.MAXIMUM_SEARCH_RESULTS, 'entities': ','.join([str(x) for x in hint_entity_ids]), 'has_referral': has_referral, 'referral_name': referral_name, 'is_all_entities': is_all_entities, 'entry_name': entry_name, })
def advanced_search_result(request): recv_entity = request.GET.getlist("entity[]") recv_attr = request.GET.getlist("attr[]") is_all_entities = request.GET.get("is_all_entities") == "true" has_referral = request.GET.get("has_referral") == "true" referral_name = request.GET.get("referral_name") attrinfo = request.GET.get("attrinfo") entry_name = request.GET.get("entry_name", "") # forbid to input large size request if len(entry_name) > CONFIG_ENTRY.MAX_QUERY_SIZE: return HttpResponse("Sending parameter is too large", status=400) # check referral params # # process of converting older param for backward compatibility hint_referral = "" if has_referral else False if referral_name: hint_referral = referral_name # check attribute params # The "attr" parameter guarantees backward compatibility. # The "atterinfo" is another parameter, # that has same purpose that indicates which attributes to search, # And "attrinfo" is prioritize than "attr". # TODO deprecate attr[] hint_attrs = [{"name": x} for x in recv_attr] if attrinfo: try: # build hint attrs from JSON encoded params hint_attrs = json.loads(attrinfo) except json.JSONDecodeError: return HttpResponse("The attrinfo parameter is not JSON", status=400) for hint_attr in hint_attrs: if "name" not in hint_attr: return HttpResponse( "The name key is required for attrinfo parameter", status=400) if not isinstance(hint_attr["name"], str): return HttpResponse("Invalid value for attrinfo parameter", status=400) if "keyword" in hint_attr: if not isinstance(hint_attr["keyword"], str): return HttpResponse("Invalid value for attrinfo parameter", status=400) if len(hint_attr["keyword"]) > CONFIG_ENTRY.MAX_QUERY_SIZE: return HttpResponse("Sending parameter is too large", status=400) # check entity params if is_all_entities: attr_names = [x["name"] for x in hint_attrs] recv_entity = list( EntityAttr.objects.filter( name__in=attr_names, is_active=True, parent_entity__is_active=True).order_by( "parent_entity__name").values_list("parent_entity__id", flat=True).distinct()) if not recv_entity: return HttpResponse("Invalid value for attribute parameter", status=400) if not recv_entity: return HttpResponse("The entity[] parameters are required", status=400) hint_entity_ids = [] for entity_id in recv_entity: if not isinstance(entity_id, int) and not entity_id.isnumeric(): return HttpResponse("Invalid entity ID is specified", status=400) entity = Entity.objects.filter(id=entity_id, is_active=True).first() if not entity: return HttpResponse("Invalid entity ID is specified", status=400) if request.user.has_permission(entity, ACLType.Readable): hint_entity_ids.append(entity.id) return render( request, "advanced_search_result.html", { "hint_attrs": hint_attrs, "results": Entry.search_entries( request.user, hint_entity_ids, hint_attrs, CONFIG.MAXIMUM_SEARCH_RESULTS, entry_name, hint_referral, ), "max_num": CONFIG.MAXIMUM_SEARCH_RESULTS, "entities": ",".join([str(x) for x in hint_entity_ids]), "has_referral": has_referral, "referral_name": referral_name, "is_all_entities": is_all_entities, "entry_name": entry_name, }, )