def test_register_entries(self):
ret = Entry.search_entries(self.user, [self.entity.id])
self.assertEqual(ret['ret_count'], 0)
self.assertEqual(ret['ret_values'], [])
register_documents(self._es, self._es._index)
ret = Entry.search_entries(self.user, [self.entity.id])
self.assertEqual(ret['ret_count'], 3)
self.assertTrue(
all([
x['entity']['id'] == self.entity.id for x in ret['ret_values']
]))
self.assertTrue(
all([
x['entry']['id'] in [y.id for y in self.entries]
for x in ret['ret_values']
]))
def test_register_entries(self):
ret = Entry.search_entries(self.user, [self.entity.id])
self.assertEqual(ret["ret_count"], 0)
self.assertEqual(ret["ret_values"], [])
register_documents(self._es, self._es._index)
ret = Entry.search_entries(self.user, [self.entity.id])
self.assertEqual(ret["ret_count"], 3)
self.assertTrue(
all([
x["entity"]["id"] == self.entity.id for x in ret["ret_values"]
]))
self.assertTrue(
all([
x["entry"]["id"] in [y.id for y in self.entries]
for x in ret["ret_values"]
]))
def advanced_search_result(request):
user = User.objects.get(id=request.user.id)
recv_entity = request.GET.getlist('entity[]')
recv_attr = request.GET.getlist('attr[]')
is_all_entities = request.GET.get('is_all_entities') == 'true'
has_referral = request.GET.get('has_referral') == 'true'
attrinfo = request.GET.get('attrinfo')
entry_name = request.GET.get('entry_name')
# check entity params
if not is_all_entities:
if not recv_entity:
return HttpResponse("The entity[] parameters are required", status=400)
if not all(
[Entity.objects.filter(id=x, is_active=True).exists() for x in recv_entity]):
return HttpResponse("Invalid entity ID is specified", status=400)
# check attribute params
if not recv_attr and not attrinfo:
return HttpResponse("The attr[] or attrinfo parameters is required", status=400)
# build hint attrs from JSON encoded params,
# or attr[] the older param to keep backward compatibility
# TODO deprecate attr[]
hint_attrs = [{'name': x} for x in recv_attr]
if attrinfo:
try:
hint_attrs = json.loads(attrinfo)
except json.JSONDecodeError:
return HttpResponse("The attrinfo parameter is not JSON", status=400)
attr_names = [x['name'] for x in hint_attrs]
if is_all_entities:
attrs = sum(
[list(EntityAttr.objects.filter(name=x, is_active=True)) for x in attr_names], [])
entities = list(set([x.parent_entity.id for x in attrs if x]))
else:
entities = recv_entity
return render(request, 'advanced_search_result.html', {
'hint_attrs': hint_attrs,
'results': Entry.search_entries(user,
entities,
hint_attrs,
CONFIG.MAXIMUM_SEARCH_RESULTS,
entry_name,
hint_referral=has_referral),
'max_num': CONFIG.MAXIMUM_SEARCH_RESULTS,
'entities': ','.join([str(x) for x in entities]),
'has_referral': has_referral,
'is_all_entities': is_all_entities,
'entry_name': entry_name,
})
def test_delete_entry(self):
register_documents(self._es, self._es._index)
# delete entry-0
entry = self.entries[0]
Entry.objects.get(id=entry.id).delete()
delete_unnecessary_documents(self._es, self._es._index)
ret = Entry.search_entries(self.user, [self.entity.id])
self.assertEqual(ret["ret_count"], 2)
self.assertFalse(
any(x["entry"]["id"] == entry.id for x in ret["ret_values"]))
def test_delete_entry(self):
register_documents(self._es, self._es._index)
# delete entry-0
entry = self.entries[0]
Entry.objects.filter(id=entry.id).delete()
delete_unnecessary_documents(self._es, self._es._index)
ret = Entry.search_entries(self.user, [self.entity.id])
self.assertEqual(ret['ret_count'], 2)
self.assertFalse(
any(x['entry']['id'] == entry.id for x in ret['ret_values']))
def advanced_search_result(request):
user = User.objects.get(id=request.user.id)
recv_entity = request.GET.getlist('entity[]')
recv_attr = request.GET.getlist('attr[]')
is_all_entities = request.GET.get('is_all_entities') == 'true'
has_referral = request.GET.get('has_referral') == 'true'
if not is_all_entities and (not recv_entity or not recv_attr):
return HttpResponse("The attr[] and entity[] parameters are required",
status=400)
elif is_all_entities and not recv_attr:
return HttpResponse("The attr[] parameters are required", status=400)
if not is_all_entities and not all([
Entity.objects.filter(id=x, is_active=True).exists()
for x in recv_entity
]):
return HttpResponse("Invalid entity ID is specified", status=400)
if is_all_entities:
attrs = sum([
list(EntityAttr.objects.filter(name=x, is_active=True))
for x in recv_attr
], [])
entities = list(set([x.parent_entity.id for x in attrs if x]))
else:
entities = recv_entity
return render(
request, 'advanced_search_result.html', {
'attrs':
recv_attr,
'results':
Entry.search_entries(user,
entities, [{
'name': x
} for x in recv_attr],
CONFIG.MAXIMUM_SEARCH_RESULTS,
hint_referral=has_referral),
'max_num':
CONFIG.MAXIMUM_SEARCH_RESULTS,
'entities':
','.join([str(x) for x in entities]),
'has_referral':
has_referral,
'is_all_entities':
is_all_entities,
})
def post(self, request, format=None):
user = User.objects.get(id=request.user.id)
hint_entity = request.data.get('entities')
hint_entry_name = request.data.get('entry_name', '')
hint_attr = request.data.get('attrinfo')
hint_referral = request.data.get('referral')
entry_limit = request.data.get('entry_limit',
CONFIG_ENTRY.MAX_LIST_ENTRIES)
if (not isinstance(hint_entity, list)
or not isinstance(hint_attr, list)
or not isinstance(entry_limit, int)):
return Response('The type of parameter is incorrect',
status=status.HTTP_400_BAD_REQUEST)
# forbid to input large size request
if any(
[len(str(x)) > CONFIG_ENTRY.MAX_QUERY_SIZE * 2
for x in hint_attr]):
return Response("Sending parameter is too large", status=400)
# convert hint_referral type to be eligible for search_entries method
if hint_referral is None:
hint_referral = False
hint_entity_ids = []
for hint in hint_entity:
try:
if Entity.objects.filter(id=hint).exists():
hint_entity_ids.append(hint)
except ValueError:
# This may happen when a string value is specified in the entities parameter
entity = Entity.objects.filter(name=hint).first()
if entity:
hint_entity_ids.append(entity.id)
resp = Entry.search_entries(
user, hint_entity_ids, hint_attr, entry_limit, **{
'hint_referral': hint_referral,
'entry_name': hint_entry_name,
})
return Response({'result': resp},
content_type='application/json; charset=UTF-8')
def export_search_result(self, job_id):
job = Job.objects.get(id=job_id)
if not job.proceed_if_ready():
return
# set flag to indicate that this job starts processing
job.update(Job.STATUS["PROCESSING"])
user = job.user
recv_data = json.loads(job.params)
has_referral = recv_data.get("has_referral", False)
referral_name = recv_data.get("referral_name")
entry_name = recv_data.get("entry_name")
hint_referral = "" if has_referral else False
if referral_name:
hint_referral = referral_name
resp = Entry.search_entries(
user,
recv_data["entities"],
recv_data["attrinfo"],
settings.ES_CONFIG["MAXIMUM_RESULTS_NUM"],
entry_name,
hint_referral,
)
io_stream = None
if recv_data["export_style"] == "yaml":
io_stream = _yaml_export(job, resp["ret_values"], recv_data,
has_referral)
elif recv_data["export_style"] == "csv":
io_stream = _csv_export(job, resp["ret_values"], recv_data,
has_referral)
if io_stream:
job.set_cache(io_stream.getvalue())
# update job status and save it except for the case that target job is canceled.
if not job.is_canceled():
job.update(Job.STATUS["DONE"])
def export_search_result(self, job_id):
job = Job.objects.get(id=job_id)
if not job.is_ready_to_process():
return
# wait dependent job is finished
job.wait_dependent_job()
# set flag to indicate that this job starts processing
job.set_status(Job.STATUS['PROCESSING'])
user = job.user
recv_data = json.loads(job.params)
has_referral = False
if 'has_referral' in recv_data:
has_referral = recv_data['has_referral']
hint_entry_name = ''
if 'entry_name' in recv_data and recv_data['entry_name']:
hint_entry_name = recv_data['entry_name']
resp = Entry.search_entries(user,
recv_data['entities'],
recv_data['attrinfo'],
settings.ES_CONFIG['MAXIMUM_RESULTS_NUM'],
hint_referral=has_referral,
entry_name=hint_entry_name)
io_stream = None
if recv_data['export_style'] == 'yaml':
io_stream = _yaml_export(job, resp['ret_values'], recv_data, has_referral)
elif recv_data['export_style'] == 'csv':
io_stream = _csv_export(job, resp['ret_values'], recv_data, has_referral)
if io_stream:
job.set_cache(io_stream.getvalue())
# update job status and save it except for the case that target job is canceled.
if not job.is_canceled():
job.set_status(Job.STATUS['DONE'])
def test_update_entry(self):
register_documents(self._es, self._es._index)
# update entry-0
entry = self.entries[0]
entry.attrs.first().add_value(self.user, 'new-attr-value')
entry.name = 'new-entry-name'
entry.save()
register_documents(self._es, self._es._index)
ret = Entry.search_entries(self.user, [self.entity.id])
self.assertEqual(ret['ret_count'], 3)
entry_info = [
x for x in ret['ret_values'] if x['entry']['id'] == entry.id
][0]
self.assertEqual(entry_info['entry']['name'], 'new-entry-name')
self.assertEqual(entry_info['attrs']['attr']['value'],
'new-attr-value')
def export_search_result(self, job_id):
job = Job.objects.get(id=job_id)
if not job.proceed_if_ready():
return
# set flag to indicate that this job starts processing
job.update(Job.STATUS['PROCESSING'])
user = job.user
recv_data = json.loads(job.params)
has_referral = recv_data.get('has_referral', False)
referral_name = recv_data.get('referral_name')
entry_name = recv_data.get('entry_name')
hint_referral = '' if has_referral else False
if referral_name:
hint_referral = referral_name
resp = Entry.search_entries(user,
recv_data['entities'],
recv_data['attrinfo'],
settings.ES_CONFIG['MAXIMUM_RESULTS_NUM'],
entry_name,
hint_referral)
io_stream = None
if recv_data['export_style'] == 'yaml':
io_stream = _yaml_export(job, resp['ret_values'], recv_data, has_referral)
elif recv_data['export_style'] == 'csv':
io_stream = _csv_export(job, resp['ret_values'], recv_data, has_referral)
if io_stream:
job.set_cache(io_stream.getvalue())
# update job status and save it except for the case that target job is canceled.
if not job.is_canceled():
job.update(Job.STATUS['DONE'])
def test_update_entry(self):
register_documents(self._es, self._es._index)
# update entry-0
entry = self.entries[0]
entry.attrs.first().add_value(self.user, "new-attr-value")
entry.name = "new-entry-name"
entry.save()
register_documents(self._es, self._es._index)
ret = Entry.search_entries(self.user, [self.entity.id], [{
"name": "attr"
}])
self.assertEqual(ret["ret_count"], 3)
entry_info = [
x for x in ret["ret_values"] if x["entry"]["id"] == entry.id
][0]
self.assertEqual(entry_info["entry"]["name"], "new-entry-name")
self.assertEqual(entry_info["attrs"]["attr"]["value"],
"new-attr-value")
def post(self, request, format=None):
hint_entities = request.data.get("entities")
hint_entry_name = request.data.get("entry_name", "")
hint_attrs = request.data.get("attrinfo")
hint_has_referral = request.data.get("has_referral", False)
hint_referral_name = request.data.get("referral_name", "")
is_output_all = request.data.get("is_output_all", True)
entry_limit = request.data.get("entry_limit", self.MAX_LIST_ENTRIES)
hint_referral = False
if hint_has_referral:
hint_referral = hint_referral_name
if (not isinstance(hint_entities, list)
or not isinstance(hint_entry_name, str)
or not isinstance(hint_attrs, list)
or not isinstance(is_output_all, bool)
or not isinstance(hint_referral, (str, bool))
or not isinstance(entry_limit, int)):
return Response("The type of parameter is incorrect",
status=status.HTTP_400_BAD_REQUEST)
# forbid to input large size request
if len(hint_entry_name) > self.MAX_QUERY_SIZE:
return Response("Sending parameter is too large", status=400)
# check attribute params
for hint_attr in hint_attrs:
if "name" not in hint_attr:
return Response(
"The name key is required for attrinfo parameter",
status=400)
if not isinstance(hint_attr["name"], str):
return Response("Invalid value for attrinfo parameter",
status=400)
if hint_attr.get("keyword"):
if not isinstance(hint_attr["keyword"], str):
return Response("Invalid value for attrinfo parameter",
status=400)
# forbid to input large size request
if len(hint_attr["keyword"]) > self.MAX_QUERY_SIZE:
return Response("Sending parameter is too large",
status=400)
# check entities params
if not hint_entities:
return Response("The entities parameters are required", status=400)
hint_entity_ids = []
for hint_entity in hint_entities:
entity = None
if isinstance(hint_entity, int):
entity = Entity.objects.filter(id=hint_entity,
is_active=True).first()
elif isinstance(hint_entity, str):
if hint_entity.isnumeric():
entity = Entity.objects.filter(
Q(id=hint_entity) | Q(name=hint_entity),
Q(is_active=True)).first()
else:
entity = Entity.objects.filter(name=hint_entity,
is_active=True).first()
if entity and request.user.has_permission(entity,
ACLType.Readable):
hint_entity_ids.append(entity.id)
resp = Entry.search_entries(
request.user,
hint_entity_ids,
hint_attrs,
entry_limit,
hint_entry_name,
hint_referral,
is_output_all,
)
# convert field values to fit entry retrieve API data type, as a workaround.
# FIXME should be replaced with DRF serializer etc
for entry in resp["ret_values"]:
for name, attr in entry["attrs"].items():
def _get_typed_value(type: int) -> str:
if type & AttrTypeValue["array"]:
if type & AttrTypeValue["string"]:
return "asArrayString"
elif type & AttrTypeValue["named"]:
return "asArrayNamedObject"
elif type & AttrTypeValue["object"]:
return "asArrayObject"
elif type & AttrTypeValue["group"]:
return "asArrayGroup"
elif type & AttrTypeValue[
"string"] or type & AttrTypeValue["text"]:
return "asString"
elif type & AttrTypeValue["named"]:
return "asNamedObject"
elif type & AttrTypeValue["object"]:
return "asObject"
elif type & AttrTypeValue["boolean"]:
return "asBoolean"
elif type & AttrTypeValue["date"]:
return "asString"
elif type & AttrTypeValue["group"]:
return "asGroup"
raise ValidationError(f"unexpected type: {type}")
entry["attrs"][name] = {
"is_readble": attr["is_readble"],
"type": attr["type"],
"value": {
_get_typed_value(attr["type"]): attr.get("value", ""),
},
}
return Response({"result": resp})
def post(self, request, format=None):
hint_entities = request.data.get("entities")
hint_entry_name = request.data.get("entry_name", "")
hint_attrs = request.data.get("attrinfo")
hint_referral = request.data.get("referral", False)
is_output_all = request.data.get("is_output_all", True)
entry_limit = request.data.get("entry_limit", CONFIG_ENTRY.MAX_LIST_ENTRIES)
if (
not isinstance(hint_entities, list)
or not isinstance(hint_entry_name, str)
or not isinstance(hint_attrs, list)
or not isinstance(is_output_all, bool)
or not isinstance(hint_referral, (str, bool))
or not isinstance(entry_limit, int)
):
return Response(
"The type of parameter is incorrect", status=status.HTTP_400_BAD_REQUEST
)
# forbid to input large size request
if len(hint_entry_name) > CONFIG_ENTRY.MAX_QUERY_SIZE:
return Response("Sending parameter is too large", status=400)
# check attribute params
for hint_attr in hint_attrs:
if "name" not in hint_attr:
return Response("The name key is required for attrinfo parameter", status=400)
if not isinstance(hint_attr["name"], str):
return Response("Invalid value for attrinfo parameter", status=400)
if hint_attr.get("keyword"):
if not isinstance(hint_attr["keyword"], str):
return Response("Invalid value for attrinfo parameter", status=400)
# forbid to input large size request
if len(hint_attr["keyword"]) > CONFIG_ENTRY.MAX_QUERY_SIZE:
return Response("Sending parameter is too large", status=400)
# check entities params
if not hint_entities:
return Response("The entities parameters are required", status=400)
hint_entity_ids = []
for hint_entity in hint_entities:
entity = None
if isinstance(hint_entity, int):
entity = Entity.objects.filter(id=hint_entity, is_active=True).first()
elif isinstance(hint_entity, str):
if hint_entity.isnumeric():
entity = Entity.objects.filter(
Q(id=hint_entity) | Q(name=hint_entity), Q(is_active=True)
).first()
else:
entity = Entity.objects.filter(name=hint_entity, is_active=True).first()
if entity and request.user.has_permission(entity, ACLType.Readable):
hint_entity_ids.append(entity.id)
resp = Entry.search_entries(
request.user,
hint_entity_ids,
hint_attrs,
entry_limit,
hint_entry_name,
hint_referral,
is_output_all,
)
return Response({"result": resp})
def advanced_search_result(request):
user = User.objects.get(id=request.user.id)
recv_entity = request.GET.getlist('entity[]')
recv_attr = request.GET.getlist('attr[]')
is_all_entities = request.GET.get('is_all_entities') == 'true'
has_referral = request.GET.get('has_referral') == 'true'
referral_name = request.GET.get('referral_name')
attrinfo = request.GET.get('attrinfo')
entry_name = request.GET.get('entry_name', '')
# forbid to input large size request
if len(entry_name) > CONFIG_ENTRY.MAX_QUERY_SIZE:
return HttpResponse("Sending parameter is too large", status=400)
# check referral params
# # process of converting older param for backward compatibility
hint_referral = '' if has_referral else False
if referral_name:
hint_referral = referral_name
# check attribute params
# The "attr" parameter guarantees backward compatibility.
# The "atterinfo" is another parameter,
# that has same purpose that indicates which attributes to search,
# And "attrinfo" is prioritize than "attr".
# TODO deprecate attr[]
hint_attrs = [{'name': x} for x in recv_attr]
if attrinfo:
try:
# build hint attrs from JSON encoded params
hint_attrs = json.loads(attrinfo)
except json.JSONDecodeError:
return HttpResponse("The attrinfo parameter is not JSON",
status=400)
for hint_attr in hint_attrs:
if 'name' not in hint_attr:
return HttpResponse(
"The name key is required for attrinfo parameter",
status=400)
if not isinstance(hint_attr['name'], str):
return HttpResponse("Invalid value for attrinfo parameter",
status=400)
if 'keyword' in hint_attr:
if not isinstance(hint_attr['keyword'], str):
return HttpResponse("Invalid value for attrinfo parameter",
status=400)
if len(hint_attr['keyword']) > CONFIG_ENTRY.MAX_QUERY_SIZE:
return HttpResponse("Sending parameter is too large",
status=400)
# check entity params
if is_all_entities:
attr_names = [x['name'] for x in hint_attrs]
recv_entity = list(
EntityAttr.objects.filter(
name__in=attr_names,
is_active=True,
parent_entity__is_active=True).order_by(
'parent_entity__name').values_list('parent_entity__id',
flat=True).distinct())
if not recv_entity:
return HttpResponse("Invalid value for attribute parameter",
status=400)
if not recv_entity:
return HttpResponse("The entity[] parameters are required", status=400)
hint_entity_ids = []
for entity_id in recv_entity:
if not isinstance(entity_id, int) and not entity_id.isnumeric():
return HttpResponse("Invalid entity ID is specified", status=400)
entity = Entity.objects.filter(id=entity_id, is_active=True).first()
if not entity:
return HttpResponse("Invalid entity ID is specified", status=400)
if user.has_permission(entity, ACLType.Readable):
hint_entity_ids.append(entity.id)
return render(
request, 'advanced_search_result.html', {
'hint_attrs':
hint_attrs,
'results':
Entry.search_entries(user, hint_entity_ids, hint_attrs,
CONFIG.MAXIMUM_SEARCH_RESULTS, entry_name,
hint_referral),
'max_num':
CONFIG.MAXIMUM_SEARCH_RESULTS,
'entities':
','.join([str(x) for x in hint_entity_ids]),
'has_referral':
has_referral,
'referral_name':
referral_name,
'is_all_entities':
is_all_entities,
'entry_name':
entry_name,
})
def advanced_search_result(request):
recv_entity = request.GET.getlist("entity[]")
recv_attr = request.GET.getlist("attr[]")
is_all_entities = request.GET.get("is_all_entities") == "true"
has_referral = request.GET.get("has_referral") == "true"
referral_name = request.GET.get("referral_name")
attrinfo = request.GET.get("attrinfo")
entry_name = request.GET.get("entry_name", "")
# forbid to input large size request
if len(entry_name) > CONFIG_ENTRY.MAX_QUERY_SIZE:
return HttpResponse("Sending parameter is too large", status=400)
# check referral params
# # process of converting older param for backward compatibility
hint_referral = "" if has_referral else False
if referral_name:
hint_referral = referral_name
# check attribute params
# The "attr" parameter guarantees backward compatibility.
# The "atterinfo" is another parameter,
# that has same purpose that indicates which attributes to search,
# And "attrinfo" is prioritize than "attr".
# TODO deprecate attr[]
hint_attrs = [{"name": x} for x in recv_attr]
if attrinfo:
try:
# build hint attrs from JSON encoded params
hint_attrs = json.loads(attrinfo)
except json.JSONDecodeError:
return HttpResponse("The attrinfo parameter is not JSON",
status=400)
for hint_attr in hint_attrs:
if "name" not in hint_attr:
return HttpResponse(
"The name key is required for attrinfo parameter",
status=400)
if not isinstance(hint_attr["name"], str):
return HttpResponse("Invalid value for attrinfo parameter",
status=400)
if "keyword" in hint_attr:
if not isinstance(hint_attr["keyword"], str):
return HttpResponse("Invalid value for attrinfo parameter",
status=400)
if len(hint_attr["keyword"]) > CONFIG_ENTRY.MAX_QUERY_SIZE:
return HttpResponse("Sending parameter is too large",
status=400)
# check entity params
if is_all_entities:
attr_names = [x["name"] for x in hint_attrs]
recv_entity = list(
EntityAttr.objects.filter(
name__in=attr_names,
is_active=True,
parent_entity__is_active=True).order_by(
"parent_entity__name").values_list("parent_entity__id",
flat=True).distinct())
if not recv_entity:
return HttpResponse("Invalid value for attribute parameter",
status=400)
if not recv_entity:
return HttpResponse("The entity[] parameters are required", status=400)
hint_entity_ids = []
for entity_id in recv_entity:
if not isinstance(entity_id, int) and not entity_id.isnumeric():
return HttpResponse("Invalid entity ID is specified", status=400)
entity = Entity.objects.filter(id=entity_id, is_active=True).first()
if not entity:
return HttpResponse("Invalid entity ID is specified", status=400)
if request.user.has_permission(entity, ACLType.Readable):
hint_entity_ids.append(entity.id)
return render(
request,
"advanced_search_result.html",
{
"hint_attrs":
hint_attrs,
"results":
Entry.search_entries(
request.user,
hint_entity_ids,
hint_attrs,
CONFIG.MAXIMUM_SEARCH_RESULTS,
entry_name,
hint_referral,
),
"max_num":
CONFIG.MAXIMUM_SEARCH_RESULTS,
"entities":
",".join([str(x) for x in hint_entity_ids]),
"has_referral":
has_referral,
"referral_name":
referral_name,
"is_all_entities":
is_all_entities,
"entry_name":
entry_name,
},
)
