def get(self, request, subscription_id, content_format, **_kwargs):
send_email = as_bool(request.GET, self.PARAM_SEND, False)
subscription = DocumentNotificationSubscription.objects.get(pk=subscription_id)
document_type = subscription.document_type
document_id = as_int(request.GET, self.PARAM_DOCUMENT, None)
if document_id:
document = Document.objects.filter(document_type=document_type, pk=document_id).first()
if not document:
return HttpResponseBadRequest('Document with id = {0} not found or has wrong type.'.format(document_id))
else:
document = Document.objects.filter(document_type=document_type).first()
if not document:
return HttpResponseBadRequest('Document id not provided and '
'there are no example documents of type {0}.'.format(document_type.code))
document_id = document.pk
field_handlers = build_field_handlers(document_type, include_suggested_fields=False)
field_values = get_document_field_values(document_type, document_id, handlers=field_handlers)
example_changes = dict()
if subscription.event in {DocumentAssignedEvent.code, DocumentChangedEvent.code} and field_values:
for h in field_handlers:
if random.random() > 0.3:
continue
field_type = FIELD_TYPES_REGISTRY.get(h.field_type) # type: FieldType
field = DocumentField.objects.filter(code=h.field_code).first()
if not field:
continue
example_value = field_type.example_python_value(field=field)
example_changes[h.field_code] = (example_value, field_values.get(h.field_code))
notification = render_notification(already_sent_user_ids=set(),
subscription=subscription,
document=document,
field_handlers=field_handlers,
field_values=field_values,
changes=example_changes,
changed_by_user=request.user
)
if not notification:
return HttpResponse('Notification contains no data.', status=200)
if content_format == self.FORMAT_HTML:
content = notification.html
content_type = 'text/html'
else:
content = notification.txt
content_type = 'text/plain'
if send_email:
log = ErrorCollectingLogger()
notification.send(log=log)
error = log.get_error()
if error:
return HttpResponseServerError(content=error, content_type='application/json')
return HttpResponse(content=content, content_type=content_type, status=200)
def get(self, request, config_id, content_format, **kwargs):
dst_user_id_or_name = request.GET[self.PARAM_DST_USER]
send_email = as_bool(request.GET, self.PARAM_SEND, False)
emulate_no_docs = as_bool(request.GET, self.PARAM_EMULATE_NO_DOCS,
False)
config = DocumentDigestConfig.objects.get(pk=config_id)
if dst_user_id_or_name:
try:
dst_user = User.objects.get(pk=int(dst_user_id_or_name))
except ValueError:
dst_user = User.objects.get(username=str(dst_user_id_or_name))
else:
dst_user = request.user
run_date = datetime.now(tz=dst_user.timezone or get_localzone())
try:
digest = render_digest(config=config,
dst_user=dst_user,
run_date=run_date,
emulate_no_docs=emulate_no_docs)
except Exception as e:
return HttpResponse(render_error(
'Exception caught while trying to render digest', e),
status=500,
content_type='text/plain')
if not digest:
return HttpResponse('Notification contains no data.', status=200)
if content_format == self.FORMAT_HTML:
content = digest.html
content_type = 'text/html'
else:
content = digest.txt
content_type = 'text/plain'
if send_email:
log = ErrorCollectingLogger()
digest.send(log)
log.raise_if_error()
return HttpResponse(content=content,
content_type=content_type,
status=200)
def validate_running_tasks(cls, request, dst_errors_dict: Dict[str, Any]):
if not cls.users_tasks_validation_enabled():
return
if not as_bool(request.GET, PARAM_OVERRIDE_WARNINGS):
user_tasks = UsersTasksValidationAdmin.get_user_task_names()
if user_tasks:
user_tasks = '; <br />'.join(user_tasks)
dst_errors_dict[cls.WARN_CODE_TASKS_RUNNING] = f'''The following background tasks are running at
def get(self, request, *args, **kwargs):
start = time.time()
add_query_syntax = as_bool(request.GET, 'add_query_syntax', False)
document_type_schema = dict()
for document_type in DocumentType.objects.all():
columns = get_columns(document_type,
include_suggested=False,
include_generic=document_type.is_generic()
) # type: List[ColumnDesc]
system_fields = FIELD_CODES_SHOW_BY_DEFAULT_GENERIC \
if document_type.is_generic() else FIELD_CODES_SHOW_BY_DEFAULT_NON_GENERIC
search_fields = set(document_type.search_fields.all().values_list(
'code', flat=True))
default_columns = {
c.name
for c in columns if c.field_code in system_fields
or c.field_code in search_fields
}
document_type_schema[
document_type.code] = _document_type_schema_to_dto(
document_type, columns, default_columns, add_query_syntax)
common_filters_by_document_type = dict()
for document_type_code, filter_id, title in SavedFilter.objects \
.filter(user__isnull=True, project_id__isnull=True, filter_type=FT_COMMON_FILTER) \
.values_list('document_type__code', 'id', 'title'):
common_filters_by_document_type[document_type_code] = {
'id': filter_id,
'title': title
}
common_filters_by_project = dict()
for project_id, filter_id, title in SavedFilter.objects \
.filter(user__isnull=True, project_id__isnull=False, filter_type=FT_COMMON_FILTER) \
.values_list('project_id', 'id', 'title'):
common_filters_by_project[project_id] = {
'id': filter_id,
'title': title
}
user_doc_grid_configs_by_project = dict()
for project_id, columns, column_filters, order_by in SavedFilter.objects \
.filter(user=request.user, project_id__isnull=False, filter_type=FT_USER_DOC_GRID_CONFIG) \
.values_list('project_id', 'columns', 'column_filters', 'order_by'):
user_doc_grid_configs_by_project[project_id] = {
'columns': columns,
'column_filters': column_filters,
'order_by': order_by
}
return Response({
'document_type_schema': document_type_schema,
'common_filters_by_document_type': common_filters_by_document_type,
'common_filters_by_project': common_filters_by_project,
'user_doc_grid_configs_by_project':
user_doc_grid_configs_by_project,
'time': time.time() - start
})
def get(self, request, document_type_code: str, *_args, **_kwargs):
start = time.time()
try:
document_type = DocumentType.objects.get(code=document_type_code)
project_ids = as_int_list(request.GET,
'project_ids') # type: List[int]
columns = as_str_list(request.GET, 'columns')
fmt = request.GET.get('fmt') or self.FMT_JSON
offset = as_int(request.GET, 'offset', None)
if offset is not None and offset < 0:
offset = None
limit = as_int(request.GET, 'limit', None)
if limit is not None and limit <= 0:
limit = None
# For json output we limit number of returned documents because we dont use streaming response for JSON
# and want to keep it fast.
if fmt == self.FMT_JSON and (
limit is None or limit > self.MAX_RETURNED_DOCUMENTS_JSON):
limit = self.MAX_RETURNED_DOCUMENTS_JSON
saved_filters = as_int_list(request.GET,
'saved_filters') # type: List[int]
column_filters = list()
for param, value in request.GET.items(): # type: str, str
if param.startswith(self.URL_PARAM_PREFIX_FILTER):
column_filters.append(
(param[len(self.URL_PARAM_PREFIX_FILTER):], value))
order_by = request.GET.get('order_by') or None # type: str
order_by = parse_order_by(order_by) if order_by else None
save_filter = as_bool(request.GET, 'save_filter',
False) # type: bool
return_reviewed = as_bool(request.GET, 'return_reviewed', False)
return_total = as_bool(request.GET, 'return_total', True)
return_data = as_bool(request.GET, 'return_data', True)
ignore_errors = as_bool(request.GET, 'ignore_errors', True)
if project_ids and save_filter:
column_filters_dict = {c: f for c, f in column_filters}
for project_id in project_ids:
SavedFilter.objects.update_or_create(
user=request.user,
document_type=document_type,
filter_type=FT_USER_DOC_GRID_CONFIG,
project_id=project_id,
defaults={
'user':
request.user,
'document_type':
document_type,
'filter_type':
FT_USER_DOC_GRID_CONFIG,
'project_id':
project_id,
'columns':
columns,
'column_filters':
column_filters_dict,
'title':
None,
'order_by': [(column, direction.value)
for column, direction in order_by]
if order_by else None
})
query_results = get_documents(
requester=request.user,
document_type=document_type,
project_ids=project_ids,
column_names=columns,
saved_filter_ids=saved_filters,
column_filters=column_filters,
order_by=order_by,
offset=offset,
limit=limit,
return_documents=return_data,
return_reviewed_count=return_reviewed,
return_total_count=return_total,
ignore_errors=ignore_errors) # type: DocumentQueryResults
if fmt.lower() == 'csv':
if not return_data:
raise APIRequestError(
'Export to csv requested with return_data=false')
else:
resp = StreamingHttpResponse(csv_gen(
query_results.column_codes, query_results.fetch(),
query_results.column_titles),
content_type='text/csv')
resp[
'Content-Disposition'] = 'attachment; filename="export.csv"'
return resp
else:
if query_results is None:
return Response({'time': time.time() - start})
# As we limit the number of returned documents for JSON we can keep response in non-streaming form.
return Response(
_query_results_to_json(query_results,
time.time() - start))
# Switch to StreamingHttpResponse if/when we really need to return very big json output.
# _query_results_to_json() returns dict with document items backed with a generator.
# But on local tests for small number of documents the streaming json output works two times
# slower than non-streaming response. CSV works the same fast.
# return StreamingHttpResponse(json_gen(_query_results_to_json(query_results, time.time() - start)),
# content_type='application/json')
except APIRequestError as e:
return e.to_response()
except Exception as e:
return APIRequestError(message='Unable to process request',
caused_by=e,
http_status_code=500).to_response()
def get(self, request, document_type_code: str, *_args, **_kwargs):
start = time.time()
try:
document_type = DocumentType.objects.get(code=document_type_code)
project_ids = as_int_list(request.GET, 'project_ids') # type: List[int]
columns = as_str_list(request.GET, 'columns')
include_annotations = as_bool(request.GET, 'associated_text')
if include_annotations:
all_annotation_columns = get_annotation_columns(document_type)
columns += [i.field_code for i in all_annotation_columns
if i.field_code.rstrip(FIELD_CODE_ANNOTATION_SUFFIX) in columns]
fmt = request.GET.get('fmt') or self.FMT_JSON
offset = as_int(request.GET, 'offset', None)
if offset is not None and offset < 0:
offset = None
limit = as_int(request.GET, 'limit', None)
if limit is not None and limit <= 0:
limit = None
# For json output we limit number of returned documents because we dont use streaming response for JSON
# and want to keep it fast.
if fmt == self.FMT_JSON and self.MAX_RETURNED_DOCUMENTS_JSON is not None \
and (limit is None or limit > self.MAX_RETURNED_DOCUMENTS_JSON):
limit = self.MAX_RETURNED_DOCUMENTS_JSON
saved_filters = as_int_list(request.GET, 'saved_filters') # type: List[int]
column_filters = list()
for param, value in request.GET.items(): # type: str, str
if param.startswith(self.URL_PARAM_PREFIX_FILTER):
column_filters.append((param[len(self.URL_PARAM_PREFIX_FILTER):], value))
order_by = request.GET.get('order_by') or None # type: str
order_by = parse_order_by(order_by) if order_by else None
save_filter = as_bool(request.GET, 'save_filter', False) # type: bool
return_reviewed = as_bool(request.GET, 'return_reviewed', False)
return_total = as_bool(request.GET, 'return_total', True)
return_data = as_bool(request.GET, 'return_data', True)
ignore_errors = as_bool(request.GET, 'ignore_errors', True)
if project_ids and save_filter:
column_filters_dict = {c: f for c, f in column_filters}
for project_id in project_ids:
with transaction.atomic():
obj = SavedFilter.objects.create(user=request.user,
document_type=document_type,
filter_type=FT_USER_DOC_GRID_CONFIG,
project_id=project_id,
columns=columns,
column_filters=column_filters_dict,
title=None,
order_by=[(column, direction.value)
for
column, direction in
order_by] if order_by
else None
)
SavedFilter.objects.filter(user=request.user,
document_type=document_type,
filter_type=FT_USER_DOC_GRID_CONFIG,
project_id=project_id) \
.exclude(pk=obj.pk) \
.delete()
query_results = query_documents(requester=request.user,
document_type=document_type,
project_ids=project_ids,
column_names=columns,
saved_filter_ids=saved_filters,
column_filters=column_filters,
order_by=order_by,
offset=offset,
limit=limit,
return_documents=return_data,
return_reviewed_count=return_reviewed,
return_total_count=return_total,
ignore_errors=ignore_errors,
include_annotation_fields=True) # type: DocumentQueryResults
if fmt in {self.FMT_XLSX, self.FMT_CSV} and not return_data:
raise APIRequestError('Export to csv/xlsx requested with return_data=false')
if fmt == self.FMT_CSV:
return _query_results_to_csv(query_results)
elif fmt == self.FMT_XLSX:
return _query_results_to_xlsx(query_results)
else:
if query_results is None:
return Response({'time': time.time() - start})
return _query_results_to_json(query_results, time.time() - start)
except APIRequestError as e:
return e.to_response()
except Exception as e:
return APIRequestError(message='Unable to process request', caused_by=e, http_status_code=500).to_response()
def get(self, request, document_type_code: str, *_args, **_kwargs):
start = time.time()
try:
document_type = DocumentType.objects.get(code=document_type_code)
project_ids = as_int_list(request.GET,
'project_ids') # type: List[int]
columns = as_str_list(request.GET, 'columns')
include_annotations = as_bool(request.GET, 'associated_text')
if include_annotations:
all_annotation_columns = get_annotation_columns(document_type)
columns += [
i.field_code for i in all_annotation_columns
if i.field_code.rstrip(FIELD_CODE_ANNOTATION_SUFFIX) in
columns
]
fmt = request.GET.get('fmt') or self.FMT_JSON
as_zip = request.GET.get('as_zip') == 'true'
offset = as_int(request.GET, 'offset', None)
if offset is not None and offset < 0:
offset = None
limit = as_int(request.GET, 'limit', None)
if limit is not None and limit <= 0:
limit = None
# For json output we limit number of returned documents because we dont use streaming response for JSON
# and want to keep it fast.
if fmt == self.FMT_JSON and self.MAX_RETURNED_DOCUMENTS_JSON is not None \
and (limit is None or limit > self.MAX_RETURNED_DOCUMENTS_JSON):
limit = self.MAX_RETURNED_DOCUMENTS_JSON
saved_filters = as_int_list(request.GET,
'saved_filters') # type: List[int]
column_filters = list()
for param, value in request.GET.items(): # type: str, str
if param.startswith(self.URL_PARAM_PREFIX_FILTER):
column_filters.append(
(param[len(self.URL_PARAM_PREFIX_FILTER):], value))
# in case if filter params are passed like &filters=a=b&c=d
filter_query_string = request.GET.get('filters')
if filter_query_string:
for param, value in ast.literal_eval(
filter_query_string).items(): # type: str, str
if param.startswith(self.URL_PARAM_PREFIX_FILTER):
column_filters.append(
(param[len(self.URL_PARAM_PREFIX_FILTER):], value))
order_by = request.GET.get('order_by') or None # type: str
order_by = parse_order_by(order_by) if order_by else None
save_filter = as_bool(request.GET, 'save_filter',
False) # type: bool
return_reviewed = as_bool(request.GET, 'return_reviewed', False)
return_total = as_bool(request.GET, 'return_total', True)
return_data = as_bool(request.GET, 'return_data', True)
ignore_errors = as_bool(request.GET, 'ignore_errors', True)
if project_ids and save_filter:
column_filters_dict = {c: f for c, f in column_filters}
for project_id in project_ids:
with transaction.atomic():
obj = SavedFilter.objects.create(
user=request.user,
document_type=document_type,
filter_type=FT_USER_DOC_GRID_CONFIG,
project_id=project_id,
columns=columns,
column_filters=column_filters_dict,
title=None,
order_by=[(column, direction.value)
for column, direction in order_by]
if order_by else None)
SavedFilter.objects.filter(user=request.user,
filter_type=FT_USER_DOC_GRID_CONFIG,
project_id=project_id) \
.exclude(pk=obj.pk) \
.delete()
# show_unprocessed = as_bool(request.GET, 'show_unprocessed', False)
# if show_unprocessed is False:
# column_filters.append((FIELD_CODE_DOC_PROCESSED, 'true'))
total_documents_query = Document.objects.filter(
document_type=document_type)
if project_ids:
total_documents_query = total_documents_query.filter(
project_id__in=project_ids)
total_documents_of_type = total_documents_query.count()
columns_to_query = columns
if columns_to_query:
columns_to_query = leave_unique_values(
['document_id', 'document_name'] + columns)
query_results = query_documents(
requester=request.user,
document_type=document_type,
project_ids=project_ids,
column_names=columns_to_query, # columns,
saved_filter_ids=saved_filters,
column_filters=column_filters,
order_by=order_by,
offset=offset,
limit=limit,
return_documents=return_data,
return_reviewed_count=return_reviewed,
return_total_count=return_total,
ignore_errors=ignore_errors,
include_annotation_fields=True) # type: DocumentQueryResults
if query_results is None:
if fmt in {self.FMT_XLSX, self.FMT_CSV} and not return_data:
raise APIRequestError('Empty data, nothing to export')
return Response({'time': time.time() - start})
# get assignees stats
assignees_query_results = query_documents(
requester=request.user,
document_type=document_type,
project_ids=project_ids,
column_names=['document_id', 'assignee_name', 'assignee_id'],
saved_filter_ids=saved_filters,
column_filters=column_filters,
return_documents=True,
return_reviewed_count=False,
include_annotation_fields=include_annotations
) # type: DocumentQueryResults
query_results.assignees = []
if assignees_query_results is not None:
df = pd.DataFrame(assignees_query_results.fetch_dicts())
if not df.empty:
df = df.groupby(['assignee_id', 'assignee_name'])\
.agg({'document_id': [('document_ids', lambda x: list(x)), ('documents_count', 'count')]})
if not df.empty:
df.columns = df.columns.droplevel()
df = df.reset_index()
df['assignee_id'] = df['assignee_id'].astype(int)
query_results.assignees = df.to_dict('records')
query_results.unfiltered_count = total_documents_of_type
if fmt in {self.FMT_XLSX, self.FMT_CSV} and not return_data:
raise APIRequestError(
'Export to csv/xlsx requested with return_data=false')
if fmt == self.FMT_CSV:
return query_results.to_csv(as_zip=as_zip)
elif fmt == self.FMT_XLSX:
return query_results.to_xlsx(as_zip=as_zip)
else:
query_dict = query_results.to_json(time_start=start)
if columns and 'items' in query_dict:
columns_to_remove = []
if 'document_id' not in columns:
columns_to_remove.append('document_id')
query_dict['items'] = self.expand_items(
query_dict['items'], columns_to_remove)
return Response(query_dict)
except APIRequestError as e:
return e.to_response()
except Exception as e:
return APIRequestError(message='Unable to process request',
caused_by=e,
http_status_code=500).to_response()
def get(self, request, *args, **kwargs):
start = time.time()
add_query_syntax = as_bool(request.GET, 'add_query_syntax', False)
document_type_schema = dict()
for document_type in DocumentType.objects.all():
columns = get_columns(document_type,
include_generic=document_type.is_generic(),
is_select=True) # type: List[ColumnDesc]
columns = [
c for c in columns
if c.field_code not in FIELD_CODES_HIDE_FROM_CONFIG_API
]
system_fields = FIELD_CODES_SHOW_BY_DEFAULT_GENERIC \
if document_type.is_generic() else FIELD_CODES_SHOW_BY_DEFAULT_NON_GENERIC
search_fields = set(document_type.search_fields.all().values_list(
'code', flat=True))
default_columns = {
c.name
for c in columns
if c.field_code not in FIELD_CODES_HIDE_FROM_CONFIG_API
and c.field_code not in FIELD_CODES_HIDE_BY_DEFAULT and
(c.field_code in system_fields or c.field_code in search_fields
)
}
document_type_schema[
document_type.code] = _document_type_schema_to_dto(
document_type, columns, default_columns, add_query_syntax)
common_filters_by_document_type = defaultdict(list) # type: Dict[List]
for document_type_code, filter_id, title, display_order in SavedFilter.objects \
.filter(project_id__isnull=True, filter_type=FT_COMMON_FILTER) \
.filter(Q(user__isnull=True) | Q(user=request.user)) \
.values_list('document_type__code', 'id', 'title', 'display_order'):
common_filters_by_document_type[document_type_code].append({
'id':
filter_id,
'title':
title,
'display_order':
display_order
})
common_filters_by_project = defaultdict(list) # type: Dict[List]
for project_id, filter_id, title, display_order in SavedFilter.objects \
.filter(project_id__isnull=False, filter_type=FT_COMMON_FILTER) \
.filter(Q(user__isnull=True) | Q(user=request.user)) \
.values_list('project_id', 'id', 'title', 'display_order'):
common_filters_by_project[project_id].append({
'id':
filter_id,
'title':
title,
'display_order':
display_order
})
user_doc_grid_configs_by_project = defaultdict(
list) # type: Dict[List]
for project_id, columns, column_filters, order_by in SavedFilter.objects \
.filter(user=request.user, project_id__isnull=False, filter_type=FT_USER_DOC_GRID_CONFIG) \
.filter(document_type_id=F('project__type_id')) \
.order_by('pk') \
.values_list('project_id', 'columns', 'column_filters', 'order_by'):
user_doc_grid_configs_by_project[project_id] = {
'columns': columns,
'column_filters': column_filters,
'order_by': order_by
}
return Response({
'document_type_schema': document_type_schema,
'common_filters_by_document_type': common_filters_by_document_type,
'common_filters_by_project': common_filters_by_project,
'user_doc_grid_configs_by_project':
user_doc_grid_configs_by_project,
'time': time.time() - start,
'scheme_migrations': {
'migrations': MIGRATION_TAGS,
'current_version': CURRENT_VERSION
}
})
