class DataQualityCheckViewSet(viewsets.ViewSet):
"""
Handles Data Quality API operations within Inventory backend.
(1) Post, wait, get…
(2) Respond with what changed
"""
# Remove lookup_field once data_quality_check_id is used and "pk" can be used
lookup_field = 'organization_id'
@swagger_auto_schema(
manual_parameters=[
AutoSchemaHelper.base_field(
"organization_id", "IN_PATH",
"Organization ID - identifier used to specify a DataQualityCheck",
True, "TYPE_INTEGER")
],
request_body=AutoSchemaHelper.schema_factory(
{
'property_view_ids': ['integer'],
'taxlot_view_ids': ['integer'],
},
description='An object containing IDs of the records to perform'
' data quality checks on. Should contain two keys- '
'property_view_ids and taxlot_view_ids, each of '
'which is an array of appropriate IDs.',
),
responses={
200:
AutoSchemaHelper.schema_factory({
'num_properties': 'integer',
'num_taxlots': 'integer',
'progress_key': 'string',
'progress': {},
})
})
@api_endpoint_class
@ajax_request_class
@has_perm_class('requires_member')
@action(detail=True, methods=['POST'])
def start(self, request, organization_id):
"""
This API endpoint will create a new data_quality check process in the background,
on potentially a subset of properties/taxlots, and return back a query key
"""
body = request.data
property_view_ids = body['property_view_ids']
taxlot_view_ids = body['taxlot_view_ids']
property_state_ids = PropertyView.objects.filter(
id__in=property_view_ids).values_list('state_id', flat=True)
taxlot_state_ids = TaxLotView.objects.filter(
id__in=taxlot_view_ids).values_list('state_id', flat=True)
# For now, organization_id is the only key currently used to identify DataQualityChecks
return_value = do_checks(organization_id, property_state_ids,
taxlot_state_ids)
return JsonResponse({
'num_properties': len(property_state_ids),
'num_taxlots': len(taxlot_state_ids),
# TODO #239: Deprecate progress_key from here and just use the 'progess.progress_key'
'progress_key': return_value['progress_key'],
'progress': return_value,
})
@swagger_auto_schema(manual_parameters=[
AutoSchemaHelper.query_org_id_field(),
AutoSchemaHelper.query_integer_field("run_id", True,
"Import file ID or cache key"),
])
@api_endpoint_class
@ajax_request_class
@has_perm_class('requires_member')
@action(detail=False, methods=['GET'])
def results_csv(self, request):
"""
Download a CSV of the results from a data quality run based on either the ID that was
given during the creation of the data quality task or the ID of the
import file which had it's records checked.
Note that it is not related to objects in the database, since the results
are stored in redis!
"""
run_id = request.query_params.get('run_id')
if run_id is None:
return JsonResponse(
{
'status': 'error',
'message':
'must include Import file ID or cache key as run_id'
},
status=status.HTTP_400_BAD_REQUEST)
data_quality_results = get_cache_raw(
DataQualityCheck.cache_key(run_id))
response = HttpResponse(content_type='text/csv')
response[
'Content-Disposition'] = 'attachment; filename="Data Quality Check Results.csv"'
writer = csv.writer(response)
if data_quality_results is None:
writer.writerow(['Error'])
writer.writerow(['data quality results not found'])
return response
writer.writerow([
'Table', 'Address Line 1', 'PM Property ID', 'Tax Lot ID',
'Custom ID', 'Field', 'Applied Label', 'Condition',
'Error Message', 'Severity'
])
for row in data_quality_results:
for result in row['data_quality_results']:
writer.writerow([
row['data_quality_results'][0]['table_name'],
row['address_line_1'],
row['pm_property_id'] if 'pm_property_id' in row else None,
row['jurisdiction_tax_lot_id']
if 'jurisdiction_tax_lot_id' in row else None,
row['custom_id_1'],
result['formatted_field'],
result.get('label', None),
result['condition'],
# the detailed_message field can have units which has superscripts/subscripts, so unidecode it!
unidecode(result['detailed_message']),
result['severity']
])
return response
@swagger_auto_schema(manual_parameters=[
AutoSchemaHelper.query_org_id_field(),
AutoSchemaHelper.query_integer_field("run_id", True,
"Import file ID or cache key"),
])
@api_endpoint_class
@ajax_request_class
@has_perm_class('requires_member')
@action(detail=False, methods=['GET'])
def results(self, request):
"""
Return the results of a data quality run based on either the ID that was
given during the creation of the data quality task or the ID of the
import file which had it's records checked.
Note that it is not related to objects in the database, since the results
are stored in redis!
"""
data_quality_id = request.query_params['run_id']
data_quality_results = get_cache_raw(
DataQualityCheck.cache_key(data_quality_id))
return JsonResponse({'data': data_quality_results})
CreateModelMixin,
)
from seed.decorators import ajax_request_class
from seed.lib.superperms.orgs.decorators import has_perm_class
from seed.lib.superperms.orgs.permissions import SEEDOrgPermissions
from seed.models.data_quality import DataQualityCheck, Rule
from seed.serializers.rules import RuleSerializer
from seed.utils.api import api_endpoint_class
from seed.utils.api_schema import AutoSchemaHelper
from seed.utils.viewsets import UpdateWithoutPatchModelMixin
_log = logging.getLogger(__name__)
nested_org_id_path_field = AutoSchemaHelper.base_field(
"nested_organization_id", "IN_PATH",
"Organization ID - identifier used to specify a DataQualityCheck and its Rules",
True, "TYPE_INTEGER")
@method_decorator(
name='list',
decorator=swagger_auto_schema(manual_parameters=[nested_org_id_path_field])
)
@method_decorator(
name='update',
decorator=swagger_auto_schema(manual_parameters=[nested_org_id_path_field])
)
@method_decorator(
name='destroy',
decorator=swagger_auto_schema(manual_parameters=[nested_org_id_path_field])
)
