def reset_cleansing_rules(self, request, pk=None):
"""
Resets an organization's data cleansing rules
---
parameter_strategy: replace
parameters:
- name: pk
description: Organization ID (Primary key)
type: integer
required: true
paramType: path
type:
status:
type: string
description: success or error
required: true
in_range_checking:
type: array[string]
required: true
description: An array of in-range error rules
missing_matching_field:
type: array[string]
required: true
description: An array of fields to verify existence
missing_values:
type: array[string]
required: true
description: An array of fields to ignore missing values
"""
org = Organization.objects.get(pk=pk)
Rules.restore_defaults(org)
return self.get_cleansing_rules(request, pk)
def get_cleansing_rules(request):
"""
Returns the cleansing rules for an org.
:param request:
:GET: Expects organization_id in the query string.
Returns::
{
'status': 'success',
'in_range_checking': An array of in-range error rules,
'missing_matching_field': An array of fields to verify existence,
'missing_values': An array of fields to ignore missing values
}
"""
org_id = request.GET.get('organization_id')
org = Organization.objects.get(pk=org_id)
result = {
'status': 'success',
'missing_matching_field': [],
'missing_values': [],
'in_range_checking': [],
# 'data_type_check': []
}
rules = Rules.objects.filter(org=org).order_by('field', 'severity')
if not rules.exists():
Rules.initialize_rules(org)
for rule in rules:
if rule.category == CATEGORY_MISSING_MATCHING_FIELD:
result['missing_matching_field'].append({
'field': rule.field,
'severity': _get_js_rule_severity(rule.severity),
})
elif rule.category == CATEGORY_MISSING_VALUES:
result['missing_values'].append({
'field': rule.field,
'severity': _get_js_rule_severity(rule.severity),
})
elif rule.category == CATEGORY_IN_RANGE_CHECKING:
result['in_range_checking'].append({
'field': rule.field,
'enabled': rule.enabled,
'type': _get_js_rule_type(rule.type),
'min': rule.min,
'max': rule.max,
'severity': _get_js_rule_severity(rule.severity),
'units': rule.units
})
# elif rule.category == CATEGORY_DATA_TYPE_CHECK:
# result['data_type_check'].append({
# 'field': rule.field
# })
return result
def reset_cleansing_rules(request):
"""
Resets an organization's data cleansing rules
:param request:
:GET: Expects organization_id in the query string.
Returns::
{
'status': 'success',
'in_range_checking': An array of in-range error rules,
'missing_matching_field': An array of fields to verify existence,
'missing_values': An array of fields to ignore missing values
}
"""
org = Organization.objects.get(pk=request.GET['organization_id'])
Rules.restore_defaults(org)
return get_cleansing_rules(request)
def reset_cleansing_rules(request):
"""
Resets an organization's data cleansing rules
:param request:
:GET: Expects organization_id in the query string.
Returns::
{
'status': 'success',
'in_range_checking': An array of in-range error rules,
'missing_matching_field': An array of fields to verify existence,
'missing_values': An array of fields to ignore missing values
}
"""
org_id = request.GET.get('organization_id')
org = Organization.objects.get(pk=org_id)
Rules.restore_defaults(org)
return get_cleansing_rules(request)
def save_cleansing_rules(request):
"""
Saves an organization's settings: name, query threshold, shared fields
Payload::
{
'organization_id: 2,
'cleansing_rules': {
'missing_matching_field': [
{
'field': 'address_line_1',
'severity': 'error'
}
],
'missing_values': [
{
'field': 'address_line_1',
'severity': 'error'
}
],
'in_range_checking': [
{
'field': 'conditioned_floor_area',
'enabled': true,
'type': 'number',
'min': null,
'max': 7000000,
'severity': 'error',
'units': 'square feet'
},
]
}
}
Returns::
{
'status': 'success',
}
"""
body = json.loads(request.body)
if body.get('organization_id') is None:
return {'status': 'error', 'message': 'missing the organization_id'}
try:
org = Organization.objects.get(pk=body['organization_id'])
except Organization.DoesNotExist:
return {'status': 'error', 'message': 'organization does not exist'}
if body.get('cleansing_rules') is None:
return {'status': 'error', 'message': 'missing the cleansing_rules'}
posted_rules = body['cleansing_rules']
updated_rules = []
for rule in posted_rules['missing_matching_field']:
updated_rules.append(
Rules(org=org,
field=rule['field'],
category=CATEGORY_MISSING_MATCHING_FIELD,
severity=_get_severity_from_js(rule['severity'])))
for rule in posted_rules['missing_values']:
updated_rules.append(
Rules(org=org,
field=rule['field'],
category=CATEGORY_MISSING_VALUES,
severity=_get_severity_from_js(rule['severity'])))
for rule in posted_rules['in_range_checking']:
updated_rules.append(
Rules(org=org,
field=rule['field'],
enabled=rule['enabled'],
category=CATEGORY_IN_RANGE_CHECKING,
type=_get_rule_type_from_js(rule['type']),
min=rule['min'],
max=rule['max'],
severity=_get_severity_from_js(rule['severity']),
units=rule['units']))
Rules.delete_rules(org)
for rule in updated_rules:
rule.save()
return {'status': 'success'}
def get_cleansing_rules(request):
"""
Returns the cleansing rules for an org.
:param request:
:GET: Expects organization_id in the query string.
Returns::
{
'status': 'success',
'in_range_checking': An array of in-range error rules,
'missing_matching_field': An array of fields to verify existence,
'missing_values': An array of fields to ignore missing values
}
"""
org_id = request.GET.get('organization_id')
org = Organization.objects.get(pk=org_id)
result = {
'status': 'success',
'missing_matching_field': [],
'missing_values': [],
'in_range_checking': [],
# 'data_type_check': []
}
rules = Rules.objects.filter(org=org).order_by('field', 'severity')
if not rules.exists():
Rules.initialize_rules(org)
for rule in rules:
if rule.category == CATEGORY_MISSING_MATCHING_FIELD:
result['missing_matching_field'].append({
'field':
rule.field,
'severity':
_get_js_rule_severity(rule.severity),
})
elif rule.category == CATEGORY_MISSING_VALUES:
result['missing_values'].append({
'field':
rule.field,
'severity':
_get_js_rule_severity(rule.severity),
})
elif rule.category == CATEGORY_IN_RANGE_CHECKING:
result['in_range_checking'].append({
'field':
rule.field,
'enabled':
rule.enabled,
'type':
_get_js_rule_type(rule.type),
'min':
rule.min,
'max':
rule.max,
'severity':
_get_js_rule_severity(rule.severity),
'units':
rule.units
})
# elif rule.category == CATEGORY_DATA_TYPE_CHECK:
# result['data_type_check'].append({
# 'field': rule.field
# })
return result
def save_cleansing_rules(request):
"""
Saves an organization's settings: name, query threshold, shared fields
Payload::
{
'organization_id: 2,
'cleansing_rules': {
'missing_matching_field': [
{
'field': 'address_line_1',
'severity': 'error'
}
],
'missing_values': [
{
'field': 'address_line_1',
'severity': 'error'
}
],
'in_range_checking': [
{
'field': 'conditioned_floor_area',
'enabled': true,
'type': 'number',
'min': null,
'max': 7000000,
'severity': 'error',
'units': 'square feet'
},
]
}
}
Returns::
{
'status': 'success',
}
"""
body = json.loads(request.body)
if body.get('organization_id') is None:
return {
'status': 'error',
'message': 'missing the organization_id'
}
try:
org = Organization.objects.get(pk=body['organization_id'])
except Organization.DoesNotExist:
return {
'status': 'error',
'message': 'organization does not exist'
}
if body.get('cleansing_rules') is None:
return {
'status': 'error',
'message': 'missing the cleansing_rules'
}
posted_rules = body['cleansing_rules']
updated_rules = []
for rule in posted_rules['missing_matching_field']:
updated_rules.append(Rules(
org=org,
field=rule['field'],
category=CATEGORY_MISSING_MATCHING_FIELD,
severity=_get_severity_from_js(rule['severity'])
))
for rule in posted_rules['missing_values']:
updated_rules.append(Rules(
org=org,
field=rule['field'],
category=CATEGORY_MISSING_VALUES,
severity=_get_severity_from_js(rule['severity'])
))
for rule in posted_rules['in_range_checking']:
updated_rules.append(Rules(
org=org,
field=rule['field'],
enabled=rule['enabled'],
category=CATEGORY_IN_RANGE_CHECKING,
type=_get_rule_type_from_js(rule['type']),
min=rule['min'],
max=rule['max'],
severity=_get_severity_from_js(rule['severity']),
units=rule['units']
))
Rules.delete_rules(org)
for rule in updated_rules:
rule.save()
return {'status': 'success'}
def save_cleansing_rules(self, request, pk=None):
"""
Saves an organization's settings: name, query threshold, shared fields
---
parameter_strategy: replace
parameters:
- name: pk
description: Organization ID (Primary key)
type: integer
required: true
paramType: path
- name: body
description: JSON body containing organization rules information
paramType: body
pytype: RulesSerializer
required: true
type:
status:
type: string
description: success or error
required: true
message:
type: string
description: error message, if any
required: true
"""
body = request.data
try:
org = Organization.objects.get(pk=pk)
except Organization.DoesNotExist:
return JsonResponse({
'status': 'error',
'message': 'organization does not exist'
}, status=status.HTTP_404_NOT_FOUND)
if body.get('cleansing_rules') is None:
return JsonResponse({
'status': 'error',
'message': 'missing the cleansing_rules'
}, status=status.HTTP_404_NOT_FOUND)
posted_rules = body['cleansing_rules']
updated_rules = []
for rule in posted_rules['missing_matching_field']:
updated_rules.append(Rules(
org=org,
field=rule['field'],
category=CATEGORY_MISSING_MATCHING_FIELD,
severity=_get_severity_from_js(rule['severity'])
))
for rule in posted_rules['missing_values']:
updated_rules.append(Rules(
org=org,
field=rule['field'],
category=CATEGORY_MISSING_VALUES,
severity=_get_severity_from_js(rule['severity'])
))
for rule in posted_rules['in_range_checking']:
updated_rules.append(Rules(
org=org,
field=rule['field'],
enabled=rule['enabled'],
category=CATEGORY_IN_RANGE_CHECKING,
type=_get_rule_type_from_js(rule['type']),
min=rule['min'],
max=rule['max'],
severity=_get_severity_from_js(rule['severity']),
units=rule['units']
))
Rules.delete_rules(org)
for rule in updated_rules:
rule.save()
return JsonResponse({'status': 'success'})
def cleansing_rules(self, request, pk=None):
"""
Returns the cleansing rules for an org.
---
parameter_strategy: replace
parameters:
- name: pk
description: Organization ID (Primary key)
type: integer
required: true
paramType: path
type:
status:
type: string
required: true
description: success or error
in_range_checking:
type: array[string]
required: true
description: An array of in-range error rules
missing_matching_field:
type: array[string]
required: true
description: An array of fields to verify existence
missing_values:
type: array[string]
required: true
description: An array of fields to ignore missing values
"""
org = Organization.objects.get(pk=pk)
result = {
'status': 'success',
'missing_matching_field': [],
'missing_values': [],
'in_range_checking': [],
# 'data_type_check': []
}
rules = Rules.objects.filter(org=org).order_by('field', 'severity')
if not rules.exists():
Rules.initialize_rules(org)
for rule in rules:
if rule.category == CATEGORY_MISSING_MATCHING_FIELD:
result['missing_matching_field'].append({
'field': rule.field,
'severity': _get_js_rule_severity(rule.severity),
})
elif rule.category == CATEGORY_MISSING_VALUES:
result['missing_values'].append({
'field': rule.field,
'severity': _get_js_rule_severity(rule.severity),
})
elif rule.category == CATEGORY_IN_RANGE_CHECKING:
result['in_range_checking'].append({
'field': rule.field,
'enabled': rule.enabled,
'type': _get_js_rule_type(rule.type),
'min': rule.min,
'max': rule.max,
'severity': _get_js_rule_severity(rule.severity),
'units': rule.units
})
# elif rule.category == CATEGORY_DATA_TYPE_CHECK:
# result['data_type_check'].append({
# 'field': rule.field
# })
return JsonResponse(result)
