def test_create_rule_using_org_id_to_establish_dq_check_relationship(self):
# Ensure no address_line_2 rules exist by default beforehand
dq = DataQualityCheck.retrieve(self.org.id)
self.assertEqual(0, dq.rules.filter(field='address_line_2').count())
base_rule_info = {
'field': 'address_line_2',
'table_name': 'PropertyState',
'enabled': True,
'data_type': Rule.TYPE_STRING,
'rule_type': Rule.RULE_TYPE_DEFAULT,
'condition': Rule.RULE_INCLUDE,
'required': False,
'not_null': False,
'min': None,
'max': None,
'text_match': 'some random text',
'severity': Rule.SEVERITY_ERROR,
'units': "",
'status_label': None,
}
url = reverse('api:v3:data_quality_check-rules-list',
kwargs={'nested_organization_id': self.org.id})
self.client.post(url,
content_type='application/json',
data=json.dumps(base_rule_info))
dq = DataQualityCheck.retrieve(self.org.id)
self.assertEqual(1, dq.rules.filter(field='address_line_2').count())
def create_organization(user=None, org_name='', *args, **kwargs):
"""
Helper script to create a user/org relationship from scratch. This is heavily used and
creates the default labels, columns, and data quality rules when a new organization is created
:param user: user inst.
:param org_name: str, name of Organization we'd like to create.
:param (optional) kwargs: 'role', int; 'status', str.
"""
from seed.models import StatusLabel as Label
organization_user = None
user_added = False
organization = Organization.objects.create(name=org_name)
if user:
organization_user, user_added = OrganizationUser.objects.get_or_create(
user=user, organization=organization)
for label in Label.DEFAULT_LABELS:
Label.objects.get_or_create(
name=label,
super_organization=organization,
defaults={'color': 'blue'},
)
# upon initializing a new organization (SuperOrganization), create
# the default columns
_create_default_columns(organization.id)
# create the default rules for this organization
DataQualityCheck.retrieve(organization.id)
return organization, organization_user, user_added
def test_ensure_default_rules(self):
dq = DataQualityCheck.retrieve(self.org)
initial_pk = dq.pk
self.assertEqual(dq.rules.count(), len(DEFAULT_RULES))
self.assertEqual(dq.results, {})
self.assertEqual(initial_pk, dq.pk)
# check again to make sure that it does not append more rules to the same org
dq = DataQualityCheck.retrieve(self.org.pk)
self.assertEqual(dq.rules.count(), len(DEFAULT_RULES))
def get_queryset(self):
# Handle the anonymous case (e.g. Swagger page load)
if not self.kwargs:
return Rule.objects.none()
org_id = self.kwargs.get('nested_organization_id')
rule_id = self.kwargs.get('pk')
if rule_id is None:
return DataQualityCheck.retrieve(org_id).rules.all()
else:
return DataQualityCheck.retrieve(org_id).rules.filter(id=rule_id)
def test_multiple_data_quality_check_objects(self):
dq = DataQualityCheck.retrieve(self.org)
self.assertEqual(dq.name, 'Default Data Quality Check')
DataQualityCheck.objects.create(organization=self.org,
name='test manual creation')
DataQualityCheck.objects.create(organization=self.org,
name='test manual creation 2')
DataQualityCheck.objects.create(organization=self.org,
name='test manual creation 3')
dq = DataQualityCheck.retrieve(self.org)
# The method above will delete the multiple objects and return the original
self.assertEqual(dq.name, 'Default Data Quality Check')
def test_property_state_quality(self):
# Import the file and run mapping
qs = PropertyState.objects.filter(
import_file=self.import_file, ).iterator()
d = DataQualityCheck.retrieve(self.org)
d.check_data('PropertyState', qs)
self.assertEqual(len(d.results), 7)
result = d.retrieve_result_by_address('95373 E Peach Avenue')
self.assertTrue(result['address_line_1'], '95373 E Peach Avenue')
res = [{
"severity": "error",
"value": "",
"field": "pm_property_id",
"table_name": "PropertyState",
"message": "PM Property ID is null",
"detailed_message": "PM Property ID is null",
"formatted_field": "PM Property ID"
}]
self.assertEqual(res, result['data_quality_results'])
result = d.retrieve_result_by_address('120243 E True Lane')
res = [{
"severity": "error",
"value": "10000000000.0",
"field": "gross_floor_area",
"table_name": "PropertyState",
"message": "Gross Floor Area out of range",
"detailed_message": "Gross Floor Area [10000000000.0] > 7000000.0",
"formatted_field": "Gross Floor Area"
}, {
"severity": "error",
"value": "0",
"field": "year_built",
"table_name": "PropertyState",
"message": "Year Built out of range",
"detailed_message": "Year Built [0] < 1700",
"formatted_field": "Year Built"
}, {
"severity": "error",
"value": "",
"field": "custom_id_1",
"table_name": "PropertyState",
"message": "Custom ID 1 (Property) is null",
"detailed_message": "Custom ID 1 (Property) is null",
"formatted_field": "Custom ID 1 (Property)"
}, {
"severity": "error",
"value": "",
"field": "pm_property_id",
"table_name": "PropertyState",
"message": "PM Property ID is null",
"detailed_message": "PM Property ID is null",
"formatted_field": "PM Property ID"
}]
self.assertItemsEqual(res, result['data_quality_results'])
result = d.retrieve_result_by_address('1234 Peach Tree Avenue')
self.assertEqual(result, None)
def test_reset_default_rules(self):
dq = DataQualityCheck.retrieve(self.org)
new_rule = {
'table_name': 'PropertyState',
'field': 'test_floor_area',
'data_type': TYPE_NUMBER,
'rule_type': RULE_TYPE_DEFAULT,
'min': 0,
'max': 7000000,
'severity': SEVERITY_ERROR,
'units': 'square feet'
}
dq.add_rule(new_rule)
self.assertEqual(dq.rules.count(), len(DEFAULT_RULES) + 1)
# change one of the default rules
rule = dq.rules.filter(field='gross_floor_area').first()
rule.min = -10000
rule.save()
self.assertEqual(
dq.rules.filter(field='gross_floor_area').first().min, -10000)
dq.reset_default_rules()
self.assertEqual(
dq.rules.filter(field='gross_floor_area').first().min, 100)
# ensure non-default rule still exists
non_def_rules = dq.rules.filter(field='test_floor_area')
self.assertEqual(non_def_rules.count(), 1)
def test_check_property_state_example_data_with_labels(self):
dq = DataQualityCheck.retrieve(self.org.id)
# Create labels and apply them to the rules being triggered later
site_eui_label = StatusLabel.objects.create(name='Check Site EUI', super_organization=self.org)
site_eui_rule = dq.rules.get(table_name='PropertyState', field='site_eui', max='1000')
site_eui_rule.status_label = site_eui_label
site_eui_rule.save()
year_built_label = StatusLabel.objects.create(name='Check Year Built', super_organization=self.org)
year_built_rule = dq.rules.get(table_name='PropertyState', field='year_built')
year_built_rule.status_label = year_built_label
year_built_rule.save()
# Create state and associate it to view
ps_data = {
'no_default_data': True,
'custom_id_1': 'abcd',
'address_line_1': '742 Evergreen Terrace',
'pm_property_id': 'PMID',
'site_eui': 525600,
'year_built': 1699,
}
ps = self.property_state_factory.get_property_state(None, **ps_data)
property = self.property_factory.get_property()
PropertyView.objects.create(
property=property, cycle=self.cycle, state=ps
)
dq.check_data(ps.__class__.__name__, [ps])
dq_results = dq.results[ps.id]['data_quality_results']
labels = [r['label'] for r in dq_results]
self.assertCountEqual(['Check Site EUI', 'Check Year Built'], labels)
def reset_default_data_quality_rules(self, request):
"""
Resets an organization's data data_quality rules
---
parameters:
- name: organization_id
description: Organization ID
type: integer
required: true
paramType: query
type:
status:
type: string
description: success or error
required: true
in_range_checking:
type: array[string]
required: true
description: An array of in-range error rules
missing_matching_field:
type: array[string]
required: true
description: An array of fields to verify existence
missing_values:
type: array[string]
required: true
description: An array of fields to ignore missing values
"""
organization = Organization.objects.get(
pk=request.query_params['organization_id'])
dq = DataQualityCheck.retrieve(organization.id)
dq.reset_default_rules()
return self.data_quality_rules(request)
def test_rule_with_label_set_to_null(self):
dq = DataQualityCheck.retrieve(self.org)
sl_data = {
'name': 'test label on rule for null',
'super_organization': self.org
}
status_label, _ = StatusLabel.objects.get_or_create(**sl_data)
new_rule = {
'name': 'Name not to be forgotten',
'table_name': 'PropertyState',
'field': 'conditioned_floor_area',
'data_type': TYPE_NUMBER,
'rule_type': RULE_TYPE_DEFAULT,
'min': 0,
'max': 7000000,
'severity': SEVERITY_ERROR,
'units': 'square feet',
'status_label': status_label
}
dq.add_rule(new_rule)
# Find data rule that has the status rule (from above)
rules = dq.rules.filter(status_label__isnull=False)
self.assertEqual(rules.count(), 1)
self.assertEqual(rules[0].status_label, status_label)
status_label.delete()
def test_rule_with_label(self):
dq = DataQualityCheck.retrieve(self.org)
rules = dq.rules.filter(status_label__isnull=False)
self.assertEqual(rules.count(), 0)
sl_data = {
'name': 'test label on rule',
'super_organization': self.org
}
status_label, _ = StatusLabel.objects.get_or_create(**sl_data)
sls = StatusLabel.objects.filter(**sl_data)
self.assertEqual(sls.count(), 1)
new_rule = {
'table_name': 'PropertyState',
'field': 'conditioned_floor_area',
'data_type': TYPE_NUMBER,
'rule_type': RULE_TYPE_DEFAULT,
'min': 0,
'max': 7000000,
'severity': SEVERITY_ERROR,
'units': 'square feet',
'status_label': status_label
}
dq.add_rule(new_rule)
rules = dq.rules.filter(status_label__isnull=False)
self.assertEqual(rules.count(), 1)
self.assertEqual(rules[0].status_label, status_label)
# delete the rule but make sure that the label does not get deleted
dq.remove_all_rules()
sls = StatusLabel.objects.filter(**sl_data)
self.assertEqual(sls.count(), 1)
def create(self, validated_data):
# For now, use an Org ID to find the DQ Check ID to apply (later, use the DQ Check ID directly)
org_id = self.context['request'].parser_context['kwargs'][
'nested_organization_id']
validated_data['data_quality_check_id'] = DataQualityCheck.retrieve(
org_id).id
return Rule.objects.create(**validated_data)
def test_add_new_rule_exception(self):
dq = DataQualityCheck.retrieve(self.org)
new_rule = {'wrong': 'data'}
with self.assertRaisesRegexp(
TypeError,
"Rule data is not defined correctly: 'wrong' is an invalid keyword argument for this function"
):
dq.add_rule(new_rule)
def test_update_rule_include_empty_text_match_validation(self):
# Start with 1 Rule
dq = DataQualityCheck.retrieve(self.org.id)
dq.remove_all_rules()
base_rule_info = {
'field': 'address_line_1',
'table_name': 'PropertyState',
'enabled': True,
'data_type': Rule.TYPE_STRING,
'rule_type': Rule.RULE_TYPE_DEFAULT,
'condition': Rule.RULE_INCLUDE,
'required': False,
'not_null': False,
'min': None,
'max': None,
'text_match': 'Test Rule 1',
'severity': Rule.SEVERITY_ERROR,
'units': "",
}
dq.add_rule(base_rule_info)
rule = dq.rules.get()
# Send invalid update request
put_data = deepcopy(base_rule_info)
put_data['text_match'] = None
url = reverse('api:v3:data_quality_check-rules-detail',
kwargs={
'nested_organization_id': self.org.id,
'pk': rule.id
})
res = self.client.put(url,
content_type='application/json',
data=json.dumps(put_data))
self.assertEqual(res.status_code, 400)
self.assertTrue('Rule must not include or exclude an empty string. ' in
json.loads(res.content)['message'])
# Remove text_match and make condition NOT_NULL, then try making condition EXCLUDE
rule.text_match = None
rule.condition = Rule.RULE_NOT_NULL
rule.save()
put_data_2 = deepcopy(base_rule_info)
del put_data_2['text_match'] # don't update text_match
put_data_2['condition'] = Rule.RULE_EXCLUDE
url = reverse('api:v3:data_quality_check-rules-detail',
kwargs={
'nested_organization_id': self.org.id,
'pk': dq.rules.get().id
})
res = self.client.put(url,
content_type='application/json',
data=json.dumps(put_data_2))
self.assertEqual(res.status_code, 400)
self.assertTrue('Rule must not include or exclude an empty string. ' in
json.loads(res.content)['message'])
def reset(self, request, nested_organization_id=None):
"""
Resets an organization's data data_quality rules
"""
# TODO: Refactor to get all the rules for a DataQualityCheck object directly.
# At that point, nested_organization_id should be changed to data_quality_check_id
dq = DataQualityCheck.retrieve(nested_organization_id)
dq.remove_all_rules()
return self.list(request, nested_organization_id)
def test_tax_lot_state_quality(self):
# Import the file and run mapping
qs = TaxLotState.objects.filter(
import_file=self.import_file).iterator()
d = DataQualityCheck.retrieve(self.org)
d.check_data('TaxLotState', qs)
# import json
# print json.dumps(d.results, indent=2)
self.assertEqual(len(d.results), 4)
def test_remove_all_rules(self):
dq = DataQualityCheck.retrieve(self.org)
count = Rule.objects.filter(data_quality_check_id=dq.pk).count()
self.assertEqual(count, len(DEFAULT_RULES))
dq.remove_all_rules()
self.assertEqual(dq.rules.count(), 0)
# ensure that the database has no rules for this dq associated with it
count = Rule.objects.filter(data_quality_check_id=dq.pk).count()
self.assertEqual(count, 0)
def test_filter_rules(self):
dq = DataQualityCheck.retrieve(self.org)
rule_count = dq.rules.filter(enabled=True).count()
# disable one of the rules
rule = dq.rules.first()
rule.enabled = False
rule.save()
rules = dq.rules.filter(enabled=True)
self.assertEqual(rules.count(), rule_count - 1)
def test_add_custom_rule_exception(self):
dq = DataQualityCheck.retrieve(self.org.id)
dq.remove_all_rules()
ex_rule = {
'table_name_does_not_exist': 'PropertyState',
}
with self.assertRaises(Exception) as exc:
dq.add_rule(ex_rule)
self.assertEqual(
str(exc.exception),
"Rule data is not defined correctly: 'table_name_does_not_exist' is an invalid keyword argument for this function"
)
def test_add_new_rule_and_reset(self):
dq = DataQualityCheck.retrieve(self.org)
new_rule = {
'table_name': 'PropertyState',
'field': 'conditioned_floor_area',
'data_type': TYPE_NUMBER,
'rule_type': RULE_TYPE_DEFAULT,
'min': 0,
'max': 7000000,
'severity': SEVERITY_ERROR,
'units': 'square feet'
}
dq.add_rule(new_rule)
self.assertEqual(dq.rules.count(), len(DEFAULT_RULES) + 1)
dq.reset_all_rules()
self.assertEqual(dq.rules.count(), len(DEFAULT_RULES))
def test_add_custom_rule(self):
dq = DataQualityCheck.retrieve(self.org.id)
dq.remove_all_rules()
ex_rule = {
'table_name': 'PropertyState',
'field': 'some_floor_area',
'data_type': Rule.TYPE_AREA,
'rule_type': Rule.RULE_TYPE_DEFAULT,
'min': 8760,
'max': 525600,
'severity': Rule.SEVERITY_ERROR,
'units': 'm**2',
}
dq.add_rule(ex_rule)
self.assertEqual(dq.rules.count(), 1)
self.assertDictContainsSubset(ex_rule, model_to_dict(dq.rules.first()))
def test_default_create(self):
dq = DataQualityCheck.retrieve(self.org.id)
self.assertEqual(dq.rules.count(), 22)
# Example rule to check
ex_rule = {
'table_name': 'PropertyState',
'field': 'conditioned_floor_area',
'data_type': Rule.TYPE_AREA,
'rule_type': Rule.RULE_TYPE_DEFAULT,
'min': 0,
'max': 7000000,
'severity': Rule.SEVERITY_ERROR,
'units': 'ft**2',
}
rule = Rule.objects.filter(table_name='PropertyState',
field='conditioned_floor_area',
severity=Rule.SEVERITY_ERROR)
self.assertDictContainsSubset(ex_rule, model_to_dict(rule.first()))
def test_update_rule_status_label_validation(self):
# Start with 1 Rule
dq = DataQualityCheck.retrieve(self.org.id)
dq.remove_all_rules()
base_rule_info = {
'field': 'address_line_1',
'table_name': 'PropertyState',
'enabled': True,
'data_type': Rule.TYPE_STRING,
'rule_type': Rule.RULE_TYPE_DEFAULT,
'condition': Rule.RULE_INCLUDE,
'required': False,
'not_null': False,
'min': None,
'max': None,
'text_match': 'Test Rule 1',
'severity': Rule.SEVERITY_ERROR,
'units': "",
}
dq.add_rule(base_rule_info)
rule = dq.rules.get()
# Send invalid update request that includes a label id from another org
new_org, _, _ = create_organization(self.user, "test-organization-a")
wrong_org_label_id = new_org.labels.first().id
put_data = deepcopy(base_rule_info)
put_data['status_label'] = wrong_org_label_id
url = reverse('api:v3:data_quality_check-rules-detail',
kwargs={
'nested_organization_id': self.org.id,
'pk': rule.id
})
res = self.client.put(url,
content_type='application/json',
data=json.dumps(put_data))
self.assertEqual(res.status_code, 400)
self.assertTrue(
f'Label with ID {wrong_org_label_id} not found in organization, {self.org.name}.'
in json.loads(res.content)['status_label'])
def test_reset_rules(self):
# Start with 1 Rule
dq = DataQualityCheck.retrieve(self.org.id)
dq.remove_all_rules()
base_rule_info = {
'field': 'address_line_1',
'table_name': 'PropertyState',
'enabled': True,
'data_type': Rule.TYPE_STRING,
'rule_type': Rule.RULE_TYPE_DEFAULT,
'condition': Rule.RULE_INCLUDE,
'required': False,
'not_null': False,
'min': None,
'max': None,
'text_match': 'Test Rule 1',
'severity': Rule.SEVERITY_ERROR,
'units': "",
}
dq.add_rule(base_rule_info)
url = reverse('api:v3:data_quality_check-rules-reset',
kwargs={'nested_organization_id': self.org.id})
response = self.client.put(url)
rules = json.loads(response.content)
self.assertEqual(len(rules), 22)
property_count = 0
taxlot_count = 0
for r in rules:
if r['table_name'] == 'PropertyState':
property_count += 1
elif r['table_name'] == 'TaxLotState':
taxlot_count += 1
self.assertEqual(taxlot_count, 2)
self.assertEqual(property_count, 20)
def test_text_match(self):
dq = DataQualityCheck.retrieve(self.org.id)
dq.remove_all_rules()
new_rule = {
'table_name': 'PropertyState',
'field': 'address_line_1',
'data_type': Rule.TYPE_STRING,
'rule_type': Rule.RULE_TYPE_DEFAULT,
'severity': Rule.SEVERITY_ERROR,
'not_null': True,
'text_match': 742,
}
dq.add_rule(new_rule)
ps_data = {
'no_default_data': True,
'custom_id_1': 'abcd',
'address_line_1': '742 Evergreen Terrace',
'pm_property_id': 'PMID',
'site_eui': 525600,
}
ps = self.property_state_factory.get_property_state(None, **ps_data)
dq.check_data(ps.__class__.__name__, [ps])
self.assertEqual(dq.results, {})
def test_check_property_state_example_data(self):
dq = DataQualityCheck.retrieve(self.org.id)
ps_data = {
'no_default_data': True,
'custom_id_1': 'abcd',
'address_line_1': '742 Evergreen Terrace',
'pm_property_id': 'PMID',
'site_eui': 525600,
}
ps = self.property_state_factory.get_property_state(None, **ps_data)
dq.check_data(ps.__class__.__name__, [ps])
# {
# 11: {
# 'id': 11,
# 'custom_id_1': 'abcd',
# 'pm_property_id': 'PMID',
# 'address_line_1': '742 Evergreen Terrace',
# 'data_quality_results': [
# {
# 'severity': 'error', 'value': '525600', 'field': 'site_eui', 'table_name': 'PropertyState', 'message': 'Site EUI out of range', 'detailed_message': 'Site EUI [525600] > 1000', 'formatted_field': 'Site EUI'
# }
# ]
# }
error_found = False
for index, row in dq.results.items():
self.assertEqual(row['custom_id_1'], 'abcd')
self.assertEqual(row['pm_property_id'], 'PMID')
self.assertEqual(row['address_line_1'], '742 Evergreen Terrace')
for violation in row['data_quality_results']:
if violation['message'] == 'Site EUI out of range':
error_found = True
self.assertEqual(violation['detailed_message'],
'Site EUI [525600] > 1000')
self.assertEqual(error_found, True)
def save_data_quality_rules(self, request, pk=None):
"""
Saves an organization's settings: name, query threshold, shared fields.
The method passes in all the fields again, so it is okay to remove
all the rules in the db, and just recreate them (albeit inefficient)
---
parameter_strategy: replace
parameters:
- name: organization_id
description: Organization ID
type: integer
required: true
paramType: query
- name: body
description: JSON body containing organization rules information
paramType: body
pytype: RulesSerializer
required: true
type:
status:
type: string
description: success or error
required: true
message:
type: string
description: error message, if any
required: true
"""
organization = Organization.objects.get(
pk=request.query_params['organization_id'])
body = request.data
if body.get('data_quality_rules') is None:
return JsonResponse(
{
'status': 'error',
'message': 'missing the data_quality_rules'
},
status=status.HTTP_404_NOT_FOUND)
posted_rules = body['data_quality_rules']
updated_rules = []
for rule in posted_rules['properties']:
updated_rules.append({
'field':
rule['field'],
'table_name':
'PropertyState',
'enabled':
rule['enabled'],
'data_type':
_get_rule_type_from_js(rule['data_type']),
'rule_type':
rule['rule_type'],
'required':
rule['required'],
'not_null':
rule['not_null'],
'min':
rule['min'],
'max':
rule['max'],
'text_match':
rule['text_match'],
'severity':
_get_severity_from_js(rule['severity']),
'units':
rule['units'],
'status_label_id':
rule['label']
})
for rule in posted_rules['taxlots']:
updated_rules.append({
'field':
rule['field'],
'table_name':
'TaxLotState',
'enabled':
rule['enabled'],
'data_type':
_get_rule_type_from_js(rule['data_type']),
'rule_type':
rule['rule_type'],
'required':
rule['required'],
'not_null':
rule['not_null'],
'min':
rule['min'],
'max':
rule['max'],
'text_match':
rule['text_match'],
'severity':
_get_severity_from_js(rule['severity']),
'units':
rule['units'],
'status_label_id':
rule['label']
})
dq = DataQualityCheck.retrieve(organization.id)
dq.remove_all_rules()
for rule in updated_rules:
try:
dq.add_rule(rule)
except TypeError as e:
return JsonResponse({
'status': 'error',
'message': e,
},
status=status.HTTP_400_BAD_REQUEST)
return self.data_quality_rules(request)
def data_quality_rules(self, request):
"""
Returns the data_quality rules for an org.
---
parameters:
- name: organization_id
description: Organization ID
type: integer
required: true
paramType: query
type:
status:
type: string
required: true
description: success or error
rules:
type: object
required: true
description: An object containing 'properties' and 'taxlots' arrays of rules
"""
organization = Organization.objects.get(
pk=request.query_params['organization_id'])
result = {
'status': 'success',
'rules': {
'properties': [],
'taxlots': []
}
}
dq = DataQualityCheck.retrieve(organization.id)
rules = dq.rules.order_by('field', 'severity')
for rule in rules:
result['rules']['properties' if rule.table_name ==
'PropertyState' else 'taxlots'].append({
'field':
rule.field,
'enabled':
rule.enabled,
'data_type':
_get_js_rule_type(rule.data_type),
'rule_type':
rule.rule_type,
'required':
rule.required,
'not_null':
rule.not_null,
'min':
rule.min,
'max':
rule.max,
'text_match':
rule.text_match,
'severity':
_get_js_rule_severity(rule.severity),
'units':
rule.units,
'label':
rule.status_label_id
})
return JsonResponse(result)
def test_check_multiple_text_match(self):
d = DataQualityCheck.retrieve(self.org)
d.remove_all_rules()
sl_data = {'name': 'No meters present', 'super_organization': self.org}
sl_ok_1, _ = StatusLabel.objects.get_or_create(**sl_data)
new_rule = {
'table_name': 'PropertyState',
'field': 'meters_present',
'data_type': TYPE_STRING,
'rule_type': RULE_TYPE_CUSTOM,
'text_match': 'OK',
'severity': SEVERITY_ERROR,
'status_label': sl_ok_1,
}
d.add_rule(new_rule)
sl_data = {
'name': 'No 12 Consectutive Months',
'super_organization': self.org
}
sl_ok_2, _ = StatusLabel.objects.get_or_create(**sl_data)
new_rule = {
'table_name': 'PropertyState',
'field': '12 Consectutive Months',
'data_type': TYPE_STRING,
'rule_type': RULE_TYPE_CUSTOM,
'text_match': 'OK',
'severity': SEVERITY_ERROR,
'status_label': sl_ok_2,
}
d.add_rule(new_rule)
sl_data = {'name': 'No Monthly Data', 'super_organization': self.org}
sl_ok_3, _ = StatusLabel.objects.get_or_create(**sl_data)
new_rule = {
'table_name': 'PropertyState',
'field': 'Monthly Data',
'data_type': TYPE_STRING,
'rule_type': RULE_TYPE_CUSTOM,
'text_match': 'OK',
'severity': SEVERITY_ERROR,
'status_label': sl_ok_3,
}
d.add_rule(new_rule)
# import data
tasks.save_raw_data(self.import_file.id)
Column.create_mappings(self.fake_mappings, self.org, self.user,
self.import_file.pk)
tasks.map_data(self.import_file.id)
tasks.match_buildings(self.import_file.id)
qs = PropertyState.objects.filter(
import_file=self.import_file,
source_type=ASSESSED_BS,
).iterator()
d.reset_results()
d.check_data('PropertyState', qs)
# Check multiple strings
props = PropertyView.objects.filter(
property__labels=sl_ok_1).select_related('state')
addresses = sorted([p.state.address_line_1 for p in props])
expected = [
u'1 International Road', u'17246 Esch Drive',
u'2581 Schiller Parkway', u'3 Northport Place',
u'84807 Buell Trail'
]
self.assertListEqual(expected, addresses)
props = PropertyView.objects.filter(
property__labels=sl_ok_2).select_related('state')
addresses = sorted([p.state.address_line_1 for p in props])
expected = [
u'1 International Road', u'2581 Schiller Parkway',
u'49705 Harper Crossing'
]
self.assertListEqual(expected, addresses)
props = PropertyView.objects.filter(
property__labels=sl_ok_3).select_related('state')
addresses = sorted([p.state.address_line_1 for p in props])
expected = [
u'1 International Road', u'17246 Esch Drive', u'84807 Buell Trail',
u'88263 Scoville Park'
]
self.assertListEqual(expected, addresses)
def test_check(self):
# data quality check
d = DataQualityCheck.retrieve(self.org)
d.remove_all_rules()
d.add_rule({
'table_name': 'PropertyState',
'field': 'gross_floor_area',
'data_type': TYPE_NUMBER,
'rule_type': RULE_TYPE_DEFAULT,
'min': 100,
'max': 7000000,
'severity': SEVERITY_ERROR,
'units': 'square feet',
})
d.add_rule({
'table_name': 'PropertyState',
'field': 'recent_sale_date',
'data_type': TYPE_DATE,
'rule_type': RULE_TYPE_DEFAULT,
'min': 18890101,
'max': 20201231,
'severity': SEVERITY_ERROR,
})
# create some status labels for testing
sl_data = {
'name': 'year - old or future',
'super_organization': self.org
}
sl_year, _ = StatusLabel.objects.get_or_create(**sl_data)
new_rule = {
'table_name': 'PropertyState',
'field': 'year_built',
'data_type': TYPE_YEAR,
'rule_type': RULE_TYPE_DEFAULT,
'min': 1700,
'max': 2019,
'severity': SEVERITY_ERROR,
'status_label': sl_year,
}
d.add_rule(new_rule)
sl_data = {
'name': 'extra data ps float error',
'super_organization': self.org
}
sl_string, _ = StatusLabel.objects.get_or_create(**sl_data)
new_rule = {
'table_name': 'PropertyState',
'field': 'extra_data_ps_alpha',
'data_type': TYPE_STRING,
'rule_type': RULE_TYPE_CUSTOM,
'text_match': 'alpha',
'severity': SEVERITY_ERROR,
'units': 'square feet',
'status_label': sl_string,
}
d.add_rule(new_rule)
sl_data = {
'name': 'extra data ps string error',
'super_organization': self.org
}
sl_float, _ = StatusLabel.objects.get_or_create(**sl_data)
new_rule = {
'table_name': 'PropertyState',
'field': 'extra_data_ps_float',
'data_type': TYPE_NUMBER,
'rule_type': RULE_TYPE_CUSTOM,
'min': 9999,
'max': 10001,
'severity': SEVERITY_ERROR,
'status_label': sl_float,
}
d.add_rule(new_rule)
sl_data = {
'name': 'jurisdiction id does not match',
'super_organization': self.org
}
sl_jurid, _ = StatusLabel.objects.get_or_create(**sl_data)
new_rule = {
'table_name': 'TaxLotState',
'field': 'jurisdiction_tax_lot_id',
'data_type': TYPE_STRING,
'rule_type': RULE_TYPE_CUSTOM,
'text_match': '1235',
'severity': SEVERITY_ERROR,
'status_label': sl_jurid,
}
d.add_rule(new_rule)
sl_data = {'name': 'No meters present', 'super_organization': self.org}
sl_ok_1, _ = StatusLabel.objects.get_or_create(**sl_data)
new_rule = {
'table_name': 'PropertyState',
'field': 'Meters Present',
'data_type': TYPE_STRING,
'rule_type': RULE_TYPE_CUSTOM,
'text_match': 'OK',
'severity': SEVERITY_ERROR,
'status_label': sl_ok_1,
}
d.add_rule(new_rule)
sl_data = {
'name': 'No 12 Consectutive Months',
'super_organization': self.org
}
sl_ok_2, _ = StatusLabel.objects.get_or_create(**sl_data)
new_rule = {
'table_name': 'PropertyState',
'field': '12 Consectutive Months',
'data_type': TYPE_STRING,
'rule_type': RULE_TYPE_CUSTOM,
'text_match': 'OK',
'severity': SEVERITY_ERROR,
'status_label': sl_ok_2,
}
d.add_rule(new_rule)
sl_data = {'name': 'No Monthly Data', 'super_organization': self.org}
sl_ok_3, _ = StatusLabel.objects.get_or_create(**sl_data)
new_rule = {
'table_name': 'PropertyState',
'field': 'Monthly Data',
'data_type': TYPE_STRING,
'rule_type': RULE_TYPE_CUSTOM,
'text_match': 'OK',
'severity': SEVERITY_ERROR,
'status_label': sl_ok_3,
}
d.add_rule(new_rule)
# import data
tasks.save_raw_data(self.import_file.id)
Column.create_mappings(self.fake_mappings, self.org, self.user,
self.import_file.pk)
tasks.map_data(self.import_file.id)
tasks.match_buildings(self.import_file.id)
qs = PropertyState.objects.filter(
import_file=self.import_file,
source_type=ASSESSED_BS,
).iterator()
d.reset_results()
d.check_data('PropertyState', qs)
result = d.retrieve_result_by_address('4 Myrtle Parkway')
res = [{
"severity": "error",
"value": "27.0",
"field": "extra_data_ps_float",
"table_name": "PropertyState",
"message": "Extra Data Ps Float out of range",
"detailed_message": "Extra Data Ps Float [27.0] < 9999.0",
"formatted_field": "Extra Data Ps Float"
}, {
"severity": "error",
"value": "5.0",
"field": "gross_floor_area",
"table_name": "PropertyState",
"message": "Gross Floor Area out of range",
"detailed_message": "Gross Floor Area [5.0] < 100.0",
"formatted_field": "Gross Floor Area"
}]
self.assertListEqual(result['data_quality_results'], res)
result = d.retrieve_result_by_address('94 Oxford Hill')
res = [{
"severity": "error",
"value": "20000.0",
"field": "extra_data_ps_float",
"table_name": "PropertyState",
"message": "Extra Data Ps Float out of range",
"detailed_message": "Extra Data Ps Float [20000.0] > 10001.0",
"formatted_field": "Extra Data Ps Float"
}, {
"severity": "error",
"value": "1888-01-01 08:00:00",
"field": "recent_sale_date",
"table_name": "PropertyState",
"message": "Recent Sale Date out of range",
"detailed_message":
"Recent Sale Date [1888-01-01 08:00:00] < 1889-01-01 00:00:00",
"formatted_field": "Recent Sale Date"
}]
self.assertListEqual(result['data_quality_results'], res)
result = d.retrieve_result_by_address("3 Portage Alley")
res = [{
'severity': u'error',
'value': 'beta',
'field': u'extra_data_ps_alpha',
'table_name': u'PropertyState',
'message': u'Extra Data Ps Alpha does not match expected value',
'detailed_message':
u'Extra Data Ps Alpha [beta] does not contain "alpha"',
'formatted_field': u'Extra Data Ps Alpha'
}]
self.assertListEqual(result['data_quality_results'], res)
# make sure that the label has been applied
props = PropertyView.objects.filter(
property__labels=sl_year).select_related('state')
addresses = sorted([p.state.address_line_1 for p in props])
expected = sorted([u'84807 Buell Trail', u'1 International Road'])
self.assertListEqual(expected, addresses)
props = PropertyView.objects.filter(
property__labels=sl_float).select_related('state')
addresses = sorted([p.state.address_line_1 for p in props])
expected = sorted([u'4 Myrtle Parkway', u'94 Oxford Hill'])
self.assertListEqual(expected, addresses)
props = PropertyView.objects.filter(
property__labels=sl_string).select_related('state')
addresses = [p.state.address_line_1 for p in props]
expected = [u'3 Portage Alley']
self.assertListEqual(expected, addresses)
# Check tax lots
qs = TaxLotState.objects.filter(
import_file=self.import_file, ).iterator()
d.reset_results()
d.check_data('TaxLotState', qs)
result = d.retrieve_result_by_tax_lot_id("1234")
res = [{
"severity": "error",
"value": "1234",
"field": "jurisdiction_tax_lot_id",
"table_name": "TaxLotState",
"message": "Jurisdiction Tax Lot ID does not match expected value",
"detailed_message":
"Jurisdiction Tax Lot ID [1234] does not contain \"1235\"",
"formatted_field": "Jurisdiction Tax Lot ID"
}]
self.assertListEqual(result['data_quality_results'], res)
# verify labels
taxlots = TaxLotView.objects.filter(
taxlot__labels=sl_jurid).select_related('state')
ids = [t.state.jurisdiction_tax_lot_id for t in taxlots]
expected = '1234'
self.assertEqual(expected, ids[0])
# Check multiple strings
props = PropertyView.objects.filter(
property__labels=sl_ok_1).select_related('state')
addresses = [p.state.address_line_1 for p in props]