def test_nested_list_in_excel_renderer(self):
data = [
{
"key_a": "data",
"key_b": "data",
"key_b": "39f97a13-4f3f-45a3-a411-970e496526cd",
'boolean_key': False
},
{
"key_a": "data",
"key_b": "data",
"key_b": "39f97a13-4f3f-45a3-a411-970e496526cd",
'boolean_key': True
},
{
"key_a": "data",
"key_b": [
{
"key_a": "data",
"key_b": "data"
}
],
"key_b": "39f97a13-4f3f-45a3-a411-970e496526cd"
}
]
_write_excel_file(data)
def test_nested_empty_list_in_excel_renderer(self):
data = [{
"key_a": "data",
"key_b": "data"
}, {
"key_a": "data",
"key_b": "data"
}, {
"key_a": "data",
"key_b": []
}]
_write_excel_file(data)
def test_nested_empty_list_in_excel_renderer(self):
data = [
{
"key_a": "data",
"key_b": "data"
},
{
"key_a": "data",
"key_b": "data"
},
{
"key_a": "data",
"key_b": []
}
]
_write_excel_file(data)
def test_nested_list_in_excel_renderer(self):
data = [{
"key_a": "data",
"key_b": "data",
"key_b": "39f97a13-4f3f-45a3-a411-970e496526cd"
}, {
"key_a": "data",
"key_b": "data",
"key_b": "39f97a13-4f3f-45a3-a411-970e496526cd"
}, {
"key_a": "data",
"key_b": [{
"key_a": "data",
"key_b": "data"
}],
"key_b": "39f97a13-4f3f-45a3-a411-970e496526cd"
}]
_write_excel_file(data)
def handle(self, *args, **kwargs):
file_path = os.path.join(
settings.BASE_DIR,
'data/new_data/setup/00012_services.json'
)
errors = []
with open(file_path) as data_file:
data = json.load(data_file)
records = data[0].get('records')
for record in records:
facility_code = record.get('code')
service_name = record.get('service_name')
record_errors = {}
record_errors['service'] = service_name
record_errors['reasons'] = []
try:
facility = Facility.objects.get(code=facility_code)
try:
service = Service.objects.get(name=service_name)
except Service.DoesNotExist:
record_errors['reasons'].append(
'The linked service is missing'
)
errors.append(record_errors)
continue
except Facility.DoesNotExist:
record_errors['reasons'].append(
'The linked facility is missing'
)
errors.append(record_errors)
continue
print FacilityService.objects.create(
facility=facility, service=service,
created_by=system_user, updated_by=system_user)
excel_file = open('facility_service_errors.xlsx', 'wb+')
excel_file.write(_write_excel_file(errors))
excel_file.close()
def test_empty_list(self):
data = []
_write_excel_file(data)
def test_write_excel_file(self):
mommy.make(County)
mommy.make(County)
url = reverse('api:common:counties_list')
response = self.client.get(url)
_write_excel_file(response.data['results'])
def test_empty_list(self):
data = []
_write_excel_file(data)
def test_write_excel_file(self):
mommy.make(County)
mommy.make(County)
url = reverse('api:common:counties_list')
response = self.client.get(url)
_write_excel_file(response.data['results'])
def handle(self, *args, **options):
# def load_chus():
data = 'data/new_data/mcul/00100_cus.json'
data = os.path.join(settings.BASE_DIR, data)
data_file = open(data)
data = json.loads(data_file.read())
records = data[0].get('records')
f_missing = 0
s_missing = 0
found = 0
cus_with_facility_errrors = []
cus_with_status_errors = []
cus_errors_due_date_errors = []
validation_errors = []
missing_facility_errors = []
for record in records:
unit_errors = {
'unit_code': record.get('code'),
'unit_name': record.get('name'),
'linked_facility': record.get('facility').get('code'),
'linked_facility_name': record.pop('facility_name', None),
'error_reasons': [],
"county": record.pop('county', None),
"division": record.pop('division', None),
}
try:
facility = Facility.objects.get(
code=record.get('facility').get('code'))
chu_status = Status.objects.get(
name=record.get('status').get('name'))
found += 1
except (Facility.DoesNotExist):
print "The facility {} could not be located".format(
record.get('facility').get('code'))
cus_with_facility_errrors.append(record)
unit_errors['error_reasons'].append(
'The chu facility has got errors, and thus was not migrated')
missing_facility_errors.append(unit_errors)
f_missing += 1
continue
except Status.DoesNotExist:
s_missing += 1
cus_with_status_errors.append(record)
print "The status {} could not be located".format(
record.get('status').get('name'))
continue
# print f_missing, s_missing, found, len(records)
non_existing_cus = 0
existin_cus = 0
households_monitored = record.get('households_monitored')
if not households_monitored or households_monitored == '' or households_monitored == ' ':
households_monitored = 0
ch_data = {
"name": record.get("name"),
"code": record.get("code"),
"facility": facility,
"status": chu_status,
"households_monitored": households_monitored,
"location": record.get('location'),
"created_by": system_user,
"updated_by": system_user
}
if record.get('date_established'):
ch_data['date_established'] = parse(
record.get('date_established'))
if record.get('date_operational'):
ch_data['date_operational'] = parse(
record.get('date_operational'))
try:
chu = CommunityHealthUnit.objects.get(code=ch_data.get('code'))
non_existing_cus += 1
print chu.name
existin_cus += 1
print "CHU with code {} already exists".format(record.get('code'))
continue
except CommunityHealthUnit.DoesNotExist:
try:
chu = CommunityHealthUnit.objects.create(**ch_data)
except ValidationError:
cus_errors_due_date_errors.append(record)
unit_errors['error_reasons'].append(
'Date date established is greater then date operational'
)
validation_errors.append(unit_errors)
continue
print non_existing_cus, existin_cus
# return
cu_email = record.get('cu_email')
cu_mobile = record.get('cu_mobile')
if cu_email:
con, created = Contact.objects.get_or_create(
contact=cu_email,
contact_type=email, created_by=system_user,
updated_by=system_user)
print CommunityHealthUnitContact.objects.get_or_create(
contact=con, health_unit=chu,
created_by=system_user, updated_by=system_user)
if cu_mobile:
con, created = Contact.objects.get_or_create(
contact=cu_mobile,
contact_type=mobile)
print CommunityHealthUnitContact.objects.get_or_create(
contact=con, health_unit=chu,
created_by=system_user, updated_by=system_user)
#chew 1
name = record.get('chew')
chew_in_charge = record.get('chew_in_charge')
if name:
chew_data = {
"first_name": name,
"created_by": system_user,
"updated_by": system_user,
"health_unit": chu
}
print CommunityHealthWorker.objects.get_or_create(**chew_data)
#chew 2
if chew_in_charge:
chew_data = {
"first_name": chew_in_charge,
"created_by": system_user,
"updated_by": system_user,
"health_unit": chu,
'is_incharge': True
}
print CommunityHealthWorker.objects.get_or_create(**chew_data)
with open('cus_with_facility_errrors', 'w+') as error_cus_file:
error_cus_file.write(json.dumps(cus_with_facility_errrors))
with open('cus_with_status_errors', 'w+') as error_cus_file:
error_cus_file.write(json.dumps(cus_with_status_errors))
with open('cus_errors_due_date_errors', 'w+') as error_cus_file:
error_cus_file.write(json.dumps(cus_errors_due_date_errors))
excel_file = open('chu_with_errorneous_facilities.xlsx', 'wb+')
excel_file.write(_write_excel_file(missing_facility_errors))
excel_file.close()
excel_file = open('chu_with_validation_errors.xlsx', 'wb+')
excel_file.write(_write_excel_file(validation_errors))
excel_file.close()
def handle(self, *args, **options):
facility_file = 'data/new_data/demo/0018_missed_live_facilities.json'
file_path = os.path.join(settings.BASE_DIR, facility_file)
with open(file_path) as facility_data:
data = json.load(facility_data)
records = data[0].get('records')
new_records = []
errors = []
for record in records:
error_rec = {
'facility_code': record.get('code'),
'facility_name': record.get('name'),
"county": record.pop('county', None),
"division": record.pop('division', None),
'reasons': [],
}
if not record:
errors.append(error_rec)
continue
latitude = record.pop('latitude', None)
longitude = record.pop('longitude', None)
if not latitude:
print "latitude missing"
error_rec['reasons'].append('Latitude is missing')
if not longitude:
print "longitude missing"
error_rec['reasons'].append('Longitude is missing')
if not latitude or not longitude:
errors.append(error_rec)
continue
try:
point = Point(x=float(longitude), y=float(latitude))
except ValueError:
error_rec['reasons'].append("Wrongly formatted geocodes")
errors.append(error_rec)
continue
try:
possible_wards = WardBoundary.objects.filter(
mpoly__contains=point)
ward = possible_wards[0].area
except IndexError:
print "bad lat long"
error_rec['reasons'].append('Erroneous coordinates ')
errors.append(error_rec)
continue
record['ward'] = {'id': str(ward.id)}
# ensure all records are in
fields = [
"code",
"name",
"official_name",
"created",
"regulatory_body",
"facility_type",
"owner",
"operation_status",
"ward",
]
has_missing_fields = False
for field in fields:
if field not in record.keys():
error_rec['reasons'].append(
'field {} is missing'.format(field))
has_missing_fields = True
if has_missing_fields:
continue
new_records.append(record)
print record.get('name'), ward.name,
# overwrite the facilities file
with open(file_path, 'w') as fac_file:
fac_data = [{
"model": "facilities.Facility",
"unique_fields": ["code"],
"records": new_records
}]
json.dump(fac_data, fac_file, indent=4)
# write the erors file
with open('new_live_facs_errors.json', 'w+') as fac_error_file:
json.dump(errors, fac_error_file, indent=4)
try:
excel_file = open('live_facilities_errors_file.xlsx', 'wb+')
excel_file.write(_write_excel_file(errors))
excel_file.close()
except:
pass
