def test_nested_list_in_excel_renderer(self): data = [ { "key_a": "data", "key_b": "data", "key_b": "39f97a13-4f3f-45a3-a411-970e496526cd", 'boolean_key': False }, { "key_a": "data", "key_b": "data", "key_b": "39f97a13-4f3f-45a3-a411-970e496526cd", 'boolean_key': True }, { "key_a": "data", "key_b": [ { "key_a": "data", "key_b": "data" } ], "key_b": "39f97a13-4f3f-45a3-a411-970e496526cd" } ] _write_excel_file(data)
def test_nested_empty_list_in_excel_renderer(self): data = [{ "key_a": "data", "key_b": "data" }, { "key_a": "data", "key_b": "data" }, { "key_a": "data", "key_b": [] }] _write_excel_file(data)
def test_nested_empty_list_in_excel_renderer(self): data = [ { "key_a": "data", "key_b": "data" }, { "key_a": "data", "key_b": "data" }, { "key_a": "data", "key_b": [] } ] _write_excel_file(data)
def test_nested_list_in_excel_renderer(self): data = [{ "key_a": "data", "key_b": "data", "key_b": "39f97a13-4f3f-45a3-a411-970e496526cd" }, { "key_a": "data", "key_b": "data", "key_b": "39f97a13-4f3f-45a3-a411-970e496526cd" }, { "key_a": "data", "key_b": [{ "key_a": "data", "key_b": "data" }], "key_b": "39f97a13-4f3f-45a3-a411-970e496526cd" }] _write_excel_file(data)
def handle(self, *args, **kwargs): file_path = os.path.join( settings.BASE_DIR, 'data/new_data/setup/00012_services.json' ) errors = [] with open(file_path) as data_file: data = json.load(data_file) records = data[0].get('records') for record in records: facility_code = record.get('code') service_name = record.get('service_name') record_errors = {} record_errors['service'] = service_name record_errors['reasons'] = [] try: facility = Facility.objects.get(code=facility_code) try: service = Service.objects.get(name=service_name) except Service.DoesNotExist: record_errors['reasons'].append( 'The linked service is missing' ) errors.append(record_errors) continue except Facility.DoesNotExist: record_errors['reasons'].append( 'The linked facility is missing' ) errors.append(record_errors) continue print FacilityService.objects.create( facility=facility, service=service, created_by=system_user, updated_by=system_user) excel_file = open('facility_service_errors.xlsx', 'wb+') excel_file.write(_write_excel_file(errors)) excel_file.close()
def test_empty_list(self): data = [] _write_excel_file(data)
def test_write_excel_file(self): mommy.make(County) mommy.make(County) url = reverse('api:common:counties_list') response = self.client.get(url) _write_excel_file(response.data['results'])
def handle(self, *args, **options): # def load_chus(): data = 'data/new_data/mcul/00100_cus.json' data = os.path.join(settings.BASE_DIR, data) data_file = open(data) data = json.loads(data_file.read()) records = data[0].get('records') f_missing = 0 s_missing = 0 found = 0 cus_with_facility_errrors = [] cus_with_status_errors = [] cus_errors_due_date_errors = [] validation_errors = [] missing_facility_errors = [] for record in records: unit_errors = { 'unit_code': record.get('code'), 'unit_name': record.get('name'), 'linked_facility': record.get('facility').get('code'), 'linked_facility_name': record.pop('facility_name', None), 'error_reasons': [], "county": record.pop('county', None), "division": record.pop('division', None), } try: facility = Facility.objects.get( code=record.get('facility').get('code')) chu_status = Status.objects.get( name=record.get('status').get('name')) found += 1 except (Facility.DoesNotExist): print "The facility {} could not be located".format( record.get('facility').get('code')) cus_with_facility_errrors.append(record) unit_errors['error_reasons'].append( 'The chu facility has got errors, and thus was not migrated') missing_facility_errors.append(unit_errors) f_missing += 1 continue except Status.DoesNotExist: s_missing += 1 cus_with_status_errors.append(record) print "The status {} could not be located".format( record.get('status').get('name')) continue # print f_missing, s_missing, found, len(records) non_existing_cus = 0 existin_cus = 0 households_monitored = record.get('households_monitored') if not households_monitored or households_monitored == '' or households_monitored == ' ': households_monitored = 0 ch_data = { "name": record.get("name"), "code": record.get("code"), "facility": facility, "status": chu_status, "households_monitored": households_monitored, "location": record.get('location'), "created_by": system_user, "updated_by": system_user } if record.get('date_established'): ch_data['date_established'] = parse( record.get('date_established')) if record.get('date_operational'): ch_data['date_operational'] = parse( record.get('date_operational')) try: chu = CommunityHealthUnit.objects.get(code=ch_data.get('code')) non_existing_cus += 1 print chu.name existin_cus += 1 print "CHU with code {} already exists".format(record.get('code')) continue except CommunityHealthUnit.DoesNotExist: try: chu = CommunityHealthUnit.objects.create(**ch_data) except ValidationError: cus_errors_due_date_errors.append(record) unit_errors['error_reasons'].append( 'Date date established is greater then date operational' ) validation_errors.append(unit_errors) continue print non_existing_cus, existin_cus # return cu_email = record.get('cu_email') cu_mobile = record.get('cu_mobile') if cu_email: con, created = Contact.objects.get_or_create( contact=cu_email, contact_type=email, created_by=system_user, updated_by=system_user) print CommunityHealthUnitContact.objects.get_or_create( contact=con, health_unit=chu, created_by=system_user, updated_by=system_user) if cu_mobile: con, created = Contact.objects.get_or_create( contact=cu_mobile, contact_type=mobile) print CommunityHealthUnitContact.objects.get_or_create( contact=con, health_unit=chu, created_by=system_user, updated_by=system_user) #chew 1 name = record.get('chew') chew_in_charge = record.get('chew_in_charge') if name: chew_data = { "first_name": name, "created_by": system_user, "updated_by": system_user, "health_unit": chu } print CommunityHealthWorker.objects.get_or_create(**chew_data) #chew 2 if chew_in_charge: chew_data = { "first_name": chew_in_charge, "created_by": system_user, "updated_by": system_user, "health_unit": chu, 'is_incharge': True } print CommunityHealthWorker.objects.get_or_create(**chew_data) with open('cus_with_facility_errrors', 'w+') as error_cus_file: error_cus_file.write(json.dumps(cus_with_facility_errrors)) with open('cus_with_status_errors', 'w+') as error_cus_file: error_cus_file.write(json.dumps(cus_with_status_errors)) with open('cus_errors_due_date_errors', 'w+') as error_cus_file: error_cus_file.write(json.dumps(cus_errors_due_date_errors)) excel_file = open('chu_with_errorneous_facilities.xlsx', 'wb+') excel_file.write(_write_excel_file(missing_facility_errors)) excel_file.close() excel_file = open('chu_with_validation_errors.xlsx', 'wb+') excel_file.write(_write_excel_file(validation_errors)) excel_file.close()
def handle(self, *args, **options): facility_file = 'data/new_data/demo/0018_missed_live_facilities.json' file_path = os.path.join(settings.BASE_DIR, facility_file) with open(file_path) as facility_data: data = json.load(facility_data) records = data[0].get('records') new_records = [] errors = [] for record in records: error_rec = { 'facility_code': record.get('code'), 'facility_name': record.get('name'), "county": record.pop('county', None), "division": record.pop('division', None), 'reasons': [], } if not record: errors.append(error_rec) continue latitude = record.pop('latitude', None) longitude = record.pop('longitude', None) if not latitude: print "latitude missing" error_rec['reasons'].append('Latitude is missing') if not longitude: print "longitude missing" error_rec['reasons'].append('Longitude is missing') if not latitude or not longitude: errors.append(error_rec) continue try: point = Point(x=float(longitude), y=float(latitude)) except ValueError: error_rec['reasons'].append("Wrongly formatted geocodes") errors.append(error_rec) continue try: possible_wards = WardBoundary.objects.filter( mpoly__contains=point) ward = possible_wards[0].area except IndexError: print "bad lat long" error_rec['reasons'].append('Erroneous coordinates ') errors.append(error_rec) continue record['ward'] = {'id': str(ward.id)} # ensure all records are in fields = [ "code", "name", "official_name", "created", "regulatory_body", "facility_type", "owner", "operation_status", "ward", ] has_missing_fields = False for field in fields: if field not in record.keys(): error_rec['reasons'].append( 'field {} is missing'.format(field)) has_missing_fields = True if has_missing_fields: continue new_records.append(record) print record.get('name'), ward.name, # overwrite the facilities file with open(file_path, 'w') as fac_file: fac_data = [{ "model": "facilities.Facility", "unique_fields": ["code"], "records": new_records }] json.dump(fac_data, fac_file, indent=4) # write the erors file with open('new_live_facs_errors.json', 'w+') as fac_error_file: json.dump(errors, fac_error_file, indent=4) try: excel_file = open('live_facilities_errors_file.xlsx', 'wb+') excel_file.write(_write_excel_file(errors)) excel_file.close() except: pass