def __init__(self):
super(Command, self).__init__()
self.ero = ElectoralRegistrationOffice.objects.filter(
name='Dundee').first()
if not self.ero:
self.ero = ElectoralRegistrationOffice(**self.ero_details)
self.ero.save()
class Command(BaseCommand):
help = 'Fills up the DB with elector data'
ero_details = {'name': 'Glasgow', 'short_name': 'glasgow', 'address_1': '', 'address_2': '',
'address_3': '', 'postcode': ''}
def __init__(self):
super(Command, self).__init__()
self.ero = ElectoralRegistrationOffice.objects.filter(name='Glasgow').first()
if not self.ero:
self.ero = ElectoralRegistrationOffice(**self.ero_details)
self.ero.save()
def add_arguments(self, parser):
parser.add_argument('filename', nargs=1, type=unicode)
def handle(self, *args, **options):
filename = options['filename'][0]
with open(filename) as myfile:
print("Reading electoral data...")
reader = csv.DictReader(myfile)
data = [preprocess_dict(x) for x in reader if x['PCODE']]
data.sort(key=groupby_key)
print("done - %d records read" % len(data))
records_done = 0
temp_list = []
for grouper, my_group in groupby(data, key=groupby_key):
my_group = list(my_group)
domecile_dict = split_dict(my_group[0], domecile_elements)
domecile_dict['electoral_registration_office'] = self.ero
domecile_obj, result = Domecile.objects.get_or_create(**domecile_dict)
for line in my_group:
contact = split_dict(line, contact_elements)
contact_obj = Contact.objects.filter(ero_number=contact['ero_number'], pd=contact['pd'],
domecile__electoral_registration_office=self.ero).first()
records_done += 1
if not contact_obj:
contact_obj = Contact(**contact)
contact_obj.domecile = domecile_obj
temp_list.append(contact_obj)
if records_done % 1000 == 0:
print("%d records done - last one %s, %s" % (records_done, contact_obj, domecile_obj))
Contact.objects.bulk_create(temp_list)
temp_list = []
if temp_list:
Contact.objects.bulk_create(temp_list)
class Command(BaseCommand):
help = 'Fills up the DB with elector data'
ero_details = {
'name': 'Dundee',
'short_name': 'dundee',
'address_1': '18 City Square',
'address_2': '',
'address_3': 'Dundee',
'postcode': 'DD1 9XE'
}
def __init__(self):
super(Command, self).__init__()
self.ero = ElectoralRegistrationOffice.objects.filter(
name='Dundee').first()
if not self.ero:
self.ero = ElectoralRegistrationOffice(**self.ero_details)
self.ero.save()
def add_arguments(self, parser):
parser.add_argument('filename', nargs=1, type=unicode)
def handle(self, *args, **options):
filename = options['filename'][0]
with open(filename) as myfile:
print("Reading electoral data...")
reader = csv.DictReader(myfile)
data = [transform_dict(x, rename_dict) for x in reader]
data.sort(key=groupby_key)
print("done - %d records read" % len(data))
records_done = 0
temp_list, error_list = [], []
for grouper, my_group in groupby(data, key=groupby_key):
my_group = list(my_group)
domecile_dict = split_dict(my_group[0], domecile_elements)
domecile_dict['electoral_registration_office'] = self.ero
domecile_obj, result = Domecile.objects.get_or_create(
**domecile_dict)
for line in my_group:
contact_dict = split_dict(line, contact_elements)
if contact_dict['date_of_attainment']:
temp = [
int(x)
for x in contact_dict['date_of_attainment'].split('/')
]
contact_dict['date_of_attainment'] = date(
temp[2], temp[1], temp[0])
else:
contact_dict['date_of_attainment'] = None
contact_obj = Contact.objects.filter(
ero_number=contact_dict['ero_number'],
domecile__electoral_registration_office=self.ero,
pd=contact_dict['pd']).first()
records_done += 1
if not contact_obj:
contact_obj = Contact(**contact_dict)
contact_obj.domecile = domecile_obj
temp_list.append(contact_obj)
if records_done % 5 == 0:
try:
print("%d records done - last one %s, %s" %
(records_done, contact_obj, domecile_obj))
except:
pass
try:
Contact.objects.bulk_create(temp_list)
except:
error_list += temp_list
temp_list = []
if temp_list:
try:
Contact.objects.bulk_create(temp_list)
except:
error_list += temp_list
if error_list:
for i in error_list:
try:
i.save()
except:
try:
print(i)
except:
pass
class Command(BaseCommand):
help = 'Fills up the DB with elector data'
ero_details = {
'name': 'North Lanarkshire',
'short_name': 'n_lanarkshire',
'address_1': 'Lanarkshire Valuation Joint Board',
'address_2': 'North Stand, Cadzow Avenue',
'address_3': 'Hamilton',
'postcode': 'ML3 0LU'
}
def __init__(self):
super(Command, self).__init__()
self.ero = ElectoralRegistrationOffice.objects.filter(
name='North Lanarkshire').first()
if not self.ero:
self.ero = ElectoralRegistrationOffice(**self.ero_details)
self.ero.save()
def add_arguments(self, parser):
parser.add_argument('filename', nargs=1, type=unicode)
def handle(self, *args, **options):
filename = options['filename'][0]
with open(filename) as myfile:
print("Reading electoral data...")
reader = csv.DictReader(myfile)
data = [transform_dict(x, rename_dict) for x in reader]
for line in data:
for i in range(1, 8):
index = 'address_' + str(i)
if not line[index]:
try:
line['postcode'] = line['address_' + str(i - 1)]
line['address_' + str(i - 1)] = ''
break
except KeyError:
line['postcode'] = ''
break
finally:
if line['postcode'] == 'OTHER ELECTORS':
line['postcode'] = ''
data.sort(key=groupby_key)
print("done - %d records read" % len(data))
records_done = 0
temp_list = []
for grouper, my_group in groupby(data, key=groupby_key):
my_group = list(my_group)
domecile_dict = split_dict(my_group[0], domecile_elements)
domecile_dict['electoral_registration_office'] = self.ero
try:
domecile_obj, result = Domecile.objects.get_or_create(
**domecile_dict)
except DataError:
print(domecile_dict)
raise
for line in my_group:
contact_dict = split_dict(line, contact_elements)
if contact_dict['date_of_attainment']:
temp = [
int(x)
for x in contact_dict['date_of_attainment'].split('/')
]
contact_dict['date_of_attainment'] = date(
temp[2], temp[1], temp[0])
else:
contact_dict['date_of_attainment'] = None
contact_obj = Contact.objects.filter(
ero_number=contact_dict['ero_number'],
domecile__electoral_registration_office=self.ero,
pd=contact_dict['pd']).first()
records_done += 1
if not contact_obj:
contact_obj = Contact(**contact_dict)
contact_obj.domecile = domecile_obj
temp_list.append(contact_obj)
if records_done % 1000 == 0:
print("%d records done - last one %s, %s" %
(records_done, contact_obj, domecile_obj))
Contact.objects.bulk_create(temp_list)
temp_list = []
if temp_list:
Contact.objects.bulk_create(temp_list)
print(temp_list)