def test_download_csv_file(self):
# upload the test CSV to get some data in the DB
self.upload_csv()
# Add one with null values
rc_with_nones = RegistrationCenterFactory(name="Center with no center_lat or center_lon",
center_lat=None,
center_lon=None)
self.assertEqual(rc_with_nones.center_lat, None)
self.assertEqual(rc_with_nones.center_lon, None)
# download the CSV file
rsp = self.client.get(self.download_csv_url)
self.assertEqual(200, rsp.status_code)
reader = UnicodeReader(StringIO(rsp.content))
for i, field in enumerate(reader.next()):
# check the header row
self.assertEqual(field, CSV_FIELDS[i])
for row in reader:
# check each row against the DB values
self.assertNotIn('None', unicode(row))
center_id = row[0]
center = RegistrationCenter.objects.get(center_id=center_id)
for i, field in enumerate(CSV_FIELDS):
# center_type is special because it is an integer in the DB, but a string in the CSV
if field == 'center_type':
db_field_as_str = center.get_center_type_display()
else:
db_field_as_str = unicode(getattr(center, field))
if db_field_as_str == 'None':
db_field_as_str = ''
self.assertEqual(row[i], db_field_as_str)
def test_download_csv_file(self):
# upload the test CSV to get some data in the DB
self.upload_csv()
# Add one with null values
rc_with_nones = RegistrationCenterFactory(
name="Center with no center_lat or center_lon",
center_lat=None,
center_lon=None)
self.assertEqual(rc_with_nones.center_lat, None)
self.assertEqual(rc_with_nones.center_lon, None)
# download the CSV file
rsp = self.client.get(self.download_csv_url)
self.assertEqual(200, rsp.status_code)
reader = UnicodeReader(StringIO(rsp.content))
for i, field in enumerate(reader.next()):
# check the header row
self.assertEqual(field, CSV_FIELDS[i])
for row in reader:
# check each row against the DB values
self.assertNotIn('None', unicode(row))
center_id = row[0]
center = RegistrationCenter.objects.get(center_id=center_id)
for i, field in enumerate(CSV_FIELDS):
# center_type is special because it is an integer in the DB, but a string in the CSV
if field == 'center_type':
db_field_as_str = center.get_center_type_display()
else:
db_field_as_str = unicode(getattr(center, field))
if db_field_as_str == 'None':
db_field_as_str = ''
self.assertEqual(row[i], db_field_as_str)
def update_center_table(_file):
"""
Import voting centers from a CSV file. It creates or updates.
Safe to run repeatedly; if a voting center already exists with the
center ID being imported it will update it if needed.
Returns a 2-tuple of (message, successful), where message is status information (including
errors, if any) and successful is a Boolean.
If any errors are reported, no imports occur.
"""
errors = []
reader = UnicodeReader(_file)
stats = {
'num_blank': 0,
'num_created': 0,
'num_dupes': 0,
'num_updated': 0,
}
line_number = 1
columns = ", ".join(CSV_FIELDS)
headers = reader.next() # gets rid of the header row
if not len(headers) == len(CSV_FIELDS):
return PARSING_ERROR.format(line_number=1, columns=columns), False
for index, header in enumerate(headers):
if not header == CSV_FIELDS[index]:
return PARSING_ERROR.format(line_number=1, columns=columns), False
# If errors happen during the import and we want Django to roll
# back the transaction, we need to exit the transaction context
# with an exception (eventually).
try:
with transaction.atomic():
for row in reader:
line_number += 1
import_center_csv_row(columns, row, line_number, stats, errors)
if errors:
errors.insert(0, force_text(ERRORS_OCCURRED_MESSAGE))
message = mark_safe('<br><br>'.join(errors))
logger.debug(errors)
# trigger rollback:
raise CenterImportFailedError
else:
message = STATUS_MESSAGE.format(blank=stats['num_blank'],
created=stats['num_created'],
dupes=stats['num_dupes'],
updated=stats['num_updated'])
except CenterImportFailedError:
# Just to trigger a rollback
logger.debug("Rolled back all imported centers due to errors.")
else:
logger.debug("No errors during import, will commit changes if nothing else goes wrong "
"during the request.")
return message, not bool(errors)
def validate_uploaded_file(file_path):
"""
Parse the file to see if it's a valid bulk sms message upload file.
If not, raises ValidationError
"""
with open(file_path, "rb") as f:
reader = UnicodeReader(f)
line_number = 0
for row in reader:
line_number += 1
if any(row):
try:
line = Line._make(row)
except TypeError:
raise ValidationError(
PARSING_ERROR.format(line_number=line_number,
columns=", ".join(FIELDS)))
if not is_phone_number_valid(line.number):
raise ValidationError(
INVALID_PHONE_ERROR.format(number=line.number,
line_number=line_number))
if not line.message.strip():
raise ValidationError(
BLANK_MESSAGE_ERROR.format(line_number=line_number))
def _request_csv(self, url, **extra):
""" Like _request() above, but also parse the response body as a CSV
as created by the voter registration dashboard and log in parsed form.
This adds the query arg which specifies CSV rendering.
"""
url += '?format=csv'
rsp = self._request(url, **extra)
content = rsp.content[2:] # skip BOM
reader = UnicodeReader(StringIO(content),
encoding="utf-16-le",
delimiter='\t')
rows = []
for row in reader:
rows.append(row)
logger.info(row)
return rows
def read_messages_from_file(file_path):
"""
Read uploaded bulk SMS file.
Generate tuples: (phone_number, message, from_shortcode).
Delete file afterward.
:param file_path:
:return:
"""
# We don't currently enable customization of the from_shortcode via file upload.
# Just use the default.
from_shortcode = None
with open(file_path, "rb") as f:
reader = UnicodeReader(f)
for row in reader:
if any(row):
line = Line._make(row)
number = int(line.number)
yield number, line.message, from_shortcode
os.remove(file_path)