def test_can_read_from_list(self):
lines_to_read = 'a,b\n1,2'.splitlines()
expected_data = [
{'a': '1', 'b': '2'}
]
names_to_values = list(csv.DictReader(lines_to_read, delimiter=','))
self.assertEqual(expected_data, names_to_values)
def uploaded_file(self):
try:
bulk_file = self.request.FILES['bulk_upload_file']
if bulk_file.size > self.MAX_SIZE:
raise BulkUploadCasesException(
_("File size too large. "
"Please upload file less than"
" {size} Megabytes").format(size=self.MAX_SIZE //
self.ONE_MB))
except KeyError:
raise BulkUploadCasesException(_("No files uploaded"))
try:
return WorkbookJSONReader(bulk_file)
except InvalidExcelFileException:
try:
csv.DictReader(
io.StringIO(bulk_file.read().decode('utf-8'),
newline=None))
raise BulkUploadCasesException(
_("CommCare HQ does not support that file type."
"Please convert to Excel 2007 or higher (.xlsx) "
"and try again."))
except UnicodeDecodeError:
raise BulkUploadCasesException(_("Unrecognized format"))
except JSONReaderError as e:
raise BulkUploadCasesException(
_('Your upload was unsuccessful. %s') % e.message)
def test_can_use_specified_fieldnames(self):
lines_to_read = '1,2\n3,4'
expected_data = [
{'a': '1', 'b': '2'},
{'a': '3', 'b': '4'},
]
with io.StringIO(lines_to_read) as csv_file:
names_to_values = list(csv.DictReader(csv_file, delimiter=',', fieldnames=['a', 'b']))
self.assertEqual(expected_data, names_to_values)
def test_can_skip_empty_rows(self):
lines_to_read = 'a\n1\n\n2\n'
expected_data = [
{'a': '1'},
{'a': '2'},
]
with io.StringIO(lines_to_read) as csv_file:
names_to_values = list(csv.DictReader(csv_file, delimiter=','))
self.assertEqual(expected_data, names_to_values)
def test_can_read_using_specific_fieldnames(self):
lines_to_read = 'a,b,c\nx,yy,zzz\n,y\n'
expected_data = [
{'a': 'x', 'b': 'yy', 'c': 'zzz'},
{'a': '', 'b': 'y', 'c': None},
]
with io.StringIO(lines_to_read) as csv_file:
names_to_values = list(csv.DictReader(csv_file, delimiter=','))
self.assertEqual(expected_data, names_to_values)
def uploaded_file(self):
try:
bulk_file = self.request.FILES['bulk_upload_file']
except KeyError:
raise BulkUploadCasesException(_("No files uploaded"))
try:
return WorkbookJSONReader(bulk_file)
except InvalidExcelFileException:
try:
csv.DictReader(io.StringIO(bulk_file.read().decode('ascii'),
newline=None))
raise BulkUploadCasesException(_("CommCare HQ no longer supports CSV upload. "
"Please convert to Excel 2007 or higher (.xlsx) "
"and try again."))
except UnicodeDecodeError:
raise BulkUploadCasesException(_("Unrecognized format"))
except JSONReaderError as e:
raise BulkUploadCasesException(_('Your upload was unsuccessful. %s') % e.message)
def _load_record_ids_from_file(self, file_path):
with open(file_path, 'rU', encoding='utf-8') as csvfile:
reader = csv.DictReader(csvfile)
for row in reader:
self.record_ids.append(row['record_id'])
def nctoshape(filein, out, variable):
''' This fuction convert a netCDF file into a shape file(Point features). Firsly It will generate
two csv files called file.csv and file_cleaned.csv respectively. After that
the file_cleaned.csv ( purified by all the NAN values) is used to extract
the corresponding shapefile rappresenting a variable's values which is the
third argument of this fuction.
-USE:
nctoshape("file_input", "path_output_folder", "variable")
'''
ds = xr.open_dataset(filein, decode_times=False)
df = ds.to_dataframe()
df.to_csv(out + "/" + filein + ".csv")
data = pd.read_csv(out + "/" + filein + ".csv")
data.dropna().to_csv(out + "/" + filein + "_Cleaned" + ".csv", index=False)
filecsv = open(out + "/" + filein + "_Cleaned" + ".csv")
listed = []
line = filecsv.readline()
for u in line.split(','):
listed.append(u)
if 'lat' in listed:
schema = {'geometry': 'Point', 'properties': {variable: 'float'}}
with collection(out + "/" + filein + ".shp", "w", "ESRI Shapefile",
schema) as output:
with open(out + "/" + filein + "_Cleaned" + ".csv", 'r') as f:
reader = csv.DictReader(f)
for row in reader:
point = Point(float(row['lon']), float(row['lat']))
output.write({
'properties': {
variable: row[variable]
},
'geometry': mapping(point)
})
elif 'Lat' in listed:
schema = {'geometry': 'Point', 'properties': {variable: 'float'}}
with collection(out + "/" + filein + ".shp", "w", "ESRI Shapefile",
schema) as output:
with open(out + "/" + filein + "_Cleaned" + ".csv", 'r') as f:
reader = csv.DictReader(f)
for row in reader:
point = Point(float(row['Lon']), float(row['Lat']))
output.write({
'properties': {
variable: row[variable]
},
'geometry': mapping(point)
})
elif 'Latitude' in listed:
schema = {'geometry': 'Point', 'properties': {variable: 'float'}}
with collection(out + "/" + filein + ".shp", "w", "ESRI Shapefile",
schema) as output:
with open(out + "/" + filein + "_Cleaned" + ".csv", 'r') as f:
reader = csv.DictReader(f)
for row in reader:
point = Point(float(row['Longitude']),
float(row['Latitude']))
output.write({
'properties': {
variable: row[variable]
},
'geometry': mapping(point)
})
elif 'latitude' in listed:
schema = {'geometry': 'Point', 'properties': {variable: 'float'}}
with collection(out + "/" + filein + ".shp", "w", "ESRI Shapefile",
schema) as output:
with open(out + "/" + filein + "_Cleaned" + ".csv", 'r') as f:
reader = csv.DictReader(f)
for row in reader:
point = Point(float(row['longitude']),
float(row['latitude']))
output.write({
'properties': {
variable: row[variable]
},
'geometry': mapping(point)
})
else:
print("Need to be added a new text format. Please contact Carmelo!")
def post(self, request, *args, **kwargs):
"""View's dispatch method automatically calls this"""
upload = request.FILES.get('bulk_upload_file')
try:
self.workbook = WorkbookJSONReader(upload)
except InvalidExcelFileException:
try:
csv.DictReader(io.StringIO(upload.read().decode('ascii'),
newline=None))
return HttpResponseBadRequest(
"CommCare HQ no longer supports CSV upload. "
"Please convert to Excel 2007 or higher (.xlsx) "
"and try again."
)
except UnicodeDecodeError:
return HttpResponseBadRequest("Unrecognized format")
except JSONReaderError as e:
messages.error(request,
'Your upload was unsuccessful. %s' % e.message)
return self.get(request, *args, **kwargs)
except HeaderValueError as e:
return HttpResponseBadRequest("Upload encountered a data type error: %s"
% e.message)
try:
self.user_specs = self.workbook.get_worksheet(title='users')
except WorksheetNotFound:
try:
self.user_specs = self.workbook.get_worksheet()
except WorksheetNotFound:
return HttpResponseBadRequest("Workbook has no worksheets")
try:
self.group_specs = self.workbook.get_worksheet(title='groups')
except WorksheetNotFound:
self.group_specs = []
try:
check_headers(self.user_specs)
except UserUploadError as e:
messages.error(request, _(e.message))
return HttpResponseRedirect(reverse(UploadCommCareUsers.urlname, args=[self.domain]))
# convert to list here because iterator destroys the row once it has
# been read the first time
self.user_specs = list(self.user_specs)
for user_spec in self.user_specs:
try:
user_spec['username'] = enforce_string_type(user_spec['username'])
except StringTypeRequiredError:
messages.error(
request,
_("Error: Expected username to be a Text type for username {0}")
.format(user_spec['username'])
)
return HttpResponseRedirect(reverse(UploadCommCareUsers.urlname, args=[self.domain]))
try:
check_existing_usernames(self.user_specs, self.domain)
except UserUploadError as e:
messages.error(request, _(e.message))
return HttpResponseRedirect(reverse(UploadCommCareUsers.urlname, args=[self.domain]))
try:
check_duplicate_usernames(self.user_specs)
except UserUploadError as e:
messages.error(request, _(e.message))
return HttpResponseRedirect(reverse(UploadCommCareUsers.urlname, args=[self.domain]))
task_ref = expose_cached_download(payload=None, expiry=1*60*60, file_extension=None)
task = bulk_upload_async.delay(
self.domain,
self.user_specs,
list(self.group_specs),
)
task_ref.set_task(task)
return HttpResponseRedirect(
reverse(
UserUploadStatusView.urlname,
args=[self.domain, task_ref.download_id]
)
)
def _get_data_csv(self, params, filters):
with open(params['path'], encoding='utf-8') as f:
return list(csv.DictReader(f))
def test_can_read_from_empty_list(self):
names_to_values = list(csv.DictReader([]))
self.assertEqual([], names_to_values)
