def test_can_read_from_list(self): lines_to_read = 'a,b\n1,2'.splitlines() expected_data = [ {'a': '1', 'b': '2'} ] names_to_values = list(csv.DictReader(lines_to_read, delimiter=',')) self.assertEqual(expected_data, names_to_values)
def uploaded_file(self): try: bulk_file = self.request.FILES['bulk_upload_file'] if bulk_file.size > self.MAX_SIZE: raise BulkUploadCasesException( _("File size too large. " "Please upload file less than" " {size} Megabytes").format(size=self.MAX_SIZE // self.ONE_MB)) except KeyError: raise BulkUploadCasesException(_("No files uploaded")) try: return WorkbookJSONReader(bulk_file) except InvalidExcelFileException: try: csv.DictReader( io.StringIO(bulk_file.read().decode('utf-8'), newline=None)) raise BulkUploadCasesException( _("CommCare HQ does not support that file type." "Please convert to Excel 2007 or higher (.xlsx) " "and try again.")) except UnicodeDecodeError: raise BulkUploadCasesException(_("Unrecognized format")) except JSONReaderError as e: raise BulkUploadCasesException( _('Your upload was unsuccessful. %s') % e.message)
def test_can_use_specified_fieldnames(self): lines_to_read = '1,2\n3,4' expected_data = [ {'a': '1', 'b': '2'}, {'a': '3', 'b': '4'}, ] with io.StringIO(lines_to_read) as csv_file: names_to_values = list(csv.DictReader(csv_file, delimiter=',', fieldnames=['a', 'b'])) self.assertEqual(expected_data, names_to_values)
def test_can_skip_empty_rows(self): lines_to_read = 'a\n1\n\n2\n' expected_data = [ {'a': '1'}, {'a': '2'}, ] with io.StringIO(lines_to_read) as csv_file: names_to_values = list(csv.DictReader(csv_file, delimiter=',')) self.assertEqual(expected_data, names_to_values)
def test_can_read_using_specific_fieldnames(self): lines_to_read = 'a,b,c\nx,yy,zzz\n,y\n' expected_data = [ {'a': 'x', 'b': 'yy', 'c': 'zzz'}, {'a': '', 'b': 'y', 'c': None}, ] with io.StringIO(lines_to_read) as csv_file: names_to_values = list(csv.DictReader(csv_file, delimiter=',')) self.assertEqual(expected_data, names_to_values)
def uploaded_file(self): try: bulk_file = self.request.FILES['bulk_upload_file'] except KeyError: raise BulkUploadCasesException(_("No files uploaded")) try: return WorkbookJSONReader(bulk_file) except InvalidExcelFileException: try: csv.DictReader(io.StringIO(bulk_file.read().decode('ascii'), newline=None)) raise BulkUploadCasesException(_("CommCare HQ no longer supports CSV upload. " "Please convert to Excel 2007 or higher (.xlsx) " "and try again.")) except UnicodeDecodeError: raise BulkUploadCasesException(_("Unrecognized format")) except JSONReaderError as e: raise BulkUploadCasesException(_('Your upload was unsuccessful. %s') % e.message)
def _load_record_ids_from_file(self, file_path): with open(file_path, 'rU', encoding='utf-8') as csvfile: reader = csv.DictReader(csvfile) for row in reader: self.record_ids.append(row['record_id'])
def nctoshape(filein, out, variable): ''' This fuction convert a netCDF file into a shape file(Point features). Firsly It will generate two csv files called file.csv and file_cleaned.csv respectively. After that the file_cleaned.csv ( purified by all the NAN values) is used to extract the corresponding shapefile rappresenting a variable's values which is the third argument of this fuction. -USE: nctoshape("file_input", "path_output_folder", "variable") ''' ds = xr.open_dataset(filein, decode_times=False) df = ds.to_dataframe() df.to_csv(out + "/" + filein + ".csv") data = pd.read_csv(out + "/" + filein + ".csv") data.dropna().to_csv(out + "/" + filein + "_Cleaned" + ".csv", index=False) filecsv = open(out + "/" + filein + "_Cleaned" + ".csv") listed = [] line = filecsv.readline() for u in line.split(','): listed.append(u) if 'lat' in listed: schema = {'geometry': 'Point', 'properties': {variable: 'float'}} with collection(out + "/" + filein + ".shp", "w", "ESRI Shapefile", schema) as output: with open(out + "/" + filein + "_Cleaned" + ".csv", 'r') as f: reader = csv.DictReader(f) for row in reader: point = Point(float(row['lon']), float(row['lat'])) output.write({ 'properties': { variable: row[variable] }, 'geometry': mapping(point) }) elif 'Lat' in listed: schema = {'geometry': 'Point', 'properties': {variable: 'float'}} with collection(out + "/" + filein + ".shp", "w", "ESRI Shapefile", schema) as output: with open(out + "/" + filein + "_Cleaned" + ".csv", 'r') as f: reader = csv.DictReader(f) for row in reader: point = Point(float(row['Lon']), float(row['Lat'])) output.write({ 'properties': { variable: row[variable] }, 'geometry': mapping(point) }) elif 'Latitude' in listed: schema = {'geometry': 'Point', 'properties': {variable: 'float'}} with collection(out + "/" + filein + ".shp", "w", "ESRI Shapefile", schema) as output: with open(out + "/" + filein + "_Cleaned" + ".csv", 'r') as f: reader = csv.DictReader(f) for row in reader: point = Point(float(row['Longitude']), float(row['Latitude'])) output.write({ 'properties': { variable: row[variable] }, 'geometry': mapping(point) }) elif 'latitude' in listed: schema = {'geometry': 'Point', 'properties': {variable: 'float'}} with collection(out + "/" + filein + ".shp", "w", "ESRI Shapefile", schema) as output: with open(out + "/" + filein + "_Cleaned" + ".csv", 'r') as f: reader = csv.DictReader(f) for row in reader: point = Point(float(row['longitude']), float(row['latitude'])) output.write({ 'properties': { variable: row[variable] }, 'geometry': mapping(point) }) else: print("Need to be added a new text format. Please contact Carmelo!")
def post(self, request, *args, **kwargs): """View's dispatch method automatically calls this""" upload = request.FILES.get('bulk_upload_file') try: self.workbook = WorkbookJSONReader(upload) except InvalidExcelFileException: try: csv.DictReader(io.StringIO(upload.read().decode('ascii'), newline=None)) return HttpResponseBadRequest( "CommCare HQ no longer supports CSV upload. " "Please convert to Excel 2007 or higher (.xlsx) " "and try again." ) except UnicodeDecodeError: return HttpResponseBadRequest("Unrecognized format") except JSONReaderError as e: messages.error(request, 'Your upload was unsuccessful. %s' % e.message) return self.get(request, *args, **kwargs) except HeaderValueError as e: return HttpResponseBadRequest("Upload encountered a data type error: %s" % e.message) try: self.user_specs = self.workbook.get_worksheet(title='users') except WorksheetNotFound: try: self.user_specs = self.workbook.get_worksheet() except WorksheetNotFound: return HttpResponseBadRequest("Workbook has no worksheets") try: self.group_specs = self.workbook.get_worksheet(title='groups') except WorksheetNotFound: self.group_specs = [] try: check_headers(self.user_specs) except UserUploadError as e: messages.error(request, _(e.message)) return HttpResponseRedirect(reverse(UploadCommCareUsers.urlname, args=[self.domain])) # convert to list here because iterator destroys the row once it has # been read the first time self.user_specs = list(self.user_specs) for user_spec in self.user_specs: try: user_spec['username'] = enforce_string_type(user_spec['username']) except StringTypeRequiredError: messages.error( request, _("Error: Expected username to be a Text type for username {0}") .format(user_spec['username']) ) return HttpResponseRedirect(reverse(UploadCommCareUsers.urlname, args=[self.domain])) try: check_existing_usernames(self.user_specs, self.domain) except UserUploadError as e: messages.error(request, _(e.message)) return HttpResponseRedirect(reverse(UploadCommCareUsers.urlname, args=[self.domain])) try: check_duplicate_usernames(self.user_specs) except UserUploadError as e: messages.error(request, _(e.message)) return HttpResponseRedirect(reverse(UploadCommCareUsers.urlname, args=[self.domain])) task_ref = expose_cached_download(payload=None, expiry=1*60*60, file_extension=None) task = bulk_upload_async.delay( self.domain, self.user_specs, list(self.group_specs), ) task_ref.set_task(task) return HttpResponseRedirect( reverse( UserUploadStatusView.urlname, args=[self.domain, task_ref.download_id] ) )
def _get_data_csv(self, params, filters): with open(params['path'], encoding='utf-8') as f: return list(csv.DictReader(f))
def test_can_read_from_empty_list(self): names_to_values = list(csv.DictReader([])) self.assertEqual([], names_to_values)