def test_import_unicode(self): # importing csv UnicodeEncodeError 347 filename = os.path.join(os.path.dirname(__file__), os.path.pardir, 'exports', 'books-unicode.csv') with open(filename, self.format.get_read_mode()) as in_stream: data = force_text(in_stream.read()) base_formats.CSV().create_dataset(data)
def export_cellphones_action(self, request, queryset): file_format = base_formats.CSV() export_data = self.formatter.cellphone_format( (self.get_export_data(file_format, queryset))) response = HttpResponse(export_data, 'text/txt') response['Content-Disposition'] = 'attachment; filename=%s' % ( "cellphones.txt", ) return response return None
def get(self, request, *args, **kwargs): file_format = base_formats.CSV() resource_class = self.get_export_resource_class() resource = resource_class(self.get_export_set(kwargs)) query = resource.get_queryset() data = resource.export(query) response = HttpResponse( file_format.export_data(data), mimetype='application/octet-stream', ) response['Content-Disposition'] = 'attachment; filename=%s' % ( self.get_export_filename(file_format), ) return response
def handle(self, *args, **options): business_membership = BusinessMembershipResource() dry_run = options.get("dry_run") if dry_run: self.stdout.write(self.style.NOTICE("Dry run")) raise_errors = options.get("raise_errors", None) if raise_errors is None: raise_errors = not dry_run import_file_name = options["file_name"][0] input_format = base_formats.CSV() read_mode = input_format.get_read_mode() try: with open(import_file_name, read_mode) as f: imported_data = Dataset().load(f.read()) except (OSError, FileNotFoundError) as e: raise CommandError(str(e)) self.stdout.write( self.style.NOTICE( f"{imported_data.height} business members will be imported.")) result = business_membership.import_data(imported_data, dry_run=dry_run) if result.has_errors(): self.stdout.write(self.style.ERROR("Errors")) for error in result.base_errors: self.stdout.write(error.error, self.style.ERROR) for line, errors in result.row_errors(): for error in errors: self.stdout.write( self.style.ERROR("Line number" + ": " + force_text(line) + " - " + force_text(error.error))) else: self.stdout.write( self.style.SUCCESS( f"All business members({imported_data.height} in total) uploaded successfully!" ))
def form_valid(self, form): choice_field = form.get_choice_field_name() import_set = form.cleaned_data[choice_field] if 'export' in self.request.POST: return HttpResponseRedirect(self.get_export_url(import_set)) input_format = base_formats.CSV() resource = self.get_import_resource_class()(import_set) uploaded_import_file = form.cleaned_data['upload'] data = uploaded_import_file.read() if not input_format.is_binary() and self.from_encoding: data = force_text(data, self.from_encoding) dataset = input_format.create_dataset(data) result = resource.import_data(dataset, raise_errors=False) if result.has_errors(): return self.form_invalid(form, result=result) return super(ImporterBase, self).form_valid(form)
class Command(BaseCommand): help = 'Import Snomed Concepts' csv_format = base_formats.CSV() data_file_path = os.path.join(settings.CONFIG_DIR, 'data/') temp_out_file = data_file_path + '/temp.csv' def truncate_table(self, model): self.stdout.write("trucating table {}...".format(model._meta.db_table)) with connection.cursor() as cursor: cursor.execute("TRUNCATE {} CASCADE".format(model._meta.db_table)) self.stdout.write("trucating table {} Done.".format( model._meta.db_table)) def reset_sequence_serial(self, model): self.stdout.write("reset serail sequence on table {}...".format( model._meta.db_table)) if not issubclass(model, SnomedConcept): with connection.cursor() as cursor: query = "SELECT setval(pg_get_serial_sequence('\"{tablename}\"','id'), coalesce(max(\"id\"), 1), max(\"id\") IS NOT null) FROM \"{tablename}\";".format( tablename=model._meta.db_table) cursor.execute(query) self.stdout.write("reset serail sequence on table {} Done.".format( model._meta.db_table)) def delete_temp_file(self): if os.path.isfile(self.temp_out_file): os.remove(self.temp_out_file) def unzip_file(self, gzip_file_path): gzip_file = gzip.GzipFile(gzip_file_path, 'rb') gzip_datas = gzip_file.read() gzip_file.close() with open(self.temp_out_file, 'wb') as uncompressed_data: uncompressed_file = File(uncompressed_data) uncompressed_file.write(gzip_datas) return self.temp_out_file def import_into_portgres(self, model, input_file): self.stdout.write("importing data into {}...".format( model._meta.db_table)) uncompressed_file = self.unzip_file(input_file) insert_count = model.objects.from_csv(uncompressed_file) self.stdout.write("{} records inserted".format(insert_count)) self.delete_temp_file() self.stdout.write("importing data into {} Done.".format( model._meta.db_table)) def import_data(self, model, input_file): self.truncate_table(model) self.reset_sequence_serial(model) self.import_into_portgres(model, input_file) def add_arguments(self, parser): # Named (optional) arguments parser.add_argument( '--snomed_concepts', action='store_true', dest='snomed_concepts', help= 'Import data from snomed_concepts.csv.gz, snomed_descendants.csv.gz, readcodes.csv.gz files', ) parser.add_argument( '--snomed_descendants', action='store_true', dest='snomed_descendants', help='Import data from snomed_descendants.csv.gz file', ) parser.add_argument( '--readcodes', action='store_true', dest='readcodes', help='Import data from readcodes.csv.gz file', ) def handle(self, *args, **options): if options['snomed_concepts']: self.import_data(SnomedConcept, self.data_file_path + 'snomed_concepts.csv.gz') self.import_data(SnomedDescendant, self.data_file_path + 'snomed_descendants.csv.gz') self.import_data(ReadCode, self.data_file_path + 'readcodes.csv.gz') if options['snomed_descendants']: self.import_data(SnomedDescendant, self.data_file_path + 'snomed_descendants.csv.gz') if options['readcodes']: self.import_data(ReadCode, self.data_file_path + 'readcodes.csv.gz')
def setUp(self): self.format = base_formats.CSV()
def test_binary_format(self): self.assertFalse(base_formats.CSV().is_binary())
def test_binary_format(self): self.assertEqual(base_formats.CSV().is_binary(), not six.PY3)
def setUp(self): self.format = base_formats.CSV() self.dataset = tablib.Dataset(headers=['id', 'username']) self.dataset.append(('1', 'x'))