def test_import_unicode(self):
# importing csv UnicodeEncodeError 347
filename = os.path.join(os.path.dirname(__file__), os.path.pardir,
'exports', 'books-unicode.csv')
with open(filename, self.format.get_read_mode()) as in_stream:
data = force_text(in_stream.read())
base_formats.CSV().create_dataset(data)
def export_cellphones_action(self, request, queryset):
file_format = base_formats.CSV()
export_data = self.formatter.cellphone_format(
(self.get_export_data(file_format, queryset)))
response = HttpResponse(export_data, 'text/txt')
response['Content-Disposition'] = 'attachment; filename=%s' % (
"cellphones.txt", )
return response
return None
def get(self, request, *args, **kwargs):
file_format = base_formats.CSV()
resource_class = self.get_export_resource_class()
resource = resource_class(self.get_export_set(kwargs))
query = resource.get_queryset()
data = resource.export(query)
response = HttpResponse(
file_format.export_data(data),
mimetype='application/octet-stream',
)
response['Content-Disposition'] = 'attachment; filename=%s' % (
self.get_export_filename(file_format), )
return response
def handle(self, *args, **options):
business_membership = BusinessMembershipResource()
dry_run = options.get("dry_run")
if dry_run:
self.stdout.write(self.style.NOTICE("Dry run"))
raise_errors = options.get("raise_errors", None)
if raise_errors is None:
raise_errors = not dry_run
import_file_name = options["file_name"][0]
input_format = base_formats.CSV()
read_mode = input_format.get_read_mode()
try:
with open(import_file_name, read_mode) as f:
imported_data = Dataset().load(f.read())
except (OSError, FileNotFoundError) as e:
raise CommandError(str(e))
self.stdout.write(
self.style.NOTICE(
f"{imported_data.height} business members will be imported."))
result = business_membership.import_data(imported_data,
dry_run=dry_run)
if result.has_errors():
self.stdout.write(self.style.ERROR("Errors"))
for error in result.base_errors:
self.stdout.write(error.error, self.style.ERROR)
for line, errors in result.row_errors():
for error in errors:
self.stdout.write(
self.style.ERROR("Line number" + ": " +
force_text(line) + " - " +
force_text(error.error)))
else:
self.stdout.write(
self.style.SUCCESS(
f"All business members({imported_data.height} in total) uploaded successfully!"
))
def form_valid(self, form):
choice_field = form.get_choice_field_name()
import_set = form.cleaned_data[choice_field]
if 'export' in self.request.POST:
return HttpResponseRedirect(self.get_export_url(import_set))
input_format = base_formats.CSV()
resource = self.get_import_resource_class()(import_set)
uploaded_import_file = form.cleaned_data['upload']
data = uploaded_import_file.read()
if not input_format.is_binary() and self.from_encoding:
data = force_text(data, self.from_encoding)
dataset = input_format.create_dataset(data)
result = resource.import_data(dataset, raise_errors=False)
if result.has_errors():
return self.form_invalid(form, result=result)
return super(ImporterBase, self).form_valid(form)
class Command(BaseCommand):
help = 'Import Snomed Concepts'
csv_format = base_formats.CSV()
data_file_path = os.path.join(settings.CONFIG_DIR, 'data/')
temp_out_file = data_file_path + '/temp.csv'
def truncate_table(self, model):
self.stdout.write("trucating table {}...".format(model._meta.db_table))
with connection.cursor() as cursor:
cursor.execute("TRUNCATE {} CASCADE".format(model._meta.db_table))
self.stdout.write("trucating table {} Done.".format(
model._meta.db_table))
def reset_sequence_serial(self, model):
self.stdout.write("reset serail sequence on table {}...".format(
model._meta.db_table))
if not issubclass(model, SnomedConcept):
with connection.cursor() as cursor:
query = "SELECT setval(pg_get_serial_sequence('\"{tablename}\"','id'), coalesce(max(\"id\"), 1), max(\"id\") IS NOT null) FROM \"{tablename}\";".format(
tablename=model._meta.db_table)
cursor.execute(query)
self.stdout.write("reset serail sequence on table {} Done.".format(
model._meta.db_table))
def delete_temp_file(self):
if os.path.isfile(self.temp_out_file):
os.remove(self.temp_out_file)
def unzip_file(self, gzip_file_path):
gzip_file = gzip.GzipFile(gzip_file_path, 'rb')
gzip_datas = gzip_file.read()
gzip_file.close()
with open(self.temp_out_file, 'wb') as uncompressed_data:
uncompressed_file = File(uncompressed_data)
uncompressed_file.write(gzip_datas)
return self.temp_out_file
def import_into_portgres(self, model, input_file):
self.stdout.write("importing data into {}...".format(
model._meta.db_table))
uncompressed_file = self.unzip_file(input_file)
insert_count = model.objects.from_csv(uncompressed_file)
self.stdout.write("{} records inserted".format(insert_count))
self.delete_temp_file()
self.stdout.write("importing data into {} Done.".format(
model._meta.db_table))
def import_data(self, model, input_file):
self.truncate_table(model)
self.reset_sequence_serial(model)
self.import_into_portgres(model, input_file)
def add_arguments(self, parser):
# Named (optional) arguments
parser.add_argument(
'--snomed_concepts',
action='store_true',
dest='snomed_concepts',
help=
'Import data from snomed_concepts.csv.gz, snomed_descendants.csv.gz, readcodes.csv.gz files',
)
parser.add_argument(
'--snomed_descendants',
action='store_true',
dest='snomed_descendants',
help='Import data from snomed_descendants.csv.gz file',
)
parser.add_argument(
'--readcodes',
action='store_true',
dest='readcodes',
help='Import data from readcodes.csv.gz file',
)
def handle(self, *args, **options):
if options['snomed_concepts']:
self.import_data(SnomedConcept,
self.data_file_path + 'snomed_concepts.csv.gz')
self.import_data(SnomedDescendant,
self.data_file_path + 'snomed_descendants.csv.gz')
self.import_data(ReadCode,
self.data_file_path + 'readcodes.csv.gz')
if options['snomed_descendants']:
self.import_data(SnomedDescendant,
self.data_file_path + 'snomed_descendants.csv.gz')
if options['readcodes']:
self.import_data(ReadCode,
self.data_file_path + 'readcodes.csv.gz')
def setUp(self):
self.format = base_formats.CSV()
def test_binary_format(self):
self.assertFalse(base_formats.CSV().is_binary())
def test_binary_format(self):
self.assertEqual(base_formats.CSV().is_binary(), not six.PY3)
def setUp(self):
self.format = base_formats.CSV()
self.dataset = tablib.Dataset(headers=['id', 'username'])
self.dataset.append(('1', 'x'))
