def import_action(self, request, *args, **kwargs):
'''
Perform a dry_run of the import to make sure the import will not
result in errors. If there where no error, save the the user
uploaded file to a local temp file that will be used by
'process_import' for the actual import.
'''
resource = self.get_resource_class()()
context = {}
import_formats = self.get_import_formats()
form = ImportForm(import_formats,
request.POST or None,
request.FILES or None)
if request.POST and form.is_valid():
input_format = import_formats[
int(form.cleaned_data['input_format'])
]()
import_file = form.cleaned_data['import_file']
# first always write the uploaded file to disk as it may be a
# memory file or else based on settings upload handlers
with tempfile.NamedTemporaryFile(delete=False) as uploaded_file:
for chunk in import_file.chunks():
uploaded_file.write(chunk)
# then read the file, using the proper format-specific mode
with open(uploaded_file.name,
input_format.get_read_mode()) as uploaded_import_file:
# warning, big files may exceed memory
data = uploaded_import_file.read()
data = data.replace(';',',')
if not input_format.is_binary() and self.from_encoding:
try:
data = unicode(data, self.from_encoding).encode('utf-8')
except:
typeManager = TypeManager()
data = typeManager.force_unicode(data, strings_only=True).encode('utf-8')
try:
dataset = input_format.create_dataset(data)
except:
pass
result = resource.import_data(dataset, dry_run=True,
raise_errors=False)
context['result'] = result
if not result.has_errors():
context['confirm_form'] = ConfirmImportForm(initial={
'import_file_name': uploaded_file.name,
'input_format': form.cleaned_data['input_format'],
})
context['form'] = form
context['opts'] = self.model._meta
context['fields'] = [f.column_name for f in resource.get_fields()]
return TemplateResponse(request, [self.import_template_name],
context, current_app=self.admin_site.name)
def get(self, *args, **kwargs):
'''
Perform a dry_run of the import to make sure the import will not
result in errors. If there where no error, save the user
uploaded file to a local temp file that will be used by
'process_import' for the actual import.
'''
resource = self.get_import_resource_class()()
context = {}
import_formats = self.get_import_formats()
form = ImportForm(import_formats, self.request.POST or None,
self.request.FILES or None)
if self.request.POST and form.is_valid():
input_format = import_formats[int(
form.cleaned_data['input_format'])]()
import_file = form.cleaned_data['import_file']
# first always write the uploaded file to disk as it may be a
# memory file or else based on settings upload handlers
with tempfile.NamedTemporaryFile(delete=False) as uploaded_file:
for chunk in import_file.chunks():
uploaded_file.write(chunk)
# then read the file, using the proper format-specific mode
with open(uploaded_file.name,
input_format.get_read_mode()) as uploaded_import_file:
# warning, big files may exceed memory
data = uploaded_import_file.read()
if not input_format.is_binary() and self.from_encoding:
data = force_text(data, self.from_encoding)
dataset = input_format.create_dataset(data)
result = resource.import_data(dataset,
dry_run=True,
raise_errors=True)
context['result'] = result
if not result.has_errors():
context['confirm_form'] = ConfirmImportForm(
initial={
'import_file_name': os.path.basename(
uploaded_file.name),
'original_file_name': uploaded_file.name,
'input_format': form.cleaned_data['input_format'],
})
else:
print("RESULT: {}".format(result))
context['form'] = form
context['opts'] = self.model._meta
context['fields'] = [f.column_name for f in resource.get_fields()]
context.update(self.kwargs)
return TemplateResponse(self.request, [self.import_template_name],
context)
def __init__(self, import_formats, archive_formats, *args, **kwargs):
ImportForm.__init__(self, import_formats, *args, **kwargs)
archive_choices = []
for i, f in enumerate(archive_formats):
archive_choices.append((
str(i),
f().get_title(),
))
if len(archive_choices) > 1:
archive_choices.insert(0, ('', '---'))
self.fields['archive_format'].choices = archive_choices
def _import_action(self, request, *args, **kwargs):
"""
Perform a dry_run of the import to make sure the import will not
result in errors. If there where no error, save the user
uploaded file to a local temp file that will be used by
'process_import' for the actual import.
"""
resource = self.get_import_resource_class()(
**self.get_import_resource_kwargs(request, *args, **kwargs))
context = {}
import_formats = self.get_import_formats()
form = ImportForm(import_formats, request.POST or None, request.FILES
or None)
context['title'] = "Import"
context['form'] = form
context['opts'] = self.model._meta
context['fields'] = [
f.column_name for f in resource.get_user_visible_fields()
]
request.current_app = self.admin_site.name
if request.POST and form.is_valid():
model = self.model
temp_file = form.cleaned_data['import_file']
indicators_file = temp_file.temporary_file_path()
import_from_admin.delay(indicators_file, model, request.user)
return redirect('admin:dashboard_' + model._meta.model_name +
'_changelist')
if django.VERSION >= (1, 8, 0):
context.update(self.admin_site.each_context(request))
elif django.VERSION >= (1, 7, 0):
context.update(self.admin_site.each_context())
return TemplateResponse(request, [self.import_template_name], context)
def get(self, request, *args, **kwargs):
if not (self.has_change_permission() and self.has_add_permission()):
raise PermissionDenied
resource = self.get_import_resource_class()(**self.get_import_resource_kwargs(request, *args, **kwargs))
context = super(ImportView, self).get_context()
import_formats = self.get_import_formats()
form = ImportForm(import_formats,
request.POST or None,
request.FILES or None)
context['title'] = _("Import") + ' ' + self.opts.verbose_name
context['form'] = form
context['opts'] = self.model._meta
context['fields'] = [f.column_name for f in resource.get_user_visible_fields()]
request.current_app = self.admin_site.name
return TemplateResponse(request, [self.import_template_name],
context)
def post(self, request, *args, **kwargs):
"""
Perform a dry_run of the import to make sure the import will not
result in errors. If there where no error, save the user
uploaded file to a local temp file that will be used by
'process_import' for the actual import.
"""
if not (self.has_change_permission() and self.has_add_permission()):
raise PermissionDenied
resource = self.get_import_resource_class()(
**self.get_import_resource_kwargs(request, *args, **kwargs))
context = super(ImportView, self).get_context()
import_formats = self.get_import_formats()
form = ImportForm(import_formats, request.POST or None, request.FILES
or None)
if request.POST and form.is_valid():
input_format = import_formats[int(
form.cleaned_data['input_format'])]()
import_file = form.cleaned_data['import_file']
# first always write the uploaded file to disk as it may be a
# memory file or else based on settings upload handlers
tmp_storage = self.get_tmp_storage_class()()
data = bytes()
for chunk in import_file.chunks():
data += chunk
tmp_storage.save(data, input_format.get_read_mode())
# then read the file, using the proper format-specific mode
# warning, big files may exceed memory
try:
data = tmp_storage.read(input_format.get_read_mode())
if not input_format.is_binary() and self.from_encoding:
data = force_text(data, self.from_encoding)
dataset = input_format.create_dataset(data)
except UnicodeDecodeError as e:
return HttpResponse(
_(u"<h1>Imported file has a wrong encoding: %s</h1>" % e))
except Exception as e:
return HttpResponse(
_(u"<h1>%s encountered while trying to read file: %s</h1>"
% (type(e).__name__, import_file.name)))
result = resource.import_data(dataset,
dry_run=True,
raise_errors=False,
file_name=import_file.name,
user=request.user)
context['result'] = result
if not result.has_errors():
context['confirm_form'] = ConfirmImportForm(
initial={
'import_file_name': tmp_storage.name,
'original_file_name': import_file.name,
'input_format': form.cleaned_data['input_format'],
})
context['title'] = _("Import") + ' ' + self.opts.verbose_name
context['form'] = form
context['opts'] = self.model._meta
context['fields'] = [
f.column_name for f in resource.get_user_visible_fields()
]
request.current_app = self.admin_site.name
return TemplateResponse(request, [self.import_template_name], context)
def get_or_post(self, *args, **kwargs):
"""
Perform a dry_run of the import to make sure the import will not
result in errors. If there where no error, save the user
uploaded file to a local temp file that will be used by
'process_import' for the actual import.
"""
context = self.get_context_data(**kwargs)
import_formats = self.get_import_formats()
form_kwargs = kwargs
form = ImportForm(import_formats, self.request.POST or None,
self.request.FILES or None, **form_kwargs)
if self.request.POST and form.is_valid():
input_format = import_formats[int(
form.cleaned_data['input_format'])]()
import_file = form.cleaned_data['import_file']
# first always write the uploaded file to disk as it may be a
# memory file or else based on settings upload handlers
with tempfile.NamedTemporaryFile(delete=False) as uploaded_file:
for chunk in import_file.chunks():
uploaded_file.write(chunk)
# then read the file, using the proper format-specific mode
with open(uploaded_file.name,
input_format.get_read_mode()) as uploaded_import_file:
# warning, big files may exceed memory
data = uploaded_import_file.read()
if not input_format.is_binary() and self.from_encoding:
data = force_text(data, self.from_encoding)
dataset = input_format.create_dataset(data)
# logger.info(dataset)
resource = self.get_import_resource_class()()
imp_kwargs = self.get_import_data_kwargs(self.request,
form=form,
*args,
**kwargs)
result = resource.import_data(dataset,
dry_run=True,
raise_errors=False,
user=self.request.user,
request=self.request,
**imp_kwargs)
# logger.info(result.__dict__)
# for row in result.rows:
# logger.info(row.__dict__)
# for row in result.invalid_rows:
# logger.info(row.__dict__)
context['result'] = result
if not result.has_errors() and not result.has_validation_errors():
initial = {
'import_file_name': os.path.basename(uploaded_file.name),
'original_file_name': import_file.name,
'input_format': form.cleaned_data['input_format'],
}
context['confirm_form'] = ConfirmImportForm(initial=initial)
else:
resource = self.get_import_resource_class()()
# set up context
context['title'] = _("Import")
context['form'] = form
context['opts'] = self.model._meta
context['fields'] = [f.column_name for f in resource.get_fields()]
context['process_import_url'] = self.process_import_url
context['import_breadcrumbs'] = self.import_breadcrumbs
return super().render_to_response(context)
def importer(request, resource, template):
'''
Perform a dry_run of the import to make sure the import will not
result in errors. If there where no error, save the user
uploaded file to a local temp file that will be used by
'process_import' for the actual import.
'''
from_encoding = "utf-8"
context = {}
import_formats = [f for f in DEFAULT_FORMATS if f().can_import()]
form = ImportForm(import_formats, request.POST or None, request.FILES
or None)
if request.POST and form.is_valid():
input_format = import_formats[int(form.cleaned_data['input_format'])]()
import_file = form.cleaned_data['import_file']
tmp_storage = TempFolderStorage()
try:
#save
data = bytes()
for chunk in import_file.chunks():
data += chunk
tmp_storage.save(data, input_format.get_read_mode())
#read
data = tmp_storage.read(input_format.get_read_mode())
#decoding
if not input_format.is_binary() and from_encoding:
data = force_text(data, from_encoding)
#dataset
dataset = input_format.create_dataset(data)
except UnicodeDecodeError as e:
return HttpResponse(
_(u"<h1>Imported file has a wrong encoding: %s</h1>" % e))
except Exception as e:
return HttpResponse(
_(u"<h1>%s encountered while trying to load or the read file: %s</h1>"
% (type(e).__name__, import_file.name)))
print(dataset)
result = resource.import_data(dataset,
dry_run=True,
raise_errors=False,
file_name=import_file.name,
user=request.user)
context['result'] = result
if not result.has_errors():
context['confirm_form'] = ConfirmImportForm(
initial={
'import_file_name': tmp_storage.name,
'original_file_name': import_file.name,
'input_format': form.cleaned_data['input_format']
})
context['form'] = form
context['fields'] = [
f.column_name for f in resource.get_user_visible_fields()
]
return TemplateResponse(request, [template], context)
def post(self, request, *args, **kwargs):
"""
Perform a dry_run of the import to make sure the import will not
result in errors. If there where no error, save the user
uploaded file to a local temp file that will be used by
'process_import' for the actual import.
"""
if not (self.has_change_permission() and self.has_add_permission()):
raise PermissionDenied
resource = self.get_import_resource_class()(**self.get_import_resource_kwargs(request, *args, **kwargs))
context = super(ImportView, self).get_context()
import_formats = self.get_import_formats()
form = ImportForm(import_formats,
request.POST or None,
request.FILES or None)
if request.POST and form.is_valid():
input_format = import_formats[
int(form.cleaned_data['input_format'])
]()
import_file = form.cleaned_data['import_file']
# first always write the uploaded file to disk as it may be a
# memory file or else based on settings upload handlers
tmp_storage = self.get_tmp_storage_class()()
data = bytes()
for chunk in import_file.chunks():
data += chunk
tmp_storage.save(data, input_format.get_read_mode())
# then read the file, using the proper format-specific mode
# warning, big files may exceed memory
try:
data = tmp_storage.read(input_format.get_read_mode())
if not input_format.is_binary() and self.from_encoding:
data = force_text(data, self.from_encoding)
dataset = input_format.create_dataset(data)
except UnicodeDecodeError as e:
return HttpResponse(_(u"<h1>Imported file has a wrong encoding: %s</h1>" % e))
except Exception as e:
return HttpResponse(_(u"<h1>%s encountered while trying to read file: %s</h1>" % (type(e).__name__,
import_file.name)))
result = resource.import_data(dataset, dry_run=True,
raise_errors=False,
file_name=import_file.name,
user=request.user)
context['result'] = result
if not result.has_errors():
context['confirm_form'] = ConfirmImportForm(initial={
'import_file_name': tmp_storage.name,
'original_file_name': import_file.name,
'input_format': form.cleaned_data['input_format'],
})
context['title'] = _("Import") + ' ' + self.opts.verbose_name
context['form'] = form
context['opts'] = self.model._meta
context['fields'] = [f.column_name for f in resource.get_user_visible_fields()]
request.current_app = self.admin_site.name
return TemplateResponse(request, [self.import_template_name],
context)
