def historian_importer(request, resource, dataset, template): context = {} if request.POST: result = resource.import_data(dataset, dry_run=True, raise_errors=False, file_name=tmp_storage.name, user=request.user) context['result'] = result if not result.has_errors(): tmp_storage = TempFolderStorage() input_format = JSON tmp_storage.save( dataset.export('json').encode(), input_format.get_read_mode) context['confirm_form'] = ConfirmImportForm( initial={ 'import_file_name': tmp_storage.name, 'original_file_name': tmp_storage.name, 'input_format': input_format }) context['fields'] = [ f.column_name for f in resource.get_user_visible_fields() ] return TemplateResponse(request, [template], context)
def write_to_tmp_storage(self, import_file, input_format): tmp_storage = TempFolderStorage() data = bytes() for chunk in import_file.chunks(): data += chunk tmp_storage.save(data, input_format.get_read_mode()) return tmp_storage
def test_temp_folder_storage(self): tmp_storage = TempFolderStorage() tmp_storage.save(self.test_string) name = tmp_storage.name tmp_storage = TempFolderStorage(name=name) self.assertEqual(self.test_string.decode('utf-8'), tmp_storage.read()) self.assertTrue(os.path.isfile(tmp_storage.get_full_path())) tmp_storage.remove() self.assertFalse(os.path.isfile(tmp_storage.get_full_path()))
def process_import(self, request): confirm_form = ConfirmImportForm(request.POST) if confirm_form.is_valid(): import_formats = self.formats input_format = import_formats[int( confirm_form.cleaned_data['input_format'])]() tmp_storage = TempFolderStorage( name=confirm_form.cleaned_data['import_file_name']) dataset = self.make_dataset(tmp_storage, input_format) self.process_dataset( dataset, confirm_form.cleaned_data['original_file_name'], request) tmp_storage.remove() return redirect(self.success_url) else: pass # TODO: handle errors
def historian_processor(request, resource, redirect_url): confirm_form = ConfirmImportForm(request.POST) if confirm_form.is_valid(): input_format = JSON tmp_storage = TempFolderStorage( name=confirm_form.cleaned_data['import_file_name']) data = tmp_storage.read(input_format.get_read_mode()) dataset = input_format.create_dataset(data) resource.import_data( dataset, dry_run=False, raise_errors=True, file_name=confirm_form.cleaned_data['original_file_name'], user=request.user) tmp_storage.remove() return HttpResponseRedirect(reverse_lazy(redirect_url))
def processor(request, resource, template, redirect_url_name): from_encoding = "utf-8" confirm_form = ConfirmImportForm(request.POST) if confirm_form.is_valid(): import_formats = [f for f in DEFAULT_FORMATS if f().can_import()] input_format = import_formats[int( confirm_form.cleaned_data['input_format'])]() tmp_storage = TempFolderStorage( name=confirm_form.cleaned_data['import_file_name']) data = tmp_storage.read(input_format.get_read_mode()) if not input_format.is_binary() and from_encoding: data = force_text(data, from_encoding) dataset = input_format.create_dataset(data) resource.import_data( dataset, dry_run=False, raise_errors=True, file_name=confirm_form.cleaned_data['original_file_name'], user=request.user) tmp_storage.remove() return HttpResponseRedirect(reverse_lazy(redirect_url_name))
def test_temp_folder_storage(self): tmp_storage = TempFolderStorage() tmp_storage.save(self.test_string) name = tmp_storage.name tmp_storage = TempFolderStorage(name=name) self.assertEqual(self.test_string.decode(), tmp_storage.read()) self.assertTrue(os.path.isfile(tmp_storage.get_full_path())) tmp_storage.remove() self.assertFalse(os.path.isfile(tmp_storage.get_full_path()))
def importer(request, resource, template): ''' Perform a dry_run of the import to make sure the import will not result in errors. If there where no error, save the user uploaded file to a local temp file that will be used by 'process_import' for the actual import. ''' from_encoding = "utf-8" context = {} import_formats = [f for f in DEFAULT_FORMATS if f().can_import()] form = ImportForm(import_formats, request.POST or None, request.FILES or None) if request.POST and form.is_valid(): input_format = import_formats[int(form.cleaned_data['input_format'])]() import_file = form.cleaned_data['import_file'] tmp_storage = TempFolderStorage() try: #save data = bytes() for chunk in import_file.chunks(): data += chunk tmp_storage.save(data, input_format.get_read_mode()) #read data = tmp_storage.read(input_format.get_read_mode()) #decoding if not input_format.is_binary() and from_encoding: data = force_text(data, from_encoding) #dataset dataset = input_format.create_dataset(data) except UnicodeDecodeError as e: return HttpResponse( _(u"<h1>Imported file has a wrong encoding: %s</h1>" % e)) except Exception as e: return HttpResponse( _(u"<h1>%s encountered while trying to load or the read file: %s</h1>" % (type(e).__name__, import_file.name))) print(dataset) result = resource.import_data(dataset, dry_run=True, raise_errors=False, file_name=import_file.name, user=request.user) context['result'] = result if not result.has_errors(): context['confirm_form'] = ConfirmImportForm( initial={ 'import_file_name': tmp_storage.name, 'original_file_name': import_file.name, 'input_format': form.cleaned_data['input_format'] }) context['form'] = form context['fields'] = [ f.column_name for f in resource.get_user_visible_fields() ] return TemplateResponse(request, [template], context)