def test_reading_from_more_models(self): model1 = FakeDjangoModel() model2 = FakeDjangoModel() importer = DjangoModelImporter() adapter1 = DjangoModelImportAdapter(model1) adapter1.column_names = self.content["Sheet1"][0] adapter2 = DjangoModelImportAdapter(model2) adapter2.column_names = self.content["Sheet2"][0] importer.append(adapter1) importer.append(adapter2) to_store = { adapter1.get_name(): self.content["Sheet1"][1:], adapter2.get_name(): self.content["Sheet2"][1:], } save_data(importer, to_store, file_type=DB_DJANGO) assert model1.objects.objs == self.result1 assert model2.objects.objs == self.result2 model1._meta.model_name = "Sheet1" model2._meta.model_name = "Sheet2" model1._meta.update(["X", "Y", "Z"]) model2._meta.update(["A", "B", "C"]) exporter = DjangoModelExporter() adapter1 = DjangoModelExportAdapter(model1) adapter2 = DjangoModelExportAdapter(model2) exporter.append(adapter1) exporter.append(adapter2) reader = DjangoBookReader() reader.open_content(exporter) data = reader.read_all() for key in data.keys(): data[key] = list(data[key]) assert data == self.content
def test_pyexcel_issue_138(): array = [["123_122", "123_1.", "123_1.0"]] save_data("test.csv", array) data = get_data("test.csv") expected = [["123_122", "123_1.", "123_1.0"]] eq_(data["test.csv"], expected) os.unlink("test.csv")
def setUp(self): self.content = OrderedDict() self.content.update({ 'Sheet 1': [ [1.0, 2.0, 3.0], [4.0, 5.0, 6.0], [7.0, 8.0, 9.0] ] }) self.content.update({ 'Sheet 2': [ ['X', 'Y', 'Z'], [1.0, 2.0, 3.0], [4.0, 5.0, 6.0] ] }) self.content.update({ 'Sheet 3': [ ['O', 'P', 'Q'], [3.0, 2.0, 1.0], [4.0, 3.0, 2.0] ] }) self.file="mybook.csvz" save_data(self.file, self.content)
def write_data(self, book): new_models = [model for model in self.models if model is not None] batch_size = self.keywords.get(params.BATCH_SIZE, None) initializers = self.keywords.get(params.INITIALIZERS, None) if initializers is None: initializers = [None] * len(new_models) mapdicts = self.keywords.get(params.MAPDICTS, None) if mapdicts is None: mapdicts = [None] * len(new_models) for sheet in book: if len(sheet.colnames) == 0: sheet.name_columns_by_row(0) colnames_array = [sheet.colnames for sheet in book] scattered = zip(new_models, colnames_array, mapdicts, initializers) importer = DjangoModelImporter() for each_model in scattered: adapter = DjangoModelImportAdapter(each_model[0]) adapter.set_column_names(each_model[1]) adapter.set_column_name_mapping_dict(each_model[2]) adapter.set_row_initializer(each_model[3]) importer.append(adapter) to_store = OrderedDict() for sheet_name in book.sheet_names(): # due book.to_dict() brings in column_names # which corrupts the data to_store[sheet_name] = book[sheet_name].array save_data(importer, to_store, file_type=DB_DJANGO, batch_size=batch_size)
def write_data(self, book): initializers = self.keywords.get(params.INITIALIZERS, None) if initializers is None: initializers = [None] * len(self.tables) mapdicts = self.keywords.get(params.MAPDICTS, None) if mapdicts is None: mapdicts = [None] * len(self.tables) for sheet in book: if len(sheet.colnames) == 0: sheet.name_columns_by_row(0) colnames_array = [sheet.colnames for sheet in book] scattered = zip(self.tables, colnames_array, mapdicts, initializers) importer = SQLTableImporter(self.session) for each_table in scattered: adapter = SQLTableImportAdapter(each_table[0]) adapter.column_names = each_table[1] adapter.column_name_mapping_dict = each_table[2] adapter.row_initializer = each_table[3] importer.append(adapter) to_store = OrderedDict() for sheet_name in book.sheet_names(): # due book.to_dict() brings in column_names # which corrupts the data to_store[sheet_name] = book[sheet_name].array save_data(importer, to_store, file_type=DB_SQL, **self.keywords)
def render_book_to_stream(self, file_stream, book, inits=None, mapdicts=None, **keywords): session, tables = file_stream thebook = book initializers = inits colnames_array = common.get_book_headers_in_array(book) if initializers is None: initializers = [None] * len(tables) if mapdicts is None: mapdicts = [None] * len(tables) scattered = zip(tables, colnames_array, mapdicts, initializers) importer = sql.SQLTableImporter(session) for each_table in scattered: adapter = sql.SQLTableImportAdapter(each_table[0]) adapter.column_names = each_table[1] adapter.column_name_mapping_dict = each_table[2] adapter.row_initializer = each_table[3] importer.append(adapter) to_store = OrderedDict() for sheet in thebook: # due book.to_dict() brings in column_names # which corrupts the data to_store[sheet.name] = sheet.get_internal_array() save_data(importer, to_store, file_type=self._file_type, **keywords)
def edit(): # shutil.copyfile(app.config['EXTRA_STUDENTS_SOURCE_PATH'], app.config['EXTRA_STUDENTS_WORKING_PATH']) records = get_array(file_name=app.config['EXTRA_STUDENTS_WORKING_PATH']) if request.method == 'POST': changed = False removes = [] for key in request.form.keys(): m = re.match('remove_(\d+)', key) if m: removes.append(int(m.group(1))) changed = True records = [r for r in records if not (r[0] in removes)] if request.form['student_number'] and request.form['first_name'] and \ request.form['last_name'] and request.form['email']: records.append([ request.form['student_number'], request.form['first_name'], request.form['last_name'], '104', request.form['gender'], request.form['email'], '9919.1' ]) changed = True if changed: save_data(app.config['EXTRA_STUDENTS_WORKING_PATH'], records, lineterminator='\n') return render_template('students.html', page_title='Edit Students', records=records)
def render_book_to_stream(self, models, book, inits=None, mapdicts=None, batch_size=None, **keywords): colnames_array = common.get_book_headers_in_array(book) new_models = [model for model in models if model is not None] initializers = inits if initializers is None: initializers = [None] * len(new_models) if mapdicts is None: mapdicts = [None] * len(new_models) scattered = zip(new_models, colnames_array, mapdicts, initializers) importer = django.DjangoModelImporter() for each_model in scattered: adapter = django.DjangoModelImportAdapter(each_model[0]) adapter.column_names = each_model[1] adapter.column_name_mapping_dict = each_model[2] adapter.row_initializer = each_model[3] importer.append(adapter) to_store = OrderedDict() for sheet in book: # due book.to_dict() brings in column_names # which corrupts the data to_store[sheet.name] = sheet.get_internal_array() save_data(importer, to_store, file_type=self._file_type, batch_size=batch_size, **keywords)
def test_issue_8(): test_file = "test_issue_8.csv" data = [[1, 2], [], [], [], [3, 4]] save_data(test_file, data) written_data = get_data(test_file, skip_empty_rows=False) eq_(data, written_data[test_file]) os.unlink(test_file)
def setUp(self): self.test_file = "test_filter.csv" sample = [ [1, 21, 31], [2, 22, 32] ] save_data(self.test_file, sample)
def test_file_handle_as_output(): test_file = "file_handle.csv" with open(test_file, 'w') as f: save_data(f, [[1, 2, 3]], 'csv', lineterminator='\n') with open(test_file, 'r') as f: content = f.read() eq_(content, '1,2,3\n')
def test_load_sheet_from_django_model(self): model=FakeDjangoModel() save_data(DB_DJANGO, self.data[1:], models={DEFAULT_SHEET_NAME: [model, self.data[0], None, None]}) assert model.objects.objs == self.result model._meta.update(["X", "Y", "Z"]) reader = DjangoModelReader(model) data = reader.to_array() assert data == self.data
def test_conversion_from_bytes_to_text(): data = [['1','2','3']] save_data("conversion.csv", data) with open("conversion.csv", "rb") as f: content = f.read() result = get_data(content, 'csv') assert result == data os.unlink("conversion.csv")
def render_sheet_to_file(self, file_name, sheet, **keywords): sheet_name = DEFAULT_SHEET_NAME if sheet.name: sheet_name = sheet.name data = {sheet_name: sheet.to_array()} save_data(file_name, data, **keywords)
def test_write_xlsx_data_to_memory(): data = {'Sheet': [[1]]} io = BytesIO() msg = "Please install one of these plugins for write data in 'xlsx': " msg += "pyexcel-xlsx,pyexcel-xlsxw" try: save_data(io, data, file_type="xlsx") except manager.SupportingPluginAvailableButNotInstalled as e: eq_(str(e), msg)
def test_conversion_from_bytes_to_text(): test_file = "conversion.csv" data = [['1','2','3']] save_data(test_file, data) with open(test_file, "rb") as f: content = f.read() result = get_data(content, 'csv') assert result['csv'] == [[1, 2, 3]] os.unlink(test_file)
def render_sheet_to_stream(self, file_stream, sheet, **keywords): sheet_name = DEFAULT_SHEET_NAME if sheet.name: sheet_name = sheet.name data = {sheet_name: sheet.to_array()} save_data(file_stream, data, file_type=self.file_type, **keywords)
def test_case_insentivity(): data = [['1', '2', '3']] io = manager.get_io("CSV") # test default format for saving is 'csv' save_data(io, data) io.seek(0) # test default format for reading is 'csv' result = get_data(io) assert result['csv'] == [[1, 2, 3]]
def test_issue_30_utf8_BOM_header(): content = [[u"人有悲歡離合", u"月有陰晴圓缺"]] test_file = "test-utf8-BOM.csv" save_data(test_file, content, encoding="utf-8-sig", lineterminator="\n") custom_encoded_content = get_data(test_file, encoding="utf-8-sig") assert custom_encoded_content[test_file] == content with open(test_file, "rb") as f: content = f.read() assert content[0:3] == b"\xef\xbb\xbf" os.unlink(test_file)
def test_generator_can_be_written(): test_filename = "generator.csv" test_fixture = os.path.join("tests", "fixtures", "test.csv") data = get_data(test_fixture, streaming=True) save_data(test_filename, data) assert os.path.exists(test_filename) data2 = get_data(test_filename) expected = get_data(test_fixture) assert data2[test_filename] == expected['test.csv'] os.unlink(test_filename)
def setUp(self): self.test_file = "test_filter.csv" sample = [ [1, 21, 31], [2, 22, 32], [3, 23, 33], [4, 24, 34], [5, 25, 35], [6, 26, 36], ] save_data(self.test_file, sample)
def write_data(self, sheet): headers = sheet.colnames if len(headers) == 0: headers = sheet.rownames importer = DjangoModelImporter() adapter = DjangoModelImportAdapter(self.model) adapter.set_column_names(headers) adapter.set_column_name_mapping_dict(self.keywords.get(params.MAPDICT, None)) adapter.set_row_initializer(self.keywords.get(params.INITIALIZER, None)) importer.append(adapter) save_data(importer, {adapter.get_name(): sheet.array}, file_type=DB_DJANGO, **self.keywords)
def write_data(self, sheet): headers = sheet.colnames if len(headers) == 0: headers = sheet.rownames importer = SQLTableImporter(self.session) adapter = SQLTableImportAdapter(self.table) adapter.column_names = headers adapter.row_initializer = self.keywords.get(params.INITIALIZER, None) adapter.column_name_mapping_dict = self.keywords.get(params.MAPDICT, None) importer.append(adapter) save_data(importer, {adapter.get_name(): sheet.array}, file_type=DB_SQL, **self.keywords)
def render_sheet_to_stream(self, file_stream, sheet, init=None, mapdict=None, **keywords): headers = common.get_sheet_headers(sheet) importer = sql.SQLTableImporter(file_stream[0]) adapter = sql.SQLTableImportAdapter(file_stream[1]) adapter.column_names = headers adapter.row_initializer = init adapter.column_name_mapping_dict = mapdict importer.append(adapter) save_data(importer, {adapter.get_name(): sheet.get_internal_array()}, file_type=self._file_type, **keywords)
def write_data(self, book): book_dict = book.to_dict() if isinstance(self.file_name, tuple): save_data(self.file_name[1], book_dict, file_type=self.file_name[0], **self.keywords) else: save_data(self.file_name, book_dict, **self.keywords)
def render_sheet_to_stream(self, model, sheet, init=None, mapdict=None, **keywords): headers = common.get_sheet_headers(sheet) importer = django.DjangoModelImporter() adapter = django.DjangoModelImportAdapter(model) adapter.column_names = headers adapter.column_name_mapping_dict = mapdict adapter.row_initializer = init importer.append(adapter) save_data(importer, {adapter.get_name(): sheet.get_internal_array()}, file_type=self._file_type, **keywords)
def test_load_sheet_from_django_model(self): model=FakeDjangoModel() importer = DjangoModelImporter() adapter = DjangoModelImportAdapter(model) adapter.set_column_names(self.data[0]) importer.append(adapter) save_data(importer, {adapter.get_name(): self.data[1:]}, file_type=DB_DJANGO) assert model.objects.objs == self.result model._meta.update(["X", "Y", "Z"]) reader = DjangoModelReader(model) data = reader.to_array() assert list(data) == self.data
def setUp(self): self.content = OrderedDict() self.content.update( {"Sheet 1": [[1.0, 2.0, 3.0], [4.0, 5.0, 6.0], [7.0, 8.0, 9.0]]} ) self.content.update( {"Sheet 2": [["X", "Y", "Z"], [1.0, 2.0, 3.0], [4.0, 5.0, 6.0]]} ) self.content.update( {"Sheet 3": [["O", "P", "Q"], [3.0, 2.0, 1.0], [4.0, 3.0, 2.0]]} ) save_data(self.file_name, self.content)
def setUp(self): self.test_file = "test_filter.ods" sample = [ [1, 21, 31], [2, 22, 32], [3, 23, 33], [4, 24, 34], [5, 25, 35], [6, 26, 36], ] save_data(self.test_file, sample) self.sheet_name = "pyexcel_sheet1"
def setUp(self): self.data = [["X", "Y", "Z"], [1, 2, 3], [4, 5, 6]] self.result = [{"Y": 2, "X": 1, "Z": 3}, {"Y": 5, "X": 4, "Z": 6}] self.model = FakeDjangoModel() importer = DjangoModelImporter() adapter = DjangoModelImportAdapter(self.model) adapter.column_names = self.data[0] importer.append(adapter) save_data( importer, {adapter.get_name(): self.data[1:]}, file_type=DB_DJANGO ) assert self.model.objects.objs == self.result self.model._meta.update(["X", "Y", "Z"])
def render_book_to_stream(self, models, thebook, inits=None, mapdicts=None, batch_size=None, **keywords): from pyexcel.book import to_book book = thebook if isinstance(thebook, BookStream): book = to_book(thebook) new_models = [model for model in models if model is not None] initializers = inits if initializers is None: initializers = [None] * len(new_models) if mapdicts is None: mapdicts = [None] * len(new_models) for sheet in book: if len(sheet.colnames) == 0: sheet.name_columns_by_row(0) colnames_array = [sheet.colnames for sheet in book] scattered = zip(new_models, colnames_array, mapdicts, initializers) importer = django.DjangoModelImporter() for each_model in scattered: adapter = django.DjangoModelImportAdapter(each_model[0]) adapter.column_names = each_model[1] adapter.column_name_mapping_dict = each_model[2] adapter.row_initializer = each_model[3] importer.append(adapter) to_store = OrderedDict() for sheet_name in book.sheet_names(): # due book.to_dict() brings in column_names # which corrupts the data to_store[sheet_name] = book[sheet_name].get_internal_array() save_data(importer, to_store, file_type=self._file_type, batch_size=batch_size, **keywords)
def osrf_to_xls_export(data, file_path): """ Workaround to export to a .xls file. Args: data (list): Data to export. file_path: File path for the output file. """ from pyexcel_xls import get_data try: #old_data = get_data(file_path) # A change in the API now returns only an array of arrays if there is only one sheet. old_data = {"OSRFramework": get_data(file_path)} except: # No information has been recovered old_data = {"OSRFramework": []} # Generating the new tabular data tabular_data = _generate_tabular_data(data, old_data) from pyexcel_xls import save_data # Storing the file save_data(file_path, tabular_data)
def osrf_to_csv_export(data, file_path): """ Workaround to export to a CSV file. Args: data (list): Data to export. file_path: File path for the output file. """ from pyexcel_io import get_data try: old_data = {"OSRFramework": get_data(file_path)} except: # No information has been recovered old_data = {"OSRFramework": []} # Generating the new tabular data. tabular_data = _generate_tabular_data(data, old_data) from pyexcel_io import save_data # Storing the file # NOTE: when working with CSV files it is no longer a dict because it is a one-sheet-format save_data(file_path, tabular_data["OSRFramework"])
def test_load_and_format_sheet_from_django_model(self): model = FakeDjangoModel() importer = DjangoModelImporter() adapter = DjangoModelImportAdapter(model) adapter.column_names = self.data[0] importer.append(adapter) save_data(importer, {adapter.get_name(): self.data[1:]}, file_type=DB_DJANGO) assert model.objects.objs == self.result model._meta.update(["X", "Y", "Z"]) def row_renderer(row): return [str(element) for element in row] # the key point of this test case reader = DjangoModelReader(model, row_renderer=row_renderer) data = reader.to_array() expected = [ ["X", "Y", "Z"], ['1', '2', '3'], ['4', '5', '6'] ] eq_(list(data), expected)
def usufyToXlsExport(d, fPath): """ Workaround to export to a .xls file. Args: ----- d: Data to export. fPath: File path for the output file. """ from pyexcel_xls import get_data try: #oldData = get_data(fPath) # A change in the API now returns only an array of arrays if there is only one sheet. oldData = {"OSRFramework": get_data(fPath)} except: # No information has been recovered oldData = {"OSRFramework": []} # Generating the new tabular data tabularData = _generateTabularData(d, oldData) from pyexcel_xls import save_data # Storing the file save_data(fPath, tabularData)
def render_book_to_stream( self, models, book, inits=None, mapdicts=None, batch_size=None, **keywords ): colnames_array = common.get_book_headers_in_array(book) new_models = [model for model in models if model is not None] initializers = inits if initializers is None: initializers = [None] * len(new_models) if mapdicts is None: mapdicts = [None] * len(new_models) scattered = zip(new_models, colnames_array, mapdicts, initializers) importer = django.DjangoModelImporter() for each_model in scattered: adapter = django.DjangoModelImportAdapter(each_model[0]) adapter.column_names = each_model[1] adapter.column_name_mapping_dict = each_model[2] adapter.row_initializer = each_model[3] importer.append(adapter) to_store = OrderedDict() for sheet in book: # due book.to_dict() brings in column_names # which corrupts the data to_store[sheet.name] = sheet.get_internal_array() save_data( importer, to_store, file_type=self._file_type, batch_size=batch_size, **keywords )
def usufyToCsvExport(d, fPath): """ Workaround to export to a CSV file. Args: ----- d: Data to export. fPath: File path for the output file. """ from pyexcel_io import get_data try: oldData = {"OSRFramework": get_data(fPath)} except: # No information has been recovered oldData = {"OSRFramework": []} # Generating the new tabular data. tabularData = _generateTabularData(d, oldData) from pyexcel_io import save_data # Storing the file # NOTE: when working with CSV files it is no longer a dict because it is a one-sheet-format save_data(fPath, tabularData["OSRFramework"])
def setUp(self): self.test_file = "test_filter.csv" sample = [[1, 21, 31], [2, 22, 32]] save_data(self.test_file, sample)
def test_library_parameter(): data = [['1', '2', '3']] io = manager.get_io("csv") save_data(io, data, 'csv', library="pyexcel-io") result = get_data(io.getvalue(), 'csv', library="pyexcel-io") eq_(result['csv'], [[1, 2, 3]])
def test_binary_file_content(): data = [['1', '2', '3']] io = manager.get_io("csvz") save_data(io, data, 'csvz') result = get_data(io.getvalue(), 'csvz') assert result['pyexcel_sheet1'] == [[1, 2, 3]]
def test_text_file_content(): data = [['1', '2', '3']] io = manager.get_io("csv") save_data(io, data, 'csv') result = get_data(io.getvalue(), 'csv') assert result['csv'] == [[1, 2, 3]]
def test_library_parameter(): data = [['1', '2', '3']] io = manager.get_io("csv") save_data(io, data, 'csv', library="built-in") result = get_data(io.getvalue(), 'csv', library="built-in") assert result['csv'] == [[1, 2, 3]]
# excel读写 # excel是一种广泛的办公文件格式,表格形式,比csv复杂一些,带一些功能样式,适合普通用户观看,由于带功能样式,所以需要专门负责的包解析.excel后缀 老格式xls,新格式xlsx,新格式兼容老格式. # 操作excel包选择,XlsxWriter xlrd pyexcel,start数,维护时间都较大同小异,下面以pyexcel讲解 from pyexcel_io import save_data from pyexcel_io import get_data data = get_data('2readfile.xlsx') # print(type(data)) sheet1 = data['Sheet1'] # print(sheet1) for index, row in enumerate(sheet1): if index <= 0: continue # print('第{}个学生,学生姓名{}'.format(index,row[1])) # 报错 # 1.不支持格式,请安装插件pyexcel-xls # 2.没有权限,管理员 # 写示例 # sheet1 = [ # {'name':'小明','age':'13','gender':'男'} # {'name':'小红','age':'12','gender':'女'} # ] sheet2 = [['小明', 13], ['小李', 12]] save_data(data=sheet2, afile='2writeexcel.xlsx')
def test_text_file_content(): data = [["1", "2", "3"]] io = manager.get_io("csv") save_data(io, data, "csv") result = get_data(io.getvalue(), "csv") eq_(result["csv"], [[1, 2, 3]])
def render_book_to_stream(self, file_stream, book, **keywords): save_data(file_stream, book.to_dict(), file_type=self.file_type, **keywords)
def setUp(self): self.test_file = "test_filter.csv" sample = [[1, 21, 31], [2, 22, 32], [3, 23, 33], [4, 24, 34], [5, 25, 35], [6, 26, 36]] save_data(self.test_file, sample)
#!/usr/bin/env python # -*- coding:utf-8 -*- from pyexcel_io import save_data sheet = [['id', 'name', 'password', 'role'], [1, '小明', 2313, '普通用户'], [2, 'admin', 123456, '管理员']] save_data(data=sheet, afile='2.xlsx')
def test_library_parameter_error_situation(): data = [['1', '2', '3']] io = manager.get_io("csv") save_data(io, data, 'csv', library="doesnot-exist")
def test_binary_file_content(): data = [["1", "2", "3"]] io = manager.get_io("csvz") save_data(io, data, "csvz") result = get_data(io.getvalue(), "csvz") eq_(result["pyexcel_sheet1"], [[1, 2, 3]])
def test_library_parameter_error_situation(): data = [["1", "2", "3"]] io = manager.get_io("csv") save_data(io, data, "csv", library="doesnot-exist")
def setUp(self): self.test_file = "test_filter.ods" sample = [[1, 21, 31], [2, 22, 32], [3, 23, 33], [4, 24, 34], [5, 25, 35], [6, 26, 36]] save_data(self.test_file, sample) self.sheet_name = "pyexcel_sheet1"
def test_library_parameter(): data = [["1", "2", "3"]] io = manager.get_io("csv") save_data(io, data, "csv", library="pyexcel-io") result = get_data(io.getvalue(), "csv", library="pyexcel-io") eq_(result["csv"], [[1, 2, 3]])
def render_book_to_file(self, file_name, book, **keywords): save_data(file_name, book.to_dict(), **keywords)
# 读写csv 、excel场景:文件读到变量,插入数据库;写,数据库信息导出。 # pip install pyexcel pyexcel-xls pyexcel-xlsx # from pyexcel_io import save_data from pyexcel_io import get_data from collections import OrderedDict # 读 excel = get_data('6test.xlsx') print(type(excel)) sheet1 = excel['Sheet1'] print( sheet1 ) # [['id', 'name', 'password', 'role'], [1, '小明', 1111, '普通用户'], [2, 'admin', 'admin', '管理员']] for index, row in enumerate(sheet1): if index == 0: # 如果表头,注意隔过。如果有列合并,以第一列下标取值 continue print(row) # 写 sheet2 = [['id', 'name', 'password', 'role'], [1, '小明', 1111, '普通用户'], [2, 'admin', 'admin', '管理员']] save_data(data=sheet2, afile='6write.xlsx') # OrderedDict 有序字典。列表有下标有序,字典本身无序,但有些业务场景要求有序会用到orderDict # { # 'stu2': {} # 'stu3': {} # 'stu1' # }