def _load_from_file(self): """Load content from a file :params str filename: an accessible file path :returns: a book """ self.__line_terminator = self._keywords.get( constants.KEYWORD_LINE_TERMINATOR, self.__line_terminator) names = self._file_name.split('.') filepattern = "%s%s*%s*.%s" % (names[0], DEFAULT_SEPARATOR, DEFAULT_SEPARATOR, names[1]) filelist = glob.glob(filepattern) if len(filelist) == 0: file_parts = os.path.split(self._file_name) return [NamedContent(file_parts[-1], self._file_name)] else: matcher = "%s%s(.*)%s(.*).%s" % (names[0], DEFAULT_SEPARATOR, DEFAULT_SEPARATOR, names[1]) tmp_file_list = [] for filen in filelist: result = re.match(matcher, filen) tmp_file_list.append((result.group(1), result.group(2), filen)) ret = [] for lsheetname, index, filen in sorted(tmp_file_list, key=lambda row: row[1]): ret.append(NamedContent(lsheetname, filen)) return ret
def _load_from_stream(self): """Load content from memory :params stream file_content: the actual file content in memory :returns: a book """ self.__load_from_memory_flag = True self.__line_terminator = self._keywords.get( constants.KEYWORD_LINE_TERMINATOR, self.__line_terminator ) separator = DEFAULT_SHEET_SEPARATOR_FORMATTER % self.__line_terminator if self.__multiple_sheets: # will be slow for large files self._file_stream.seek(0) content = self._file_stream.read() sheets = content.split(separator) named_contents = [] for sheet in sheets: if sheet == "": # skip empty named sheet continue lines = sheet.split(self.__line_terminator) result = re.match(constants.SEPARATOR_MATCHER, lines[0]) new_content = "\n".join(lines[1:]) new_sheet = NamedContent( result.group(1), compact.StringIO(new_content) ) named_contents.append(new_sheet) return named_contents else: if hasattr(self._file_stream, "seek"): self._file_stream.seek(0) return [NamedContent(self._file_type, self._file_stream)]
def read_sheet(self, index): name = self.content_array[index].name content = self.zipfile.read(self.content_array[index].payload) encoding_guess = chardet.detect(content) sheet = StringIO(content.decode(encoding_guess["encoding"])) return CSVinMemoryReader(NamedContent(name, sheet), **self.keywords)
def test_writer(self): native_sheet = NamedContent("test", []) content = [[1, 2], [3, 4], [5, 6]] writer = ArrayWriter(None, native_sheet, "test") writer.write_row(content[0]) writer.write_array(content[1:]) assert native_sheet.payload == content
def test_utf16_decoding(): test_file = os.path.join("tests", "fixtures", "csv-encoding-utf16.csv") reader = CSVFileReader(NamedContent("csv", test_file), encoding="utf-16") content = list(reader.to_array()) if PY2: content[0] = [s.encode("utf-8") for s in content[0]] expected = [["Äkkilähdöt", "Matkakirjoituksia", "Matkatoimistot"]] eq_(content, expected)
def test_sheet_memory_reader_delimiters(self): io = manager.get_io(self.file_type) with open(self.test_file, "r") as f: io.write(f.read()) io.seek(0) r = CSVinMemoryReader(NamedContent(self.file_type, io), delimiters=None) result = list(r.to_array()) self.assertEqual(result, [[1, 2, 3], [4, 5, 6], [7, 8, 9]])
def test_sheet_memory_reader(self): io = manager.get_io(self.file_type) with open(self.test_file, "r") as f: io.write(f.read()) io.seek(0) r = CSVinMemoryReader(NamedContent(self.file_type, io)) result = list(r.to_array()) self.assertEqual(result, self.expected_data)
def test_utf16_memory_decoding(): test_content = u"Äkkilähdöt,Matkakirjoituksia,Matkatoimistot" test_content = BytesIO(test_content.encode("utf-16")) reader = CSVinMemoryReader(NamedContent("csv", test_content), encoding="utf-16") content = list(reader.to_array()) if PY2: content[0] = [s.encode("utf-8") for s in content[0]] expected = [["Äkkilähdöt", "Matkakirjoituksia", "Matkatoimistot"]] eq_(content, expected)
def test_utf16_memory_decoding(): test_content = u'Äkkilähdöt,Matkakirjoituksia,Matkatoimistot' test_content = BytesIO(test_content.encode('utf-16')) reader = CSVinMemoryReader(NamedContent('csv', test_content), encoding="utf-16") content = list(reader.to_array()) if PY2: content[0] = [s.encode('utf-8') for s in content[0]] expected = [['Äkkilähdöt', 'Matkakirjoituksia', 'Matkatoimistot']] eq_(content, expected)
def __init__(self, file_stream, file_type, multiple_sheets=False, **keywords): """Load content from memory :params stream file_content: the actual file content in memory :returns: a book """ self.handles = [] self.keywords = keywords if file_type == constants.FILE_FORMAT_TSV: self.keywords["dialect"] = constants.KEYWORD_TSV_DIALECT self.file_type = file_type self.__load_from_memory_flag = True self.__line_terminator = keywords.get( constants.KEYWORD_LINE_TERMINATOR, constants.DEFAULT_CSV_NEWLINE) separator = DEFAULT_SHEET_SEPARATOR_FORMATTER % self.__line_terminator if multiple_sheets: # will be slow for large files file_stream.seek(0) content = file_stream.read() sheets = content.split(separator) named_contents = [] for sheet in sheets: if sheet == "": # skip empty named sheet continue lines = sheet.split(self.__line_terminator) result = re.match(constants.SEPARATOR_MATCHER, lines[0]) new_content = "\n".join(lines[1:]) new_sheet = NamedContent(result.group(1), compact.StringIO(new_content)) named_contents.append(new_sheet) self.content_array = named_contents else: if hasattr(file_stream, "seek"): file_stream.seek(0) self.content_array = [NamedContent(self.file_type, file_stream)]
def __init__(self, file_name, file_type, **keywords): """Load content from a file :params str filename: an accessible file path :returns: a book """ self.handles = [] self.keywords = keywords if file_type == constants.FILE_FORMAT_TSV: self.keywords["dialect"] = constants.KEYWORD_TSV_DIALECT self.__line_terminator = keywords.get( constants.KEYWORD_LINE_TERMINATOR, DEFAULT_NEWLINE) names = os.path.splitext(file_name) filepattern = "%s%s*%s*%s" % ( names[0], constants.DEFAULT_MULTI_CSV_SEPARATOR, constants.DEFAULT_MULTI_CSV_SEPARATOR, names[1], ) filelist = glob.glob(filepattern) if len(filelist) == 0: file_parts = os.path.split(file_name) self.content_array = [NamedContent(file_parts[-1], file_name)] else: matcher = "%s%s(.*)%s(.*)%s" % ( names[0], constants.DEFAULT_MULTI_CSV_SEPARATOR, constants.DEFAULT_MULTI_CSV_SEPARATOR, names[1], ) tmp_file_list = [] for filen in filelist: result = re.match(matcher, filen) tmp_file_list.append((result.group(1), result.group(2), filen)) ret = [] for lsheetname, index, filen in sorted(tmp_file_list, key=lambda row: row[1]): ret.append(NamedContent(lsheetname, filen)) self.content_array = ret
def __init__(self, file_alike_object, file_type, **keywords): self.content_array = [] try: self.zipfile = zipfile.ZipFile(file_alike_object, "r") sheets = [ NamedContent(_get_sheet_name(name), name) for name in self.zipfile.namelist() ] self.content_array = sheets self.keywords = keywords if file_type == constants.FILE_FORMAT_TSVZ: self.keywords["dialect"] = constants.KEYWORD_TSV_DIALECT except zipfile.BadZipfile: print("StringIO instance was passed by any chance?") raise
def __init__(self, name, payload): NamedContent.__init__(self, name, payload) self.colnames = []
def test_sheet_file_reader_delimiters(self): r = CSVFileReader(NamedContent(self.file_type, self.test_file), delimiters=None) result = list(r.to_array()) self.assertEqual(result, [[1, 2, 3], [4, 5, 6], [7, 8, 9]])
def test_sheet_file_reader(self): r = CSVFileReader(NamedContent(self.file_type, self.test_file)) result = list(r.to_array()) self.assertEqual(result, self.expected_data)
def test_sheet_file_reader(self): r = CSVFileReader(NamedContent(self.file_type, self.test_file), auto_detect_delimiter=False) result = list(r.to_array()) self.assertEqual(result, [[1], [4, 5, 6], ["", 7]])
def __init__(self, name, payload): NamedContent.__init__(self, name, payload) self.colnames = []