def _get_last_column(self): if typepy.is_not_empty_sequence(self.headers): return len(self.headers) - 1 if typepy.is_not_empty_sequence(self.value_matrix): return len(self.value_matrix[0]) - 1 raise ValueError("data not found")
def _get_last_column(self): if typepy.is_not_empty_sequence(self.header_list): return len(self.header_list) - 1 if typepy.is_not_empty_sequence(self.value_matrix): return len(self.value_matrix[0]) - 1 raise ValueError("data not found")
def __create_table_from_tabledata(self, tabledata, index_attr_list=None): self.validate_access_permission(["w", "a"]) validate_table_name(tabledata.table_name) logger.debug( "__create_table_from_tabledata: table={}, headers={}".format( tabledata.table_name, tabledata.header_list)) attr_name_list = self.__sanitize_attr_name_list(tabledata.header_list) try: self.__validate_attr_name_list(attr_name_list) except pathvalidate.ReservedNameError: pass if typepy.is_empty_sequence(tabledata.value_matrix): raise ValueError("input data is null: '{} ({})'".format( tabledata.table_name, ", ".join(attr_name_list))) self.__verify_value_matrix(attr_name_list, tabledata.value_matrix) self.create_table( tabledata.table_name, self.__get_attr_desc_list(attr_name_list, tabledata.value_matrix)) self.insert_many(tabledata.table_name, tabledata.value_matrix) if typepy.is_not_empty_sequence(index_attr_list): self.create_index_list( tabledata.table_name, self.__sanitize_attr_name_list(index_attr_list)) self.commit()
def __parse_html(self, table): header_list = [] data_matrix = [] self.__parse_tag_id(table) row_list = table.find_all("tr") re_table_val = re.compile("td|th") for row in row_list: td_list = row.find_all("td") if typepy.is_empty_sequence(td_list): if typepy.is_not_empty_sequence(header_list): continue th_list = row.find_all("th") if typepy.is_empty_sequence(th_list): continue header_list = [row.text.strip() for row in th_list] continue data_matrix.append([value.get_text().strip() for value in row.find_all(re_table_val)]) if typepy.is_empty_sequence(data_matrix): raise ValueError("data matrix is empty") self._loader.inc_table_count() return TableData( self._make_table_name(), header_list, data_matrix, dp_extractor=self._loader.dp_extractor, )
def _to_data_matrix(self): try: return [[self.__modify_item(data) for data in row] for row in self._csv_reader if typepy.is_not_empty_sequence(row)] except (csv.Error, UnicodeDecodeError) as e: raise DataError(e)
def __create_table_from_tabledata( self, tabledata, index_attr_list=None): self.validate_access_permission(["w", "a"]) validate_table_name(tabledata.table_name) logger.debug( "__create_table_from_tabledata: table={}, headers={}".format( tabledata.table_name, tabledata.header_list)) attr_name_list = self.__sanitize_attr_name_list(tabledata.header_list) try: self.__validate_attr_name_list(attr_name_list) except pathvalidate.ReservedNameError: pass if typepy.is_empty_sequence(tabledata.value_matrix): raise ValueError("input data is null: '{} ({})'".format( tabledata.table_name, ", ".join(attr_name_list))) self.__verify_value_matrix(attr_name_list, tabledata.value_matrix) self.create_table( tabledata.table_name, self.__get_attr_desc_list( attr_name_list, tabledata.value_matrix)) self.insert_many(tabledata.table_name, tabledata.value_matrix) if typepy.is_not_empty_sequence(index_attr_list): self.create_index_list( tabledata.table_name, self.__sanitize_attr_name_list(index_attr_list)) self.commit()
def test_normal(self, table_name, header_list, row_list): tabledata = TableData(table_name, header_list, row_list) dataframe = pandas.DataFrame(row_list) if typepy.is_not_empty_sequence(header_list): dataframe.columns = header_list print("lhs: {}".format(tabledata.as_dataframe())) print("rhs: {}".format(dataframe)) assert tabledata.as_dataframe().equals(dataframe)
def is_empty_record(self): """ :return: |True| if the tabular data is not an empty nested list. :rtype: bool """ try: return not typepy.is_not_empty_sequence(self.value_dp_matrix[0]) except (TypeError, IndexError): return True
def _get_closing_row_items(self) -> List[str]: if typepy.is_not_empty_sequence(self.headers): return [ "], columns=[{}])".format(", ".join([ '"{}"'.format(MultiByteStrDecoder(header).unicode_str) for header in self.headers ])) ] return ["])"]
def num_columns(self) -> Optional[int]: if typepy.is_not_empty_sequence(self.headers): return len(self.headers) try: return len(self.rows[0]) except TypeError: return None except IndexError: return 0
def _write_header(self) -> None: if not self.is_write_header: return if typepy.is_not_empty_sequence(self.headers): self._write_line(':header: "{:s}"'.format('", "'.join( MultiByteStrDecoder(header).unicode_str for header in self.headers))) self._write_line(":widths: " + ", ".join( str(col_dp.ascii_char_width) for col_dp in self._column_dp_list)) self._write_line()
def _get_closing_row_items(self): if typepy.is_not_empty_sequence(self.headers): return [ "], columns=[{}])".format( ", ".join( [ '"{}"'.format(MultiByteStrDecoder(header).unicode_str) for header in self.headers ] ) ) ] return ["])"]
def _write_header(self): if not self.is_write_header: return if typepy.is_not_empty_sequence(self.headers): self._write_line( ':header: "{:s}"'.format( '", "'.join( [MultiByteStrDecoder(header).unicode_str for header in self.headers] ) ) ) self._write_line( ":widths: " + ", ".join([str(col_dp.ascii_char_width) for col_dp in self._column_dp_list]) ) self._write_line()
def _fetch_index_schema(self, table_name): self.__update_sqlite_master_db() result = self.__execute_sqlite_master( "SELECT {:s} FROM {:s} WHERE {:s} AND {:s}".format( "sql", self._SQLITE_MASTER_TABLE_NAME, "{:s} = '{:s}'".format("tbl_name", table_name), "{:s} = '{:s}'".format("type", "index"), ), self.global_debug_query, ) try: return [ record[0] for record in result.fetchall() if typepy.is_not_empty_sequence(record[0]) ] except TypeError: raise DataNotFoundError( "index not found in '{}'".format(table_name))
def _get_index_schema(self, table_name): self.__update_sqlite_master_db() try: result = self._con_sql_master.select( "sql", table_name=self._SQLITE_MASTER_TABLE_NAME, where=" AND ".join([ SqlQuery.make_where("tbl_name", table_name), SqlQuery.make_where("type", "index"), ])) except simplesqlite.TableNotFoundError as e: raise DataNotFoundError(e) try: return [ record[0] for record in result.fetchall() if typepy.is_not_empty_sequence(record[0]) ] except TypeError: raise DataNotFoundError( "index not found in '{}'".format(table_name))
def test_normal(self, value, expected): assert is_not_empty_sequence(value) == expected
def test_normal_profile(self, con_profile): profile_list = con_profile.get_profile() assert typepy.is_not_empty_sequence(profile_list)
def test_normal_profile(self, con_profile): profile_list = con_profile.get_profile() assert typepy.is_not_empty_sequence(profile_list)