def _get_last_column(self):
if typepy.is_not_empty_sequence(self.headers):
return len(self.headers) - 1
if typepy.is_not_empty_sequence(self.value_matrix):
return len(self.value_matrix[0]) - 1
raise ValueError("data not found")
def _get_last_column(self):
if typepy.is_not_empty_sequence(self.header_list):
return len(self.header_list) - 1
if typepy.is_not_empty_sequence(self.value_matrix):
return len(self.value_matrix[0]) - 1
raise ValueError("data not found")
def __create_table_from_tabledata(self, tabledata, index_attr_list=None):
self.validate_access_permission(["w", "a"])
validate_table_name(tabledata.table_name)
logger.debug(
"__create_table_from_tabledata: table={}, headers={}".format(
tabledata.table_name, tabledata.header_list))
attr_name_list = self.__sanitize_attr_name_list(tabledata.header_list)
try:
self.__validate_attr_name_list(attr_name_list)
except pathvalidate.ReservedNameError:
pass
if typepy.is_empty_sequence(tabledata.value_matrix):
raise ValueError("input data is null: '{} ({})'".format(
tabledata.table_name, ", ".join(attr_name_list)))
self.__verify_value_matrix(attr_name_list, tabledata.value_matrix)
self.create_table(
tabledata.table_name,
self.__get_attr_desc_list(attr_name_list, tabledata.value_matrix))
self.insert_many(tabledata.table_name, tabledata.value_matrix)
if typepy.is_not_empty_sequence(index_attr_list):
self.create_index_list(
tabledata.table_name,
self.__sanitize_attr_name_list(index_attr_list))
self.commit()
def __parse_html(self, table):
header_list = []
data_matrix = []
self.__parse_tag_id(table)
row_list = table.find_all("tr")
re_table_val = re.compile("td|th")
for row in row_list:
td_list = row.find_all("td")
if typepy.is_empty_sequence(td_list):
if typepy.is_not_empty_sequence(header_list):
continue
th_list = row.find_all("th")
if typepy.is_empty_sequence(th_list):
continue
header_list = [row.text.strip() for row in th_list]
continue
data_matrix.append([value.get_text().strip() for value in row.find_all(re_table_val)])
if typepy.is_empty_sequence(data_matrix):
raise ValueError("data matrix is empty")
self._loader.inc_table_count()
return TableData(
self._make_table_name(),
header_list,
data_matrix,
dp_extractor=self._loader.dp_extractor,
)
def _to_data_matrix(self):
try:
return [[self.__modify_item(data) for data in row]
for row in self._csv_reader
if typepy.is_not_empty_sequence(row)]
except (csv.Error, UnicodeDecodeError) as e:
raise DataError(e)
def __create_table_from_tabledata(
self, tabledata, index_attr_list=None):
self.validate_access_permission(["w", "a"])
validate_table_name(tabledata.table_name)
logger.debug(
"__create_table_from_tabledata: table={}, headers={}".format(
tabledata.table_name, tabledata.header_list))
attr_name_list = self.__sanitize_attr_name_list(tabledata.header_list)
try:
self.__validate_attr_name_list(attr_name_list)
except pathvalidate.ReservedNameError:
pass
if typepy.is_empty_sequence(tabledata.value_matrix):
raise ValueError("input data is null: '{} ({})'".format(
tabledata.table_name, ", ".join(attr_name_list)))
self.__verify_value_matrix(attr_name_list, tabledata.value_matrix)
self.create_table(
tabledata.table_name,
self.__get_attr_desc_list(
attr_name_list, tabledata.value_matrix))
self.insert_many(tabledata.table_name, tabledata.value_matrix)
if typepy.is_not_empty_sequence(index_attr_list):
self.create_index_list(
tabledata.table_name,
self.__sanitize_attr_name_list(index_attr_list))
self.commit()
def test_normal(self, table_name, header_list, row_list):
tabledata = TableData(table_name, header_list, row_list)
dataframe = pandas.DataFrame(row_list)
if typepy.is_not_empty_sequence(header_list):
dataframe.columns = header_list
print("lhs: {}".format(tabledata.as_dataframe()))
print("rhs: {}".format(dataframe))
assert tabledata.as_dataframe().equals(dataframe)
def is_empty_record(self):
"""
:return: |True| if the tabular data is not an empty nested list.
:rtype: bool
"""
try:
return not typepy.is_not_empty_sequence(self.value_dp_matrix[0])
except (TypeError, IndexError):
return True
def _get_closing_row_items(self) -> List[str]:
if typepy.is_not_empty_sequence(self.headers):
return [
"], columns=[{}])".format(", ".join([
'"{}"'.format(MultiByteStrDecoder(header).unicode_str)
for header in self.headers
]))
]
return ["])"]
def num_columns(self) -> Optional[int]:
if typepy.is_not_empty_sequence(self.headers):
return len(self.headers)
try:
return len(self.rows[0])
except TypeError:
return None
except IndexError:
return 0
def _write_header(self) -> None:
if not self.is_write_header:
return
if typepy.is_not_empty_sequence(self.headers):
self._write_line(':header: "{:s}"'.format('", "'.join(
MultiByteStrDecoder(header).unicode_str
for header in self.headers)))
self._write_line(":widths: " + ", ".join(
str(col_dp.ascii_char_width) for col_dp in self._column_dp_list))
self._write_line()
def _get_closing_row_items(self):
if typepy.is_not_empty_sequence(self.headers):
return [
"], columns=[{}])".format(
", ".join(
[
'"{}"'.format(MultiByteStrDecoder(header).unicode_str)
for header in self.headers
]
)
)
]
return ["])"]
def _write_header(self):
if not self.is_write_header:
return
if typepy.is_not_empty_sequence(self.headers):
self._write_line(
':header: "{:s}"'.format(
'", "'.join(
[MultiByteStrDecoder(header).unicode_str for header in self.headers]
)
)
)
self._write_line(
":widths: "
+ ", ".join([str(col_dp.ascii_char_width) for col_dp in self._column_dp_list])
)
self._write_line()
def _fetch_index_schema(self, table_name):
self.__update_sqlite_master_db()
result = self.__execute_sqlite_master(
"SELECT {:s} FROM {:s} WHERE {:s} AND {:s}".format(
"sql",
self._SQLITE_MASTER_TABLE_NAME,
"{:s} = '{:s}'".format("tbl_name", table_name),
"{:s} = '{:s}'".format("type", "index"),
),
self.global_debug_query,
)
try:
return [
record[0] for record in result.fetchall()
if typepy.is_not_empty_sequence(record[0])
]
except TypeError:
raise DataNotFoundError(
"index not found in '{}'".format(table_name))
def _get_index_schema(self, table_name):
self.__update_sqlite_master_db()
try:
result = self._con_sql_master.select(
"sql",
table_name=self._SQLITE_MASTER_TABLE_NAME,
where=" AND ".join([
SqlQuery.make_where("tbl_name", table_name),
SqlQuery.make_where("type", "index"),
]))
except simplesqlite.TableNotFoundError as e:
raise DataNotFoundError(e)
try:
return [
record[0] for record in result.fetchall()
if typepy.is_not_empty_sequence(record[0])
]
except TypeError:
raise DataNotFoundError(
"index not found in '{}'".format(table_name))
def test_normal(self, value, expected):
assert is_not_empty_sequence(value) == expected
def test_normal_profile(self, con_profile):
profile_list = con_profile.get_profile()
assert typepy.is_not_empty_sequence(profile_list)
def test_normal_profile(self, con_profile):
profile_list = con_profile.get_profile()
assert typepy.is_not_empty_sequence(profile_list)
