def parse_overflow_page_chain(page_hex_string):
if (len(page_hex_string) == 0):
_adel_log.log(
"parse_overflow_page_chain: WARNING! empty hexadecimal page string received",
2)
return ""
if (len(page_hex_string) < 8):
_adel_log.log(
"parse_overflow_page_chain: WARNING! hexadecimal page string is too short: "
+ str(page_hex_string), 2)
return ""
# Check whether there is another overflow page: first 8 nibbles is 4-byte integer pointer to next overflow page or 00 00 00 00 if no further overflow page exists
next_overflow_page_number = int(
page_hex_string[0:8],
16) # will be zero if we reached the last overflow page in the chain
# Build content string: append all content of this page
overflow_page_content = page_hex_string[8:]
_adel_log.log(
"parse_overflow_page_chain: OK - overflow page parsed" %
vars(), 4)
if next_overflow_page_number != 0:
# There is at least one more overflow page: append further content
_adel_log.log(
"parse_overflow_page_chain: ----> parsing next overflow page in chain, page number is: %(next_overflow_page_number)s...."
% vars(), 4)
overflow_page_content += parse_overflow_page_chain(
_sqliteFileHandler.read_page(next_overflow_page_number))
return overflow_page_content
def parse_table_btree_interior_page(page_hex_string, page_offset):
# Parse the page header
header = parse_btree_page_header(page_hex_string, page_offset)
# Ensure that we deal with a correct page
header_length = len(header)
if (header_length != 7 or header[0] != 5):
# No valid header_length
_adel_log.log(
"parse_table_btree_interior_page: ERROR - invalid page type in table b-tree interior page header",
1)
_adel_log.log(
" Page header was said to start at page offset: "
+ str(page_offset), 1)
_adel_log.log(" Printing page content....",
1)
_adel_log.log(page_hex_string, 1)
return []
# initialize resulting list
content_list = []
# Initialize node list
node_pointers = []
# Initialize page content list
page_contents = []
# Parse cell pointer array
cell_pointers = parse_cell_pointer_array(
page_hex_string, (page_offset + header[header_length - 1]), header[2])
# Parse cells
for cell_pointer in cell_pointers:
node_pointers.append(
parse_table_btree_interior_cell(page_hex_string, cell_pointer))
# This is an interior page, thus we append the right-most pointer as well
node_pointers.append([header[5], 0])
# Iterate through every node
for node_tuple in node_pointers:
_adel_log.log(
"parse_table_btree_interior_page: ----> fetching child page to parse, page number: "
+ str(node_tuple[0]) + "....", 3)
child_page = _sqliteFileHandler.read_page(node_tuple[0])
# Ensure we fetched a valid page
if (child_page == ""):
_adel_log.log(
"parse_table_btree_interior_page: ERROR - invalid node tuple detected, cannot reference child page pointer: "
+ str(node_tuple), 1)
continue
# Parse child pages
page_contents = parse_table_btree_page(child_page, 0)
for page_content in page_contents:
content_list.append(page_content)
_adel_log.log(
"parse_table_btree_interior_page: ----> child page parsed, page number: "
+ str(node_tuple[0]) + "....", 4)
return content_list
def parse_table_btree_interior_page(page_hex_string, page_offset):
# Parse the page header
header = parse_btree_page_header(page_hex_string, page_offset)
# Ensure that we deal with a correct page
header_length = len(header)
if (header_length != 7 or header[0] != 5):
# No valid header_length
_adel_log.log("parse_table_btree_interior_page: ERROR - invalid page type in table b-tree interior page header", 1)
_adel_log.log(" Page header was said to start at page offset: " + str(page_offset), 1)
_adel_log.log(" Printing page content....", 1)
_adel_log.log(page_hex_string, 1)
return []
# initialize resulting list
content_list = []
# Initialize node list
node_pointers = []
# Initialize page content list
page_contents = []
# Parse cell pointer array
cell_pointers = parse_cell_pointer_array(page_hex_string, (page_offset + header[header_length - 1]), header[2])
# Parse cells
for cell_pointer in cell_pointers:
node_pointers.append(parse_table_btree_interior_cell(page_hex_string, cell_pointer))
# This is an interior page, thus we append the right-most pointer as well
node_pointers.append([header[5], 0])
# Iterate through every node
for node_tuple in node_pointers:
_adel_log.log("parse_table_btree_interior_page: ----> fetching child page to parse, page number: " + str(node_tuple[0]) + "....", 3)
child_page = _sqliteFileHandler.read_page(node_tuple[0])
# Ensure we fetched a valid page
if (child_page == ""):
_adel_log.log("parse_table_btree_interior_page: ERROR - invalid node tuple detected, cannot reference child page pointer: " + str(node_tuple), 1)
continue
# Parse child pages
page_contents = parse_table_btree_page(child_page, 0)
for page_content in page_contents:
content_list.append(page_content)
_adel_log.log("parse_table_btree_interior_page: ----> child page parsed, page number: " + str(node_tuple[0]) + "....", 4)
return content_list
def parse_overflow_page_chain(page_hex_string):
if (len(page_hex_string) == 0):
_adel_log.log("parse_overflow_page_chain: WARNING! empty hexadecimal page string received", 2)
return ""
if (len(page_hex_string) < 8):
_adel_log.log("parse_overflow_page_chain: WARNING! hexadecimal page string is too short: " + str(page_hex_string), 2)
return ""
# Check whether there is another overflow page: first 8 nibbles is 4-byte integer pointer to next overflow page or 00 00 00 00 if no further overflow page exists
next_overflow_page_number = int(page_hex_string[0:8], 16) # will be zero if we reached the last overflow page in the chain
# Build content string: append all content of this page
overflow_page_content = page_hex_string[8:]
_adel_log.log("parse_overflow_page_chain: OK - overflow page parsed" % vars(), 4)
if next_overflow_page_number != 0:
# There is at least one more overflow page: append further content
_adel_log.log("parse_overflow_page_chain: ----> parsing next overflow page in chain, page number is: %(next_overflow_page_number)s...." % vars(), 4)
overflow_page_content += parse_overflow_page_chain(_sqliteFileHandler.read_page(next_overflow_page_number))
return overflow_page_content
def parse_table_btree_leaf_cell(page_hex_string, page_offset, cell_pointers,
free_block_pointer):
# 1 byte is represented by two characters in the hexString, so internally we need to calculate the offset in nibbles
page_offset_in_bytes = page_offset # store for log reasons only
page_offset = page_offset * 2 # now dealing with nibbles because we treat a string (1 character = 1 nibble)
db_page_size_in_bytes = _sqliteFileHandler.DB_PAGESIZE_IN_BYTES
usable_page_space = db_page_size_in_bytes - _sqliteFileHandler.DB_RESERVED_SPACE
_adel_log.log(
"parse_table_btree_leaf_cell: ----> parsing b-tree leaf cell at offset %(page_offset_in_bytes)s...."
% vars(), 4)
# Get total number of bytes of payload
bytes_of_payload_tuple = _sqliteVarInt.parse_next_var_int(
page_hex_string[page_offset:(page_offset + 18)]
) # a variable integer can be maximum 9 byte (= 18 nibbles) long
bytes_of_payload = bytes_of_payload_tuple[0]
_adel_log.log(
"parse_table_btree_leaf_cell: OK - payload is %(bytes_of_payload)s bytes long"
% vars(), 4)
# Get row_id
row_id_string = page_hex_string[(page_offset +
(bytes_of_payload_tuple[1] * 2)):(
page_offset +
(bytes_of_payload_tuple[1] + 9) * 2)]
row_id_tuple = _sqliteVarInt.parse_next_var_int(row_id_string)
row_id = row_id_tuple[0]
_adel_log.log(
"parse_table_btree_leaf_cell: ----> extracting contents for row_id %(row_id)s...."
% vars(), 4)
# Check for overflow pages and append content of those pages, if any
# Calculate the overflow limits for table b-tree leaf cell
remaining_page_space = db_page_size_in_bytes - page_offset_in_bytes
if (bytes_of_payload > (remaining_page_space)):
# We expext content to overflow, because there is not enough space left on this page
_adel_log.log(
"parse_table_btree_leaf_cell: OK - payload is too large for this page, there are overflow pages"
% vars(), 4)
# Check at which position the next cell starts
next_cell = usable_page_space
for cell_pointer in cell_pointers:
if (cell_pointer > page_offset_in_bytes) and (cell_pointer <
next_cell):
next_cell = cell_pointer
# Check at which position the next freeblock starts (we ignore theoretically possible freebytes in this case,
# Because we expect no freebyte at the end of a cell that overflows to another page
next_free_block = usable_page_space
free_blocks = parse_free_blocks(page_hex_string, free_block_pointer)
for free_block in free_blocks:
if (free_block[0] > page_offset_in_bytes) and (free_block[0] <
next_free_block):
next_free_block = free_block[0]
# Get the end of this record: either closest following cell or closest following freeblock or end of page
end_of_record = usable_page_space
# Check of the end of this record is given through a following cell
if (next_cell != usable_page_space) and (
(next_cell <= next_free_block) or
(next_free_block == usable_page_space)):
# next element is not end of page but a cell
end_of_record = next_cell
# Check of the end of this record is given through a following free block
if (next_free_block != usable_page_space) and (
(next_free_block < next_cell) or (next_cell == usable_page_space)):
# Next element is not end of page but a free block
end_of_record = next_free_block
# Cut record hex string from the beginning to the offset of the next following element
record_hex_string = page_hex_string[(page_offset + (
(bytes_of_payload_tuple[1] + row_id_tuple[1]) * 2)):(
end_of_record * 2)]
record_hex_string_length = len(
record_hex_string
) / 2 # string length is count in nibbles, we need bytes here
# Save overflow page pointer at the end of record hex string
first_overflow_page_number = int(
record_hex_string[((record_hex_string_length - 4) *
2):(record_hex_string_length * 2)], 16)
_adel_log.log(
"parse_table_btree_leaf_cell: ----> parsing overflow page chain beginning at page %(first_overflow_page_number)s...."
% vars(), 4)
# Cut off overflow page number from record_hex_string
record_hex_string = record_hex_string[(
0):((record_hex_string_length - 4) * 2)]
first_overflow_page_string = _sqliteFileHandler.read_page(
first_overflow_page_number)
# Ensure that read page could retrieve an existing page
if (first_overflow_page_string == ""):
_adel_log.log(
"parse_table_btree_leaf_cell: ERROR - invalid overflow page pointer, cannot reference first overflow page: "
+ str(first_overflow_page_number), 1)
return []
# Append content from overflow pages
record_hex_string += parse_overflow_page_chain(
first_overflow_page_string)
# Ensure correct length of string (maybe not all bytes of the last overflow page in the chain contain content)
record_hex_string_length = len(
record_hex_string
) / 2 # string length is count in nibbles, we need bytes here
if (bytes_of_payload < record_hex_string_length):
# Cut record hex string again
record_hex_string = record_hex_string[:bytes_of_payload * 2]
else:
# The entire payload is stored on this page
record_hex_string = page_hex_string[(page_offset + (
(bytes_of_payload_tuple[1] + row_id_tuple[1]) * 2)):(
page_offset + ((bytes_of_payload_tuple[1] + row_id_tuple[1] +
bytes_of_payload_tuple[0]) * 2))]
# Parse the record
read_content_list = parse_record(record_hex_string)
# Build the resulting list (including the row_id used sqlite internally)
cell_content_list = []
cell_content_list.append(row_id)
for element in range(len(read_content_list)):
cell_content_list.append(read_content_list[element])
# Return results
_adel_log.log(
"parse_table_btree_leaf_cell: OK - returning list of cell contents",
4)
_adel_log.log(
"parse_table_btree_leaf_cell: ----> b-tree leaf cell at offset %(page_offset_in_bytes)s parsed"
% vars(), 4)
return cell_content_list
def parse_db(file_name):
global DB_FILE_SIZE_IN_BYTES
global ROW_ID_COLUMN
_adel_log.log("\n############ SQLite PARSER -> " + file_name + " ############ \n", 2)
_adel_log.log("parse_db: ----> parsing sqlite3 database file....", 3)
# Open the database
DB_FILE_SIZE_IN_BYTES = _sqliteFileHandler.open_db(file_name)
if DB_FILE_SIZE_IN_BYTES == 0:
# file could not be opened correctly
return []
# Read first page of database file
first_page_hex_string = _sqliteFileHandler.read_page(1)
# ensure that read page could retrieve an existing page
if (first_page_hex_string == ""):
_adel_log.log("parse_db: ERROR - cannot read first page of database", 1)
return []
# Parse the database header on the first page (first 100 bytes in the database file)
parse_db_header(first_page_hex_string)
if HEADER_DATABASE_TEXT_ENCODING > 1:
_adel_log.log("parse_db: ERROR - database text encoding " + str(HEADER_DATABASE_TEXT_ENCODING) + " not supported in this version of FSP", 1)
return []
# Parse database schema (first page of the database file is root b-tree page for the schema btree)
# Database schema is stored in a well defined way (sqlite master table)
# CREATE TABLE sqlite_master(
# type text, # must be one of the following: ['table', 'index', 'view', 'trigger']
# name text,
# tbl_name text,
# rootpage integer,
# sql text
# );
_adel_log.log("\nparseDB: ----> parsing sqlite3 database SCHEMA....", 3)
db_schemata = _sqlitePageParser.parse_table_btree_page(first_page_hex_string, 100) # 100 bytes database file header
_adel_log.log("parse_db: ----> sqlite3 database SCHEMA parsed", 3)
# Initialize the resulting content list
result_list = []
final_list = []
# loop through all schemata of the database
for db_schema in db_schemata:
if len(db_schema) != 5 + 1: # +1 due to manually added leading rowID
_adel_log.log("parse_db: WARNING! invalid length of database schema statement entry detected: ", 2)
_adel_log.log(str(db_schema), 2)
continue
# Reset result list for new element
result_list = []
# Parse this database element (table, index, view or trigger)
if (_helpersStringOperations.starts_with_string(str(db_schema[1]), "TABLE") == 0):
# PARSE TABLE STATEMENT
# Ensure that we treat a valid schema
db_schemata_statement = db_schema[len(db_schema) - 1]
if ((db_schemata_statement == None) or (db_schemata_statement == "")):
_adel_log.log("parse_db: WARNING! missing database schema statement entry detected, printing schema statement:", 2)
_adel_log.log(str(db_schema), 3)
continue
sql_statement = (db_schema[5]) # db_schema[5] is expected to be the "sql text" as defined in sqlite_master
_adel_log.log("\nparseDB: ----> parsing new database structure with SQL statement:", 3)
_adel_log.log(str(sql_statement), 3)
# Extract and check command (expected to be CREATE)
command_tuple = _helpersStringOperations.split_at_first_occurrence(sql_statement, " ")
if (len(command_tuple) == 0):
_adel_log.log("parse_db: WARNING! invalid sql COMMAND detected, continuing with next database element (e.g. next table)", 2)
continue
if (_helpersStringOperations.starts_with_string(str(command_tuple[0]), "CREATE") != 0):
_adel_log.log("parse_db: WARNING! invalid sql COMMAND detected, expected \"CREATE\" but found: " + str(command_tuple[0]), 2)
_adel_log.log(" continuing with next database element (e.g. next table)", 2)
continue
# Extract and check first command operand (expected to be TEMP, TEMPORARY, TABLE or VIRTUAL TABLE)
type_tuple = _helpersStringOperations.split_at_first_occurrence(command_tuple[1], " ")
if len(type_tuple) == 0:
_adel_log.log("parse_db: WARNING! invalid sql COMMAND TYPE detected, continuing with next database element (e.g. next table)", 2)
continue
# According to the syntax diagrams of the sqlite SQL create table statement there are TEMP or TEMPORARY key words allowed at this place
if (_helpersStringOperations.starts_with_string(str(type_tuple[0]), "TEMP") == 0
or _helpersStringOperations.starts_with_string(str(type_tuple[0]), "TEMPORARY") == 0
or _helpersStringOperations.starts_with_string(str(type_tuple[0]), "VIRTUAL") == 0):
# Ignore and proceed with next fragement (must then be TABLE)
type_tuple = _helpersStringOperations.split_at_first_occurrence(type_tuple[1], " ")
if len(type_tuple) == 0:
_adel_log.log("parse_db: WARNING! invalid sql COMMAND TYPE after TEMP(ORARY) detected, continuing with next database element (e.g. next table)", 2)
continue
# This fragment must be table
if (_helpersStringOperations.starts_with_string(str(type_tuple[0]), "TABLE") != 0):
_adel_log.log("parse_db: WARNING! invalid sql COMMAND TYPE detected, expected \"TABLE\" but found: " + str(type_tuple[0]), 2)
_adel_log.log(" continuing with next database element (e.g. next table)", 2)
continue
# Extract and check second command operand (expected to be table name)
name_tuple = []
next_space = _helpersStringOperations.split_at_first_occurrence(type_tuple[1], " ")
next_parenthesis = _helpersStringOperations.split_at_first_occurrence(type_tuple[1], "(")
if (next_space < next_parenthesis):
# "IF NOT EXISTS" statement possible
if (_helpersStringOperations.starts_with_string(str(_helpersStringOperations.crop_whitespace(type_tuple[1])), "IF") == 0):
type_tuple[1] = type_tuple[1][2:]
if (_helpersStringOperations.starts_with_string(str(_helpersStringOperations.crop_whitespace(type_tuple[1])), "NOT") == 0):
type_tuple[1] = type_tuple[1][3:]
if (_helpersStringOperations.starts_with_string(str(_helpersStringOperations.crop_whitespace(type_tuple[1])), "EXISTS") == 0):
type_tuple[1] = type_tuple[1][6:]
type_tuple[1] = _helpersStringOperations.crop_whitespace(type_tuple[1])
# Extract name tuple
name_tuple = _helpersStringOperations.split_at_first_occurrence(type_tuple[1], " ")
if len(name_tuple) == 0:
name_tuple = _helpersStringOperations.split_at_first_occurrence(type_tuple[1], "(")
if len(name_tuple) == 0:
_adel_log.log("parse_db: WARNING! invalid sql COMMAND TYPE NAME detected, continuing with next database element (e.g. next table)", 2)
continue
# Append leading opening parenthesis that we cut off before
name_tuple[1] = "(" + str(name_tuple[1])
else:
# "AS ..." statement possible
tmp_string = _helpersStringOperations.crop_whitespace(name_tuple[1])
if (tmp_string.startswith("AS")):
_adel_log.log("parse_db: OK - \"AS\" statement detected: " + str(tmp_string), 3)
_adel_log.log("parse_db: OK - no data stored, thus continuing with next database element (e.g. next table)", 3)
continue
else:
name_tuple = _helpersStringOperations.split_at_first_occurrence(type_tuple[1], "(")
if len(name_tuple) == 0:
_adel_log.log("parse_db: WARNING! invalid sql COMMAND TYPE NAME detected, continuing with next database element (e.g. next table)", 2)
continue
# Append leading opening parenthesis that we cut off before
name_tuple[1] = "(" + str(name_tuple[1])
# Now ready to parse TABLE
_adel_log.log("parse_db: ----> parsing database structure " + str(type_tuple[0]) + " \"" + str(name_tuple[0]) + "\"", 3)
_adel_log.log("parse_db: ----> parsing SQL statement of " + str(type_tuple[0]) + "....", 3)
_adel_log.log("parse_db: OK - SQL statement is of type: " + str(command_tuple[0]) + " " + str(type_tuple[0]), 3)
# Parse and append sql statement
name_tuple[1] = _helpersStringOperations.cut_first_last_exclude(name_tuple[1], "(", ")")
result_list.append(parse_sql_statement_params(name_tuple[1]))
# Ensure we deal with a real table, virtual tables have no b-tree and thus the b-tree root page pointer is 0
if (db_schema[4] == 0):
_adel_log.log("parse_db: OK - this table holds no content (e.g. virtual table), continuing with next database element (e.g. next table)", 3)
_adel_log.log("parse_db: ----> database structure " + str(type_tuple[0]) + " \"" + str(name_tuple[0]) + "\" parsed", 3)
# Append result from table, index, view or trigger to final list
final_list.append(result_list)
continue
# Parse and append table contents
btree_root_page_string = _sqliteFileHandler.read_page(db_schema[4])
# Ensure that read page could retrieve an existing page
if (btree_root_page_string == ""):
_adel_log.log("parse_db: ERROR - could not refer to b-tree root page: " + str(db_schema[4]), 1)
_adel_log.log(" continuing with next database element (e.g. next table)", 1)
continue
_adel_log.log("parse_db: ----> parsing contents of " + str(type_tuple[0]) + "....", 3)
table_contents = _sqlitePageParser.parse_table_btree_page(btree_root_page_string, 0)
# Check whether the table contains a dedicated row ID column
if (ROW_ID_COLUMN == 0):
# Table has no dedicated row ID column, add "rowID" to the table statement (the rowID is already extractet)
index_of_last_element_in_result_list = len(result_list) - 1
temp_list = result_list[index_of_last_element_in_result_list]
result_list[index_of_last_element_in_result_list] = [["rowID", "INTEGER"]]
for element in range(len(temp_list)):
result_list[index_of_last_element_in_result_list].append(temp_list[element])
# Append table contents to the result list
for row in table_contents:
result_list.append(row)
else:
# Table has a dedicated row ID column (integer primary key column), link values stored as row ID in the b-tree to this column (at the place of this column)
# Append table contents to the result list
for row in table_contents:
# Replace "None" entries in integer primary key column of each row through the actual row ID
row[ROW_ID_COLUMN] = row[0]
# Delete manually appended row ID column (in parse_sql_statement_params)
temp_row = row
row = []
for index in range(len(temp_row) - 1):
row.append(temp_row[index + 1])
# Append corrected row
result_list.append(row)
# Append result from table, index, view or trigger to final list
final_list.append(result_list)
_adel_log.log("parse_db: ----> database structure " + str(type_tuple[0]) + " \"" + str(name_tuple[0]) + "\" parsed", 3)
# TODO: comment out the following print statements in productive environment
#_adel_log.log("\n_sqliteParser.py:234, parse_db ----> printing database schema for " + str(type_tuple[0]) + " \"" + str(name_tuple[0]) + "\" for test purposes:", 4)
#_adel_log.log(str(db_schema[len(db_schema) - 1]), 4)
#_adel_log.log("\n_sqliteParser.py:236, parse_db ----> printing database contents for " + str(type_tuple[0]) + " \"" + str(name_tuple[0]) + "\" for test purposes:", 4)
#for result in result_list:
# _adel_log.log(str(result), 4)
# comment out the above print statements in productive environment
# PARSE INDEX STATEMENT
#if ((str(db_schema[1]) == "INDEX") or (str(db_schema[1]) == "Index") or (str(db_schema[1]) == "index")):
# TODO: implement if necessary
# IGNORED RIGHT NOW
# PARSE VIEW STATEMENT
#if ((str(db_schema[1]) == "VIEW") or (str(db_schema[1]) == "View") or (str(db_schema[1]) == "view")):
# TODO: implement if necessary
# IGNORED RIGHT NOW
# PARSE TRIGGER STATEMENT
#if ((str(db_schema[1]) == "TRIGGER") or (str(db_schema[1]) == "Trigger") or (str(db_schema[1]) == "trigger")):
# TODO: implement if necessary
# IGNORED RIGHT NOW
_adel_log.log("\nparseDB: ----> returning contents of the database file", 3)
# Close the database file
_sqliteFileHandler.close_db()
_adel_log.log("parse_db: ----> sqlite3 database file parsed", 3)
return final_list
def parse_table_btree_leaf_cell(page_hex_string, page_offset, cell_pointers, free_block_pointer):
# 1 byte is represented by two characters in the hexString, so internally we need to calculate the offset in nibbles
page_offset_in_bytes = page_offset # store for log reasons only
page_offset = page_offset * 2 # now dealing with nibbles because we treat a string (1 character = 1 nibble)
db_page_size_in_bytes = _sqliteFileHandler.DB_PAGESIZE_IN_BYTES
usable_page_space = db_page_size_in_bytes - _sqliteFileHandler.DB_RESERVED_SPACE
_adel_log.log("parse_table_btree_leaf_cell: ----> parsing b-tree leaf cell at offset %(page_offset_in_bytes)s...." % vars(), 4)
# Get total number of bytes of payload
bytes_of_payload_tuple = _sqliteVarInt.parse_next_var_int(page_hex_string[page_offset:(page_offset + 18)]) # a variable integer can be maximum 9 byte (= 18 nibbles) long
bytes_of_payload = bytes_of_payload_tuple[0]
_adel_log.log("parse_table_btree_leaf_cell: OK - payload is %(bytes_of_payload)s bytes long" % vars(), 4)
# Get row_id
row_id_string = page_hex_string[(page_offset + (bytes_of_payload_tuple[1] * 2)):(page_offset + (bytes_of_payload_tuple[1] + 9) * 2)]
row_id_tuple = _sqliteVarInt.parse_next_var_int(row_id_string)
row_id = row_id_tuple[0]
_adel_log.log("parse_table_btree_leaf_cell: ----> extracting contents for row_id %(row_id)s...." % vars(), 4)
# Check for overflow pages and append content of those pages, if any
# Calculate the overflow limits for table b-tree leaf cell
remaining_page_space = db_page_size_in_bytes - page_offset_in_bytes
if (bytes_of_payload > (remaining_page_space)):
# We expext content to overflow, because there is not enough space left on this page
_adel_log.log("parse_table_btree_leaf_cell: OK - payload is too large for this page, there are overflow pages" % vars(), 4)
# Check at which position the next cell starts
next_cell = usable_page_space
for cell_pointer in cell_pointers:
if (cell_pointer > page_offset_in_bytes) and (cell_pointer < next_cell):
next_cell = cell_pointer
# Check at which position the next freeblock starts (we ignore theoretically possible freebytes in this case,
# Because we expect no freebyte at the end of a cell that overflows to another page
next_free_block = usable_page_space
free_blocks = parse_free_blocks(page_hex_string, free_block_pointer)
for free_block in free_blocks:
if (free_block[0] > page_offset_in_bytes) and (free_block[0] < next_free_block):
next_free_block = free_block[0]
# Get the end of this record: either closest following cell or closest following freeblock or end of page
end_of_record = usable_page_space
# Check of the end of this record is given through a following cell
if (next_cell != usable_page_space) and ((next_cell <= next_free_block) or (next_free_block == usable_page_space)):
# next element is not end of page but a cell
end_of_record = next_cell
# Check of the end of this record is given through a following free block
if (next_free_block != usable_page_space) and ((next_free_block < next_cell) or (next_cell == usable_page_space)):
# Next element is not end of page but a free block
end_of_record = next_free_block
# Cut record hex string from the beginning to the offset of the next following element
record_hex_string = page_hex_string[(page_offset + ((bytes_of_payload_tuple[1] + row_id_tuple[1]) * 2)):(end_of_record * 2)]
record_hex_string_length = len(record_hex_string) / 2 # string length is count in nibbles, we need bytes here
# Save overflow page pointer at the end of record hex string
first_overflow_page_number = int(record_hex_string[((record_hex_string_length - 4) * 2):(record_hex_string_length * 2)], 16)
_adel_log.log("parse_table_btree_leaf_cell: ----> parsing overflow page chain beginning at page %(first_overflow_page_number)s...." % vars(), 4)
# Cut off overflow page number from record_hex_string
record_hex_string = record_hex_string[(0):((record_hex_string_length - 4) * 2)]
first_overflow_page_string = _sqliteFileHandler.read_page(first_overflow_page_number)
# Ensure that read page could retrieve an existing page
if (first_overflow_page_string == ""):
_adel_log.log("parse_table_btree_leaf_cell: ERROR - invalid overflow page pointer, cannot reference first overflow page: " + str(first_overflow_page_number), 1)
return []
# Append content from overflow pages
record_hex_string += parse_overflow_page_chain(first_overflow_page_string)
# Ensure correct length of string (maybe not all bytes of the last overflow page in the chain contain content)
record_hex_string_length = len(record_hex_string) / 2 # string length is count in nibbles, we need bytes here
if (bytes_of_payload < record_hex_string_length):
# Cut record hex string again
record_hex_string = record_hex_string[:bytes_of_payload * 2]
else:
# The entire payload is stored on this page
record_hex_string = page_hex_string[(page_offset + ((bytes_of_payload_tuple[1] + row_id_tuple[1]) * 2)):(page_offset + ((bytes_of_payload_tuple[1] + row_id_tuple[1] + bytes_of_payload_tuple[0]) * 2))]
# Parse the record
read_content_list = parse_record(record_hex_string)
# Build the resulting list (including the row_id used sqlite internally)
cell_content_list = []
cell_content_list.append(row_id)
for element in range(len(read_content_list)):
cell_content_list.append(read_content_list[element])
# Return results
_adel_log.log("parse_table_btree_leaf_cell: OK - returning list of cell contents", 4)
_adel_log.log("parse_table_btree_leaf_cell: ----> b-tree leaf cell at offset %(page_offset_in_bytes)s parsed" % vars(), 4)
return cell_content_list
