def _single_length_title(self, table, row_index, current_col):
'''
Returns true if the row is a single length title element with no other row titles. Useful
for tracking pre-data titles that belong in their own block.
'''
if len(table[row_index]) - current_col <= 0:
return False
return (is_text_cell(table[row_index][current_col]) and all(
not is_text_cell(table[row_index][next_column])
for next_column in xrange(current_col + 1, len(table[row_index]))))
def _single_length_title(self, table, row_index, current_col):
'''
Returns true if the row is a single length title element with no other row titles. Useful
for tracking pre-data titles that belong in their own block.
'''
if len(table[row_index]) - current_col <= 0:
return False
return (is_text_cell(table[row_index][current_col]) and
all(not is_text_cell(table[row_index][next_column])
for next_column in xrange(current_col + 1, len(table[row_index]))))
def _find_block_start(self, table, used_cells, possible_block_start,
start_pos, end_pos):
'''
Finds the start of a block from a suggested start location.
This location can be at a lower column but not a lower row.
Note this also finds the lowest row of block_end.
'''
current_col = possible_block_start[1]
block_start = list(possible_block_start)
block_end = list(possible_block_start)
repeat = True
checked_all = False
# Repeat until we've met satisfactory conditions for
# catching all edge cases or we've checked all valid
# block locations
while(not checked_all and repeat):
block_end[0] = max(block_end[0], possible_block_start[0])
block_end[1] = max(block_end[1], current_col)
table_column = TableTranspose(table)[current_col]
used_column = TableTranspose(used_cells)[current_col]
# We need to find a non empty cell before we can stop
blank_start = is_empty_cell(table_column[possible_block_start[0]])
# Unless we have assume_complete_blocks set to True
if blank_start and self.assume_complete_blocks:
# Found a blank? We're done
repeat = False
break
blank_exited = not blank_start
blank_repeat_threshold = 3
parent_title = blank_start or is_text_cell(table_column[possible_block_start[0]])
#TODO refactor code below into new function for easier reading
# Analyze the beginning columns
for row_index in range(possible_block_start[0], end_pos[0]+1):
# Ensure we catch the edge case of the data reaching the edge of
# the table -- block_end should then equal end_pos
if blank_exited:
block_end[0] = max(block_end[0], row_index)
if row_index == end_pos[0] or used_column[row_index]:
# We've gone through the whole range
checked_all = True
break
elif not blank_exited:
blank_exited = not is_empty_cell(table_column[row_index])
elif is_empty_cell(table_column[row_index]):
current_col += 1
break
else:
# Go find the left most column that's still valid
table_row = table[row_index]
used_row = used_cells[row_index]
for column_index in range(current_col, start_pos[1]-1, -1):
if is_empty_cell(table_row[column_index]) or used_row[column_index]:
break
else:
block_start[1] = min(block_start[1], column_index)
# Check if we've seen few enough cells to guess that we have a repeating title
repeat = blank_start or 1+row_index-possible_block_start[0] <= blank_repeat_threshold
return block_start, block_end
def _fill_column_holes(self):
'''
Same as _fill_row_holes but for columns.
'''
for column_index in range(self.start[1], self.end[1]):
table_column = TableTranspose(self.table)[column_index]
column_start = table_column[self.start[0]]
if is_text_cell(column_start):
self._check_fill_title_column(column_index)
def _fill_column_holes(self):
'''
Same as _fill_row_holes but for columns.
'''
for column_index in range(self.start[1], self.end[1]):
table_column = TableTranspose(self.table)[column_index]
column_start = table_column[self.start[0]]
if is_text_cell(column_start):
self._check_fill_title_column(column_index)
def _fill_row_holes(self):
'''
Fill any remaining row title cells that are empty. This must be done after the other passes
to avoid preemptively filling in empty cells reserved for other operations.
'''
for row_index in range(self.start[0], self.max_title_row):
table_row = self.table[row_index]
row_start = table_row[self.start[1]]
if is_text_cell(row_start):
self._check_fill_title_row(row_index)
def _fill_row_holes(self):
'''
Fill any remaining row title cells that are empty. This must be done after the other passes
to avoid preemptively filling in empty cells reserved for other operations.
'''
for row_index in range(self.start[0], self.max_title_row):
table_row = self.table[row_index]
row_start = table_row[self.start[1]]
if is_text_cell(row_start):
self._check_fill_title_row(row_index)
def _fill_row_holes(self):
'''
Fill any remaining row title cells that are empty.
This must be done after the other passes to avoid
preemptively filling in empty cells reserved for
other operations.
TODO potential optimization, mark rows/columns that
were already processed and skip them here.
'''
for row_index in range(self.start[0], self.end[0]):
table_row = self.table[row_index]
row_start = table_row[self.start[1]]
if is_text_cell(row_start):
self._check_fill_title_row(row_index)
def _fill_row_holes(self):
'''
Fill any remaining row title cells that are empty.
This must be done after the other passes to avoid
preemptively filling in empty cells reserved for
other operations.
TODO potential optimization, mark rows/columns that
were already processed and skip them here.
'''
for row_index in range(self.start[0], self.end[0]):
table_row = self.table[row_index]
row_start = table_row[self.start[1]]
if is_text_cell(row_start):
self._check_fill_title_row(row_index)
def _check_interpret_cell(self, cell, prior_cell, row_index, column_index):
'''
Helper function which checks cell type and performs cell translation to strings where
necessary.
Returns:
A tuple of the form '(cell, changed)' where 'changed' indicates if 'cell' differs from
input.
'''
changed = False
if (not is_empty_cell(cell) and
not is_text_cell(cell)):
self.flag_change(self.flags, 'interpreted', (row_index, column_index),
self.worksheet, self.FLAGS['converted-to-string'])
cell = str(cell)
changed = True
# If we find a blank cell, propagate the prior title
elif is_empty_cell(cell):
self.flag_change(self.flags, 'interpreted', (row_index, column_index),
self.worksheet, self.FLAGS['copied-title'])
cell = prior_cell
changed = True
return cell, changed
def _check_interpret_cell(self, cell, prior_cell, row_index, column_index):
'''
Helper function which checks cell type and performs cell translation to strings where
necessary.
Returns:
A tuple of the form '(cell, changed)' where 'changed' indicates if 'cell' differs from
input.
'''
changed = False
if (not is_empty_cell(cell) and not is_text_cell(cell)):
self.flag_change(self.flags, 'interpreted',
(row_index, column_index), self.worksheet,
self.FLAGS['converted-to-string'])
cell = str(cell)
changed = True
# If we find a blank cell, propagate the prior title
elif is_empty_cell(cell):
self.flag_change(self.flags, 'interpreted',
(row_index, column_index), self.worksheet,
self.FLAGS['copied-title'])
cell = prior_cell
changed = True
return cell, changed
def _find_block_start(self, table, used_cells, possible_block_start,
start_pos, end_pos):
'''
Finds the start of a block from a suggested start location.
This location can be at a lower column but not a lower row.
Note this also finds the lowest row of block_end.
'''
current_col = possible_block_start[1]
block_start = list(possible_block_start)
block_end = list(possible_block_start)
repeat = True
checked_all = False
# Repeat until we've met satisfactory conditions for
# catching all edge cases or we've checked all valid
# block locations
while (not checked_all and repeat):
block_end[0] = max(block_end[0], possible_block_start[0])
block_end[1] = max(block_end[1], current_col)
table_column = TableTranspose(table)[current_col]
used_column = TableTranspose(used_cells)[current_col]
# We need to find a non empty cell before we can stop
blank_start = is_empty_cell(table_column[possible_block_start[0]])
# Unless we have assume_complete_blocks set to True
if blank_start and self.assume_complete_blocks:
# Found a blank? We're done
repeat = False
break
blank_exited = not blank_start
blank_repeat_threshold = 3
parent_title = blank_start or is_text_cell(
table_column[possible_block_start[0]])
#TODO refactor code below into new function for easier reading
# Analyze the beginning columns
for row_index in range(possible_block_start[0], end_pos[0] + 1):
# Ensure we catch the edge case of the data reaching the edge of
# the table -- block_end should then equal end_pos
if blank_exited:
block_end[0] = max(block_end[0], row_index)
if row_index == end_pos[0] or used_column[row_index]:
# We've gone through the whole range
checked_all = True
break
elif not blank_exited:
blank_exited = not is_empty_cell(table_column[row_index])
elif is_empty_cell(table_column[row_index]):
current_col += 1
break
else:
# Go find the left most column that's still valid
table_row = table[row_index]
used_row = used_cells[row_index]
for column_index in range(current_col, start_pos[1] - 1,
-1):
if is_empty_cell(table_row[column_index]
) or used_row[column_index]:
break
else:
block_start[1] = min(block_start[1], column_index)
# Check if we've seen few enough cells to guess that we have a repeating title
repeat = blank_start or 1 + row_index - possible_block_start[
0] <= blank_repeat_threshold
return block_start, block_end