def test_sigle_block_with_padding(self):
"""Test stripping away empty csv lins
"""
test_data = [
["", ""],
["hello", "world"],
["hello", "world", "uet"],
["hello", "world"],
["hello", "world"],
]
offests, data_blocks = zip(*import_helper.split_array(test_data))
self.assertEqual(len(data_blocks), 1)
self.assertEqual(data_blocks[0], test_data[1:])
self.assertEqual(offests[0], 1)
test_data = [
["", ""],
["", ""],
["", ""],
["hello", "world"],
["hello", "world", "uet"],
["hello", "world"],
["hello", "world"],
["", ""],
["", ""],
]
offests, data_blocks = zip(*import_helper.split_array(test_data))
self.assertEqual(len(data_blocks), 1)
self.assertEqual(data_blocks[0], test_data[3:7])
self.assertEqual(offests[0], 3)
def test_sigle_block_with_padding(self):
"""Test stripping away empty csv lins
"""
test_data = [
["", ""],
["Object type", "world"],
["hello", "world", "uet"],
["hello", "world"],
["hello", "world"],
]
offsets, data_blocks, _ = zip(*import_helper.split_array(test_data))
self.assertEqual(len(data_blocks), 1)
self.assertEqual(data_blocks[0], test_data[1:])
self.assertEqual(offsets[0], 1)
test_data = [
["", ""],
["", ""],
["", ""],
["Object type", "world"],
["hello", "world", "uet"],
["hello", "world"],
["hello", "world"],
["", ""],
["", ""],
]
offsets, data_blocks, _ = zip(*import_helper.split_array(test_data))
self.assertEqual(len(data_blocks), 1)
self.assertEqual(data_blocks[0], test_data[3:7])
self.assertEqual(offsets[0], 3)
def initialize_block_converters(self):
""" Initialize block converters.
Prepare BlockConverters and order them like specified in self.CLASS_ORDER.
"""
offsets, data_blocks = split_array(self.csv_data)
for offset, data in zip(offsets, data_blocks):
if len(data) < 2:
continue # empty block
class_name = data[1][0].strip().lower()
object_class = self.exportable.get(class_name)
raw_headers, rows = extract_relevant_data(data)
block_converter = BlockConverter(self,
object_class=object_class,
rows=rows,
raw_headers=raw_headers,
offset=offset,
class_name=class_name)
block_converter.check_block_restrictions()
self.block_converters.append(block_converter)
order = defaultdict(int)
order.update({c: i for i, c in enumerate(self.CLASS_ORDER)})
order["Person"] = -1
self.block_converters.sort(key=lambda x: order[x.name])
def test_multiple_blocks(self):
"""Test splitting blocks of csv file
Test that split_array function splits a csv file by one or more empty
lines. The lines with only empty strings represent comma only lines in a
read csv file.
"""
test_data = [
["", ""],
["hello", "world"],
["hello", "world", "uet"],
["", ""],
["hello", "world"],
["hello", "world"],
]
offests, data_blocks = zip(*import_helper.split_array(test_data))
self.assertEqual(len(data_blocks), 2)
self.assertEqual(data_blocks[0], test_data[1:3])
self.assertEqual(data_blocks[1], test_data[4:6])
self.assertEqual(offests[0], 1)
self.assertEqual(offests[1], 4)
test_data = [
["", ""],
["hello", "world"],
["hello", "world", "uet"],
["hello", "world"],
["", ""],
["", ""],
["hello", "world"],
["", ""],
["", ""],
["hello", "world"],
["hello", "world"],
["hello", "world"],
]
offests, data_blocks = zip(*import_helper.split_array(test_data))
self.assertEqual(len(data_blocks), 3)
self.assertEqual(data_blocks[0], test_data[1:4])
self.assertEqual(data_blocks[1], test_data[6:7])
self.assertEqual(data_blocks[2], test_data[9:])
self.assertEqual(offests[0], 1)
self.assertEqual(offests[1], 6)
self.assertEqual(offests[2], 9)
def test_multiple_blocks(self):
"""Test splitting blocks of csv file
Test that split_array function splits a csv file by one or more empty
lines. The lines with only empty strings represent comma only lines in a
read csv file.
"""
test_data = [
["", ""],
["hello", "world"],
["hello", "world", "uet"],
["", ""],
["hello", "world"],
["hello", "world"],
]
offests, data_blocks = import_helper.split_array(test_data)
self.assertEqual(len(data_blocks), 2)
self.assertEqual(data_blocks[0], test_data[1:3])
self.assertEqual(data_blocks[1], test_data[4:6])
self.assertEqual(offests[0], 1)
self.assertEqual(offests[1], 4)
test_data = [
["", ""],
["hello", "world"],
["hello", "world", "uet"],
["hello", "world"],
["", ""],
["", ""],
["hello", "world"],
["", ""],
["", ""],
["hello", "world"],
["hello", "world"],
["hello", "world"],
]
offests, data_blocks = import_helper.split_array(test_data)
self.assertEqual(len(data_blocks), 3)
self.assertEqual(data_blocks[0], test_data[1:4])
self.assertEqual(data_blocks[1], test_data[6:7])
self.assertEqual(data_blocks[2], test_data[9:])
self.assertEqual(offests[0], 1)
self.assertEqual(offests[1], 6)
self.assertEqual(offests[2], 9)
def test_multiple_blocks(self):
"""Test splitting blocks of csv file
Test that split_array function splits a csv file by lines beginning with
"Object type" cell
"""
test_data = [
["", ""],
["Object type", "foo"],
["hello", "world", "uet"],
["", ""],
["Object type", "bar"],
["hello", "world"],
]
offsets, data_blocks, _ = zip(*import_helper.split_array(test_data))
self.assertEqual(len(data_blocks), 2)
self.assertEqual(data_blocks[0], test_data[1:3])
self.assertEqual(data_blocks[1], test_data[4:6])
self.assertEqual(offsets[0], 1)
self.assertEqual(offsets[1], 4)
test_data = [
["", ""],
["Object type", "world"],
["hello", "world", "uet"],
["hello", "world"],
["", ""],
["", ""],
["Object type", "foo"],
["", ""],
["", ""],
["Object type", "bar"],
["hello", "world"],
["hello", "world"],
]
offsets, data_blocks, _ = zip(*import_helper.split_array(test_data))
self.assertEqual(len(data_blocks), 3)
self.assertEqual(data_blocks[0], test_data[1:4])
self.assertEqual(data_blocks[1], test_data[6:7])
self.assertEqual(data_blocks[2], test_data[9:])
self.assertEqual(offsets[0], 1)
self.assertEqual(offsets[1], 6)
self.assertEqual(offsets[2], 9)
def test_multiple_blocks(self):
"""Test splitting blocks of csv file
Test that split_array function splits a csv file by lines beginning with
"Object type" cell
"""
test_data = [
["", ""],
["Object type", "foo"],
["hello", "world", "uet"],
["", ""],
["Object type", "bar"],
["hello", "world"],
]
offsets, data_blocks, _ = zip(*import_helper.split_array(test_data))
self.assertEqual(len(data_blocks), 2)
self.assertEqual(data_blocks[0], test_data[1:3])
self.assertEqual(data_blocks[1], test_data[4:6])
self.assertEqual(offsets[0], 1)
self.assertEqual(offsets[1], 4)
test_data = [
["", ""],
["Object type", "world"],
["hello", "world", "uet"],
["hello", "world"],
["", ""],
["", ""],
["Object type", "foo"],
["", ""],
["", ""],
["Object type", "bar"],
["hello", "world"],
["hello", "world"],
]
offsets, data_blocks, _ = zip(*import_helper.split_array(test_data))
self.assertEqual(len(data_blocks), 3)
self.assertEqual(data_blocks[0], test_data[1:4])
self.assertEqual(data_blocks[1], test_data[6:7])
self.assertEqual(data_blocks[2], test_data[9:])
self.assertEqual(offsets[0], 1)
self.assertEqual(offsets[1], 6)
self.assertEqual(offsets[2], 9)
def test_sigle_block(self):
"""Test splitting of a single csv block
The array reprisents a read csv file with no lines that contain only
commas.
"""
test_data = [
["hello", "world"],
["hello", "world"],
["hello", "world"],
]
offests, data_blocks = zip(*import_helper.split_array(test_data))
self.assertEqual(len(data_blocks), 1)
self.assertEqual(data_blocks[0], test_data)
self.assertEqual(offests[0], 0)
def test_sigle_block(self):
"""Test splitting of a single csv block
The array represents a read csv file with no lines that contain only
commas.
"""
test_data = [
["Object type", "world"],
["hello", "world"],
["hello", "world"],
]
offsets, data_blocks, _ = zip(*import_helper.split_array(test_data))
self.assertEqual(len(data_blocks), 1)
self.assertEqual(data_blocks[0], test_data)
self.assertEqual(offsets[0], 0)
def block_converters_from_csv(self):
"""Prepare BlockConverters and order them like specified in
self.CLASS_ORDER.
"""
offsets, data_blocks = split_array(self.csv_data)
for offset, data in zip(offsets, data_blocks):
if len(data) < 2:
continue # empty block
class_name = data[1][0].strip().lower()
object_class = self.exportable.get(class_name)
raw_headers, rows = extract_relevant_data(data)
block_converter = BlockConverter(self, object_class=object_class,
rows=rows, raw_headers=raw_headers,
offset=offset, class_name=class_name)
block_converter.check_block_restrictions()
self.block_converters.append(block_converter)
order = defaultdict(int)
order.update({c: i for i, c in enumerate(self.CLASS_ORDER)})
order["Person"] = -1
self.block_converters.sort(key=lambda x: order[x.name])
