def test_sigle_block_with_padding(self): """Test stripping away empty csv lins """ test_data = [ ["", ""], ["hello", "world"], ["hello", "world", "uet"], ["hello", "world"], ["hello", "world"], ] offests, data_blocks = zip(*import_helper.split_array(test_data)) self.assertEqual(len(data_blocks), 1) self.assertEqual(data_blocks[0], test_data[1:]) self.assertEqual(offests[0], 1) test_data = [ ["", ""], ["", ""], ["", ""], ["hello", "world"], ["hello", "world", "uet"], ["hello", "world"], ["hello", "world"], ["", ""], ["", ""], ] offests, data_blocks = zip(*import_helper.split_array(test_data)) self.assertEqual(len(data_blocks), 1) self.assertEqual(data_blocks[0], test_data[3:7]) self.assertEqual(offests[0], 3)
def test_sigle_block_with_padding(self): """Test stripping away empty csv lins """ test_data = [ ["", ""], ["Object type", "world"], ["hello", "world", "uet"], ["hello", "world"], ["hello", "world"], ] offsets, data_blocks, _ = zip(*import_helper.split_array(test_data)) self.assertEqual(len(data_blocks), 1) self.assertEqual(data_blocks[0], test_data[1:]) self.assertEqual(offsets[0], 1) test_data = [ ["", ""], ["", ""], ["", ""], ["Object type", "world"], ["hello", "world", "uet"], ["hello", "world"], ["hello", "world"], ["", ""], ["", ""], ] offsets, data_blocks, _ = zip(*import_helper.split_array(test_data)) self.assertEqual(len(data_blocks), 1) self.assertEqual(data_blocks[0], test_data[3:7]) self.assertEqual(offsets[0], 3)
def initialize_block_converters(self): """ Initialize block converters. Prepare BlockConverters and order them like specified in self.CLASS_ORDER. """ offsets, data_blocks = split_array(self.csv_data) for offset, data in zip(offsets, data_blocks): if len(data) < 2: continue # empty block class_name = data[1][0].strip().lower() object_class = self.exportable.get(class_name) raw_headers, rows = extract_relevant_data(data) block_converter = BlockConverter(self, object_class=object_class, rows=rows, raw_headers=raw_headers, offset=offset, class_name=class_name) block_converter.check_block_restrictions() self.block_converters.append(block_converter) order = defaultdict(int) order.update({c: i for i, c in enumerate(self.CLASS_ORDER)}) order["Person"] = -1 self.block_converters.sort(key=lambda x: order[x.name])
def test_multiple_blocks(self): """Test splitting blocks of csv file Test that split_array function splits a csv file by one or more empty lines. The lines with only empty strings represent comma only lines in a read csv file. """ test_data = [ ["", ""], ["hello", "world"], ["hello", "world", "uet"], ["", ""], ["hello", "world"], ["hello", "world"], ] offests, data_blocks = zip(*import_helper.split_array(test_data)) self.assertEqual(len(data_blocks), 2) self.assertEqual(data_blocks[0], test_data[1:3]) self.assertEqual(data_blocks[1], test_data[4:6]) self.assertEqual(offests[0], 1) self.assertEqual(offests[1], 4) test_data = [ ["", ""], ["hello", "world"], ["hello", "world", "uet"], ["hello", "world"], ["", ""], ["", ""], ["hello", "world"], ["", ""], ["", ""], ["hello", "world"], ["hello", "world"], ["hello", "world"], ] offests, data_blocks = zip(*import_helper.split_array(test_data)) self.assertEqual(len(data_blocks), 3) self.assertEqual(data_blocks[0], test_data[1:4]) self.assertEqual(data_blocks[1], test_data[6:7]) self.assertEqual(data_blocks[2], test_data[9:]) self.assertEqual(offests[0], 1) self.assertEqual(offests[1], 6) self.assertEqual(offests[2], 9)
def test_multiple_blocks(self): """Test splitting blocks of csv file Test that split_array function splits a csv file by one or more empty lines. The lines with only empty strings represent comma only lines in a read csv file. """ test_data = [ ["", ""], ["hello", "world"], ["hello", "world", "uet"], ["", ""], ["hello", "world"], ["hello", "world"], ] offests, data_blocks = import_helper.split_array(test_data) self.assertEqual(len(data_blocks), 2) self.assertEqual(data_blocks[0], test_data[1:3]) self.assertEqual(data_blocks[1], test_data[4:6]) self.assertEqual(offests[0], 1) self.assertEqual(offests[1], 4) test_data = [ ["", ""], ["hello", "world"], ["hello", "world", "uet"], ["hello", "world"], ["", ""], ["", ""], ["hello", "world"], ["", ""], ["", ""], ["hello", "world"], ["hello", "world"], ["hello", "world"], ] offests, data_blocks = import_helper.split_array(test_data) self.assertEqual(len(data_blocks), 3) self.assertEqual(data_blocks[0], test_data[1:4]) self.assertEqual(data_blocks[1], test_data[6:7]) self.assertEqual(data_blocks[2], test_data[9:]) self.assertEqual(offests[0], 1) self.assertEqual(offests[1], 6) self.assertEqual(offests[2], 9)
def test_multiple_blocks(self): """Test splitting blocks of csv file Test that split_array function splits a csv file by lines beginning with "Object type" cell """ test_data = [ ["", ""], ["Object type", "foo"], ["hello", "world", "uet"], ["", ""], ["Object type", "bar"], ["hello", "world"], ] offsets, data_blocks, _ = zip(*import_helper.split_array(test_data)) self.assertEqual(len(data_blocks), 2) self.assertEqual(data_blocks[0], test_data[1:3]) self.assertEqual(data_blocks[1], test_data[4:6]) self.assertEqual(offsets[0], 1) self.assertEqual(offsets[1], 4) test_data = [ ["", ""], ["Object type", "world"], ["hello", "world", "uet"], ["hello", "world"], ["", ""], ["", ""], ["Object type", "foo"], ["", ""], ["", ""], ["Object type", "bar"], ["hello", "world"], ["hello", "world"], ] offsets, data_blocks, _ = zip(*import_helper.split_array(test_data)) self.assertEqual(len(data_blocks), 3) self.assertEqual(data_blocks[0], test_data[1:4]) self.assertEqual(data_blocks[1], test_data[6:7]) self.assertEqual(data_blocks[2], test_data[9:]) self.assertEqual(offsets[0], 1) self.assertEqual(offsets[1], 6) self.assertEqual(offsets[2], 9)
def test_multiple_blocks(self): """Test splitting blocks of csv file Test that split_array function splits a csv file by lines beginning with "Object type" cell """ test_data = [ ["", ""], ["Object type", "foo"], ["hello", "world", "uet"], ["", ""], ["Object type", "bar"], ["hello", "world"], ] offsets, data_blocks, _ = zip(*import_helper.split_array(test_data)) self.assertEqual(len(data_blocks), 2) self.assertEqual(data_blocks[0], test_data[1:3]) self.assertEqual(data_blocks[1], test_data[4:6]) self.assertEqual(offsets[0], 1) self.assertEqual(offsets[1], 4) test_data = [ ["", ""], ["Object type", "world"], ["hello", "world", "uet"], ["hello", "world"], ["", ""], ["", ""], ["Object type", "foo"], ["", ""], ["", ""], ["Object type", "bar"], ["hello", "world"], ["hello", "world"], ] offsets, data_blocks, _ = zip(*import_helper.split_array(test_data)) self.assertEqual(len(data_blocks), 3) self.assertEqual(data_blocks[0], test_data[1:4]) self.assertEqual(data_blocks[1], test_data[6:7]) self.assertEqual(data_blocks[2], test_data[9:]) self.assertEqual(offsets[0], 1) self.assertEqual(offsets[1], 6) self.assertEqual(offsets[2], 9)
def test_sigle_block(self): """Test splitting of a single csv block The array reprisents a read csv file with no lines that contain only commas. """ test_data = [ ["hello", "world"], ["hello", "world"], ["hello", "world"], ] offests, data_blocks = zip(*import_helper.split_array(test_data)) self.assertEqual(len(data_blocks), 1) self.assertEqual(data_blocks[0], test_data) self.assertEqual(offests[0], 0)
def test_sigle_block(self): """Test splitting of a single csv block The array represents a read csv file with no lines that contain only commas. """ test_data = [ ["Object type", "world"], ["hello", "world"], ["hello", "world"], ] offsets, data_blocks, _ = zip(*import_helper.split_array(test_data)) self.assertEqual(len(data_blocks), 1) self.assertEqual(data_blocks[0], test_data) self.assertEqual(offsets[0], 0)
def block_converters_from_csv(self): """Prepare BlockConverters and order them like specified in self.CLASS_ORDER. """ offsets, data_blocks = split_array(self.csv_data) for offset, data in zip(offsets, data_blocks): if len(data) < 2: continue # empty block class_name = data[1][0].strip().lower() object_class = self.exportable.get(class_name) raw_headers, rows = extract_relevant_data(data) block_converter = BlockConverter(self, object_class=object_class, rows=rows, raw_headers=raw_headers, offset=offset, class_name=class_name) block_converter.check_block_restrictions() self.block_converters.append(block_converter) order = defaultdict(int) order.update({c: i for i, c in enumerate(self.CLASS_ORDER)}) order["Person"] = -1 self.block_converters.sort(key=lambda x: order[x.name])