def test_read_bulks_and_assume_metadata_line_in_source_file(self): data = [ '{"index": {"_index": "test_index", "_type": "test_type"}', '{"key": "value1"}', '{"index": {"_index": "test_index", "_type": "test_type"}', '{"key": "value2"}', '{"index": {"_index": "test_index", "_type": "test_type"}', '{"key": "value3"}', '{"index": {"_index": "test_index", "_type": "test_type"}', '{"key": "value4"}', '{"index": {"_index": "test_index", "_type": "test_type"}', '{"key": "value5"}', '{"index": {"_index": "test_index", "_type": "test_type"}', '{"key": "value6"}', '{"index": {"_index": "test_index", "_type": "test_type"}', '{"key": "value7"}' ] bulk_size = 3 source = params.Slice(io.StringAsFileSource, 0, len(data)) am_handler = params.SourceActionMetaData(source) reader = params.IndexDataReader(data, batch_size=bulk_size, bulk_size=bulk_size, file_source=source, action_metadata=am_handler, index_name="test_index", type_name="test_type") expected_bulk_sizes = [3, 3, 1] # lines should include meta-data expected_line_sizes = [6, 6, 2] self.assert_bulks_sized(reader, expected_bulk_sizes, expected_line_sizes)
def test_read_bulks_and_assume_metadata_line_in_source_file(self): data = [ '{"index": {"_index": "test_index", "_type": "test_type"}', '{"key": "value1"}', '{"index": {"_index": "test_index", "_type": "test_type"}', '{"key": "value2"}', '{"index": {"_index": "test_index", "_type": "test_type"}', '{"key": "value3"}', '{"index": {"_index": "test_index", "_type": "test_type"}', '{"key": "value4"}', '{"index": {"_index": "test_index", "_type": "test_type"}', '{"key": "value5"}', '{"index": {"_index": "test_index", "_type": "test_type"}', '{"key": "value6"}', '{"index": {"_index": "test_index", "_type": "test_type"}', '{"key": "value7"}' ] bulk_size = 3 source = params.Slice(StringAsFileSource, 0, len(data)) am_handler = params.SourceActionMetaData(source) reader = params.IndexDataReader(data, batch_size=bulk_size, bulk_size=bulk_size, file_source=source, action_metadata=am_handler, index_name="test_index", type_name="test_type") # always double the amount as one line contains the data and one line contains the index command expected_bulk_sizes = [6, 6, 2] self.assert_bulks_sized(reader, expected_bulk_sizes)
def test_source_file_action_meta_data(self): source = params.Slice(io.StringAsFileSource, 0, 5) generator = params.SourceActionMetaData(source) data = [ '{"index": {"_index": "test_index", "_type": "test_type", "_id": "1"}}', '{"index": {"_index": "test_index", "_type": "test_type", "_id": "2"}}', '{"index": {"_index": "test_index", "_type": "test_type", "_id": "3"}}', '{"index": {"_index": "test_index", "_type": "test_type", "_id": "4"}}', '{"index": {"_index": "test_index", "_type": "test_type", "_id": "5"}}', ] source.open(data, "r") self.assertEqual(data, list(generator)) source.close()