def writer_builder_csvpool(self):
return ObjectBuilder(
class_name=ProcessWriter,
constructor_params={
'next_writer_builder':
ObjectBuilder(
class_name=CSVWriter,
constructor_params={
'csv_file_path':
self.config['writer']['file']['csv_file_path'],
'csv_file_path_prefix':
self.config['writer']['file']['csv_file_path_prefix'],
'csv_file_path_suffix_parts':
self.config['writer']['file']
['csv_file_path_suffix_parts'],
'csv_keep_file':
self.config['writer']['file']['csv_keep_file'],
'dst_schema':
self.config['writer']['file']['dst_schema'],
'dst_table':
self.config['writer']['file']['dst_table'],
'next_writer_builder':
ObjectBuilder(class_name=CHCSVWriter,
constructor_params=self.config['writer']
['clickhouse']),
'converter_builder':
self.converter_builder(CONVERTER_CSV),
})
})
def writer_builder_chwriter(self):
return ObjectBuilder(
class_name=CHWriter,
constructor_params={
'connection_settings': {
'host':
self.config['writer']['clickhouse']['connection_settings']
['host'],
'port':
self.config['writer']['clickhouse']['connection_settings']
['port'],
'user':
self.config['writer']['clickhouse']['connection_settings']
['user'],
'password':
self.config['writer']['clickhouse']['connection_settings']
['password'],
},
'dst_schema':
self.config['writer']['clickhouse']['dst_schema'],
'dst_table':
self.config['writer']['clickhouse']['dst_table'],
'dst_distribute':
self.config['writer']['clickhouse']['dst_distribute'],
'next_writer_builder':
None,
'converter_builder':
self.converter_builder(CONVERTER_CH),
})
def converter_builder(self, which):
if which == CONVERTER_CSV:
return ObjectBuilder(
instance=CSVWriteConverter(
defaults=self.config['converter']['csv']['column_default_value'],
column_skip=self.config['converter']['csv']['column_skip'],
))
elif which == CONVERTER_CH:
if not self.config['converter']['clickhouse']['converter_file'] or not self.config['converter']['clickhouse']['converter_class']:
# default converter
return ObjectBuilder(instance=CHWriteConverter(column_skip=self.config['converter']['clickhouse']['column_skip']))
else:
# explicitly specified converter
_class = Util.class_from_file(
self.config['converter']['clickhouse']['converter_file'],
self.config['converter']['clickhouse']['converter_class']
)
return ObjectBuilder(instance=_class(column_skip=self.config['converter']['clickhouse']['column_skip']))
def __init__(
self,
writer_builder=None,
key_builder=None,
max_bucket_size=10000,
max_belt_size=1,
max_interval_between_rotations=60,
):
super().__init__(
writer_builder=writer_builder,
key_builder=ObjectBuilder(class_name=BBIndexGenerator),
max_bucket_size=max_bucket_size,
max_belt_size=max_belt_size,
max_interval_between_rotations=max_interval_between_rotations,
)