def writer_builder_csvpool(self): return ObjectBuilder( class_name=ProcessWriter, constructor_params={ 'next_writer_builder': ObjectBuilder( class_name=CSVWriter, constructor_params={ 'csv_file_path': self.config['writer']['file']['csv_file_path'], 'csv_file_path_prefix': self.config['writer']['file']['csv_file_path_prefix'], 'csv_file_path_suffix_parts': self.config['writer']['file'] ['csv_file_path_suffix_parts'], 'csv_keep_file': self.config['writer']['file']['csv_keep_file'], 'dst_schema': self.config['writer']['file']['dst_schema'], 'dst_table': self.config['writer']['file']['dst_table'], 'next_writer_builder': ObjectBuilder(class_name=CHCSVWriter, constructor_params=self.config['writer'] ['clickhouse']), 'converter_builder': self.converter_builder(CONVERTER_CSV), }) })
def writer_builder_chwriter(self): return ObjectBuilder( class_name=CHWriter, constructor_params={ 'connection_settings': { 'host': self.config['writer']['clickhouse']['connection_settings'] ['host'], 'port': self.config['writer']['clickhouse']['connection_settings'] ['port'], 'user': self.config['writer']['clickhouse']['connection_settings'] ['user'], 'password': self.config['writer']['clickhouse']['connection_settings'] ['password'], }, 'dst_schema': self.config['writer']['clickhouse']['dst_schema'], 'dst_table': self.config['writer']['clickhouse']['dst_table'], 'dst_distribute': self.config['writer']['clickhouse']['dst_distribute'], 'next_writer_builder': None, 'converter_builder': self.converter_builder(CONVERTER_CH), })
def converter_builder(self, which): if which == CONVERTER_CSV: return ObjectBuilder( instance=CSVWriteConverter( defaults=self.config['converter']['csv']['column_default_value'], column_skip=self.config['converter']['csv']['column_skip'], )) elif which == CONVERTER_CH: if not self.config['converter']['clickhouse']['converter_file'] or not self.config['converter']['clickhouse']['converter_class']: # default converter return ObjectBuilder(instance=CHWriteConverter(column_skip=self.config['converter']['clickhouse']['column_skip'])) else: # explicitly specified converter _class = Util.class_from_file( self.config['converter']['clickhouse']['converter_file'], self.config['converter']['clickhouse']['converter_class'] ) return ObjectBuilder(instance=_class(column_skip=self.config['converter']['clickhouse']['column_skip']))
def __init__( self, writer_builder=None, key_builder=None, max_bucket_size=10000, max_belt_size=1, max_interval_between_rotations=60, ): super().__init__( writer_builder=writer_builder, key_builder=ObjectBuilder(class_name=BBIndexGenerator), max_bucket_size=max_bucket_size, max_belt_size=max_belt_size, max_interval_between_rotations=max_interval_between_rotations, )