def meets_conditions(cls, config):
if not config.filter_before_options['name'].endswith('NoFilter'):
cls._log_skip_reason('custom filter configured')
return False
if not config.filter_after_options['name'].endswith('NoFilter'):
cls._log_skip_reason('custom filter configured')
return False
if not config.transform_options['name'].endswith('NoTransform'):
cls._log_skip_reason('custom transform configured')
return False
if not config.grouper_options['name'].endswith('NoGrouper'):
cls._log_skip_reason('custom grouper configured')
return False
if config.writer_options.get('options', {}).get('items_limit'):
cls._log_skip_reason('items limit configuration (items_limit)')
return False
if config.writer_options.get('options',
{}).get('items_per_buffer_write'):
cls._log_skip_reason(
'buffer limit configuration (items_per_buffer_write)')
return False
if config.writer_options.get('options',
{}).get('size_per_buffer_write'):
cls._log_skip_reason(
'buffer limit configuration (size_per_buffer_write)')
return False
write_buffer = config.writer_options['options'].get('write_buffer')
if write_buffer and not write_buffer.endswith('base.WriteBuffer'):
cls._log_skip_reason('custom write buffer configuration')
return False
module_loader = ModuleLoader()
try:
with closing(
module_loader.load_class(
config.reader_options['name'])) as reader:
pass
with closing(
module_loader.load_class(
config.writer_options['name'])) as writer:
pass
except:
cls._log_skip_reason("Can't load reader and/or writer")
return False
if not callable(getattr(reader, 'get_read_streams', None)) or\
not callable(getattr(reader, 'open_stream', None)):
cls._log_skip_reason(
"Reader doesn't support get_read_streams()/open_stream()")
return False
if not hasattr(writer, 'write_stream'):
cls._log_skip_reason("Writer doesn't support write_stream()")
return False
return True
def meets_conditions(cls, config):
if not config.filter_before_options['name'].endswith('NoFilter'):
cls._log_skip_reason('custom filter configured')
return False
if not config.filter_after_options['name'].endswith('NoFilter'):
cls._log_skip_reason('custom filter configured')
return False
if not config.transform_options['name'].endswith('NoTransform'):
cls._log_skip_reason('custom transform configured')
return False
if not config.grouper_options['name'].endswith('NoGrouper'):
cls._log_skip_reason('custom grouper configured')
return False
if config.writer_options.get('options', {}).get('items_limit'):
cls._log_skip_reason('items limit configuration (items_limit)')
return False
if config.writer_options.get('options', {}).get('items_per_buffer_write'):
cls._log_skip_reason('buffer limit configuration (items_per_buffer_write)')
return False
if config.writer_options.get('options', {}).get('size_per_buffer_write'):
cls._log_skip_reason('buffer limit configuration (size_per_buffer_write)')
return False
module_loader = ModuleLoader()
try:
with closing(module_loader.load_class(config.reader_options['name'])) as reader:
pass
with closing(module_loader.load_class(config.writer_options['name'])) as writer:
pass
except:
cls._log_skip_reason("Can't load reader and/or writer")
return False
if not callable(getattr(reader, 'get_read_streams', None)) or\
not callable(getattr(reader, 'open_stream', None)):
cls._log_skip_reason("Reader doesn't support get_read_streams()/open_stream()")
return False
if not hasattr(writer, 'write_stream'):
cls._log_skip_reason("Writer doesn't support write_stream()")
return False
return True
def _get_write_buffer(self):
module_loader = ModuleLoader()
write_buffer_module = self.read_option('write_buffer')
write_buffer_class = module_loader.load_class(write_buffer_module)
write_buffer_options = {
'name': self.read_option('write_buffer'),
'options': self.read_option('write_buffer_options'),
}
file_handler = self._items_group_files_handler(write_buffer_class,
**write_buffer_options['options'])
kwargs = {
'items_per_buffer_write': self.read_option('items_per_buffer_write'),
'size_per_buffer_write': self.read_option('size_per_buffer_write'),
'items_group_files_handler': file_handler,
'compression_format': self.compression_format,
'hash_algorithm': self.hash_algorithm,
}
return module_loader.load_write_buffer(write_buffer_options, self.metadata, **kwargs)
