def meets_conditions(cls, config): if not config.filter_before_options['name'].endswith('NoFilter'): cls._log_skip_reason('custom filter configured') return False if not config.filter_after_options['name'].endswith('NoFilter'): cls._log_skip_reason('custom filter configured') return False if not config.transform_options['name'].endswith('NoTransform'): cls._log_skip_reason('custom transform configured') return False if not config.grouper_options['name'].endswith('NoGrouper'): cls._log_skip_reason('custom grouper configured') return False if config.writer_options.get('options', {}).get('items_limit'): cls._log_skip_reason('items limit configuration (items_limit)') return False if config.writer_options.get('options', {}).get('items_per_buffer_write'): cls._log_skip_reason( 'buffer limit configuration (items_per_buffer_write)') return False if config.writer_options.get('options', {}).get('size_per_buffer_write'): cls._log_skip_reason( 'buffer limit configuration (size_per_buffer_write)') return False write_buffer = config.writer_options['options'].get('write_buffer') if write_buffer and not write_buffer.endswith('base.WriteBuffer'): cls._log_skip_reason('custom write buffer configuration') return False module_loader = ModuleLoader() try: with closing( module_loader.load_class( config.reader_options['name'])) as reader: pass with closing( module_loader.load_class( config.writer_options['name'])) as writer: pass except: cls._log_skip_reason("Can't load reader and/or writer") return False if not callable(getattr(reader, 'get_read_streams', None)) or\ not callable(getattr(reader, 'open_stream', None)): cls._log_skip_reason( "Reader doesn't support get_read_streams()/open_stream()") return False if not hasattr(writer, 'write_stream'): cls._log_skip_reason("Writer doesn't support write_stream()") return False return True
def meets_conditions(cls, config): if not config.filter_before_options['name'].endswith('NoFilter'): cls._log_skip_reason('custom filter configured') return False if not config.filter_after_options['name'].endswith('NoFilter'): cls._log_skip_reason('custom filter configured') return False if not config.transform_options['name'].endswith('NoTransform'): cls._log_skip_reason('custom transform configured') return False if not config.grouper_options['name'].endswith('NoGrouper'): cls._log_skip_reason('custom grouper configured') return False if config.writer_options.get('options', {}).get('items_limit'): cls._log_skip_reason('items limit configuration (items_limit)') return False if config.writer_options.get('options', {}).get('items_per_buffer_write'): cls._log_skip_reason('buffer limit configuration (items_per_buffer_write)') return False if config.writer_options.get('options', {}).get('size_per_buffer_write'): cls._log_skip_reason('buffer limit configuration (size_per_buffer_write)') return False module_loader = ModuleLoader() try: with closing(module_loader.load_class(config.reader_options['name'])) as reader: pass with closing(module_loader.load_class(config.writer_options['name'])) as writer: pass except: cls._log_skip_reason("Can't load reader and/or writer") return False if not callable(getattr(reader, 'get_read_streams', None)) or\ not callable(getattr(reader, 'open_stream', None)): cls._log_skip_reason("Reader doesn't support get_read_streams()/open_stream()") return False if not hasattr(writer, 'write_stream'): cls._log_skip_reason("Writer doesn't support write_stream()") return False return True
def _get_write_buffer(self): module_loader = ModuleLoader() write_buffer_module = self.read_option('write_buffer') write_buffer_class = module_loader.load_class(write_buffer_module) write_buffer_options = { 'name': self.read_option('write_buffer'), 'options': self.read_option('write_buffer_options'), } file_handler = self._items_group_files_handler(write_buffer_class, **write_buffer_options['options']) kwargs = { 'items_per_buffer_write': self.read_option('items_per_buffer_write'), 'size_per_buffer_write': self.read_option('size_per_buffer_write'), 'items_group_files_handler': file_handler, 'compression_format': self.compression_format, 'hash_algorithm': self.hash_algorithm, } return module_loader.load_write_buffer(write_buffer_options, self.metadata, **kwargs)