def __init__(self, config, onChange):
DataProcessor.__init__(self, config, onChange)
self._targetJobs = config.getInt('target partitions', -1, onChange = onChange)
self._targetJobsDS = config.getInt('target partitions per nickname', -1, onChange = onChange)
self._entries = {None: 0}
self._files = {None: 0}
self._config = config
def __init__(self, config, datasource_name):
DataProcessor.__init__(self, config, datasource_name)
self._location_filter = config.get_filter(
self._get_dproc_opt('location filter'),
'',
default_matcher='BlackWhiteMatcher',
default_filter='StrictListFilter')
def __init__(self, config, onChange):
DataProcessor.__init__(self, config, onChange)
self._locationfilter = config.getFilter('dataset location filter',
'',
defaultMatcher='blackwhite',
defaultFilter='strict',
onChange=onChange)
def __init__(self, config):
DataProcessor.__init__(self, config)
self._targetJobs = config.getInt('target partitions', -1, onChange = DataProcessor.triggerDataResync)
self._targetJobsDS = config.getInt('target partitions per nickname', -1, onChange = DataProcessor.triggerDataResync)
self._entries = {None: 0}
self._files = {None: 0}
self._config = config
def __init__(self, config, datasource_name):
DataProcessor.__init__(self, config, datasource_name)
self._empty_files = config.get_bool(
self._get_dproc_opt('remove empty files'), True)
self._empty_block = config.get_bool(
self._get_dproc_opt('remove empty blocks'), True)
(self._removed_files, self._removed_blocks) = (0, 0)
def __init__(self, config, datasource_name):
DataProcessor.__init__(self, config, datasource_name)
self._limit_files = config.get_int(
self._get_dproc_opt(['limit files', 'limit urls']), -1)
self._limit_files_fraction = config.get_float(
self._get_dproc_opt(
['limit files fraction', 'limit urls fraction']), -1.)
(self._limit_files_per_ds, self._files_per_ds) = ({}, {})
def __init__(self, config):
DataProcessor.__init__(self, config)
# Ensure the same nickname is used consistently in all blocks of a dataset
self._checkConsistency = config.getBool('nickname check consistency', True)
self._checkConsistencyData = {}
# Check if two different datasets have the same nickname
self._checkCollision = config.getBool('nickname check collision', True)
self._checkCollisionData = {}
def __init__(self, config):
DataProcessor.__init__(self, config)
self._targetJobs = config.getInt('target partitions', -1)
self._targetJobsDS = config.getInt('target partitions per nickname', -1)
self._writeSettings = (self._targetJobs != -1) or (self._targetJobsDS != -1)
self._entries = {None: 0}
self._files = {None: 0}
self._config = config
def __init__(self, config):
DataProcessor.__init__(self, config)
# Ensure the same nickname is used consistently in all blocks of a dataset
self._checkConsistency = config.getBool('nickname check consistency',
True)
self._checkConsistencyData = {}
# Check if two different datasets have the same nickname
self._checkCollision = config.getBool('nickname check collision', True)
self._checkCollisionData = {}
def __init__(self, config):
DataProcessor.__init__(self, config)
self._checkURLOpt = 'dataset check unique url'
self._checkURL = config.getEnum(self._checkURLOpt, DatasetUniqueMode,
DatasetUniqueMode.abort)
self._checkBlockOpt = 'dataset check unique block'
self._checkBlock = config.getEnum(self._checkBlockOpt,
DatasetUniqueMode,
DatasetUniqueMode.abort)
def __init__(self, config, onChange):
DataProcessor.__init__(self, config, onChange)
self._emptyFiles = config.getBool('dataset remove empty files',
True,
onChange=onChange)
self._emptyBlock = config.getBool('dataset remove empty blocks',
True,
onChange=onChange)
(self._removedFiles, self._removedBlocks) = (0, 0)
def __init__(self, config, datasource_name):
DataProcessor.__init__(self, config, datasource_name)
self._sort_ds = config.get_bool(self._get_dproc_opt('sort'), False)
self._sort_block = config.get_bool(self._get_dproc_opt('block sort'),
False)
self._sort_files = config.get_bool(self._get_dproc_opt('files sort'),
False)
self._sort_location = config.get_bool(
self._get_dproc_opt('location sort'), False)
def __init__(self, config):
DataProcessor.__init__(self, config)
self._targetJobs = config.getInt('target partitions', -1)
self._targetJobsDS = config.getInt('target partitions per nickname',
-1)
self._writeSettings = (self._targetJobs != -1) or (self._targetJobsDS
!= -1)
self._entries = {None: 0}
self._files = {None: 0}
self._config = config
def __init__(self, config):
DataProcessor.__init__(self, config)
internal_config = config.changeView(viewClass = 'SimpleConfigView', setSections = ['dataprocessor'])
internal_config.set('dataset processor', 'NullDataProcessor')
self._url_filter = config.getFilter(['dataset ignore files', 'dataset ignore urls'], '', negate = True,
filterParser = lambda value: self._parseFilter(internal_config, value),
filterStr = lambda value: str.join('\n', value.split()),
matchKey = itemgetter(DataProvider.URL),
defaultMatcher = 'blackwhite', defaultFilter = 'weak',
onChange = DataProcessor.triggerDataResync)
def __init__(self, config, onChange):
DataProcessor.__init__(self, config, onChange)
internal_config = config.changeView(viewClass = 'SimpleConfigView', setSections = ['dataprocessor'])
internal_config.set('dataset processor', 'NullDataProcessor')
config.set('dataset ignore urls matcher case sensitive', 'False')
self._url_filter = config.getFilter(['dataset ignore files', 'dataset ignore urls'], '', negate = True,
filterParser = lambda value: self._parseFilter(internal_config, value),
filterStr = lambda value: str.join('\n', value.split()),
defaultMatcher = 'blackwhite', defaultFilter = 'weak',
onChange = onChange)
def __init__(self, config, datasource_name):
DataProcessor.__init__(self, config, datasource_name)
self._target_jobs = config.get_int(
join_config_locations(['', datasource_name], 'target partitions'), -1)
self._target_jobs_ds = config.get_int(
join_config_locations(['', datasource_name], 'target partitions per nickname'), -1)
self._entries = {None: 0}
self._files = {None: 0}
self._config = None
if self.enabled():
self._config = config
def __init__(self, config, datasource_name):
DataProcessor.__init__(self, config, datasource_name)
config.set('%s ignore urls matcher case sensitive' % datasource_name,
'False')
self._url_filter = config.get_filter(
self._get_dproc_opt(['ignore files', 'ignore urls']),
'',
negate=True,
default_matcher='BlackWhiteMatcher',
default_filter='WeakListFilter',
filter_parser=lambda value: self._parse_filter(config, value),
filter_str=lambda value: str.join('\n', value.split()))
def __init__(self, config, onChange):
DataProcessor.__init__(self, config, onChange)
self._sortDS = config.getBool('dataset sort', False, onChange=onChange)
self._sortBlock = config.getBool('dataset block sort',
False,
onChange=onChange)
self._sortFiles = config.getBool('dataset files sort',
False,
onChange=onChange)
self._sortLocation = config.getBool('dataset location sort',
False,
onChange=onChange)
def __init__(self, config, datasource_name):
DataProcessor.__init__(self, config, datasource_name)
self._target_jobs = config.get_int(
join_config_locations(['', datasource_name], 'target partitions'),
-1)
self._target_jobs_ds = config.get_int(
join_config_locations(['', datasource_name],
'target partitions per nickname'), -1)
self._entries = {None: 0}
self._files = {None: 0}
self._config = None
if self.enabled():
self._config = config
def __init__(self, config, onChange):
DataProcessor.__init__(self, config, onChange)
internal_config = config.changeView(viewClass='SimpleConfigView',
setSections=['dataprocessor'])
internal_config.set('dataset processor', 'NullDataProcessor')
config.set('dataset ignore urls matcher case sensitive', 'False')
self._url_filter = config.getFilter(
['dataset ignore files', 'dataset ignore urls'],
'',
negate=True,
filterParser=lambda value: self._parseFilter(
internal_config, value),
filterStr=lambda value: str.join('\n', value.split()),
defaultMatcher='blackwhite',
defaultFilter='weak',
onChange=onChange)
def __init__(self, config):
DataProcessor.__init__(self, config)
self._emptyFiles = config.getBool('dataset remove empty files', True)
self._emptyBlock = config.getBool('dataset remove empty blocks', True)
def __init__(self, config):
DataProcessor.__init__(self, config)
self._entries = 0
self._blocks = 0
def __init__(self, config, onChange): DataProcessor.__init__(self, config, onChange) self._limitEntries = config.getInt(['dataset limit events', 'dataset limit entries'], -1, onChange = onChange)
def __init__(self, config, onChange):
DataProcessor.__init__(self, config, onChange)
self._limitEntries = config.getInt(
['dataset limit events', 'dataset limit entries'],
-1,
onChange=onChange)
def __init__(self, config): DataProcessor.__init__(self, config) self._checkURLOpt = 'dataset check unique url' self._checkURL = config.getEnum(self._checkURLOpt, DatasetUniqueMode, DatasetUniqueMode.abort) self._checkBlockOpt = 'dataset check unique block' self._checkBlock = config.getEnum(self._checkBlockOpt, DatasetUniqueMode, DatasetUniqueMode.abort)
def __init__(self, config, onChange):
DataProcessor.__init__(self, config, onChange)
self._sortDS = config.getBool('dataset sort', False, onChange = onChange)
self._sortBlock = config.getBool('dataset block sort', False, onChange = onChange)
self._sortFiles = config.getBool('dataset files sort', False, onChange = onChange)
self._sortLocation = config.getBool('dataset location sort', False, onChange = onChange)
def __init__(self, config): DataProcessor.__init__(self, config) self.reset()
def __init__(self, config):
DataProcessor.__init__(self, config)
self._checkURL = config.getEnum('dataset check unique url', DatasetUniqueMode, DatasetUniqueMode.abort,
onChange = DataProcessor.triggerDataResync)
self._checkBlock = config.getEnum('dataset check unique block', DatasetUniqueMode, DatasetUniqueMode.abort,
onChange = DataProcessor.triggerDataResync)
def __init__(self, config):
DataProcessor.__init__(self, config)
self._locationfilter = config.getFilter('dataset location filter', '',
defaultMatcher = 'blackwhite', defaultFilter = 'strict',
onChange = DataProcessor.triggerDataResync)
def __init__(self, config):
DataProcessor.__init__(self, config)
self._emptyFiles = config.getBool('dataset remove empty files', True, onChange = DataProcessor.triggerDataResync)
self._emptyBlock = config.getBool('dataset remove empty blocks', True, onChange = DataProcessor.triggerDataResync)
def __init__(self, config, onChange): DataProcessor.__init__(self, config, onChange) self._limitFiles = config.getInt(['dataset limit files', 'dataset limit urls'], -1, onChange = onChange)
def __init__(self, config, onChange):
DataProcessor.__init__(self, config, onChange)
def __init__(self, config): DataProcessor.__init__(self, config) self._ignoreURLs = config.getList(['dataset ignore urls', 'dataset ignore files'], [])
def __init__(self, config, onChange):
DataProcessor.__init__(self, config, onChange)
self._emptyFiles = config.getBool('dataset remove empty files', True, onChange = onChange)
self._emptyBlock = config.getBool('dataset remove empty blocks', True, onChange = onChange)
(self._removedFiles, self._removedBlocks) = (0, 0)
def __init__(self, config, onChange): DataProcessor.__init__(self, config, onChange)
def __init__(self, config): DataProcessor.__init__(self, config) self._limitFiles = config.getInt(['dataset limit files', 'dataset limit urls'], -1, onChange = DataProcessor.triggerDataResync)
def __init__(self, config): DataProcessor.__init__(self, config) self._entries = 0 self._blocks = 0
def __init__(self, config, datasource_name): DataProcessor.__init__(self, config, datasource_name) (self._entries, self._blocks, self._files) = (0, 0, 0)
def __init__(self, config):
DataProcessor.__init__(self, config)
self._emptyFiles = config.getBool('dataset remove empty files', True)
self._emptyBlock = config.getBool('dataset remove empty blocks', True)
def __init__(self, config, datasource_name): DataProcessor.__init__(self, config, datasource_name) (self._entries, self._blocks, self._files) = (0, 0, 0)
def __init__(self, config, onChange):
DataProcessor.__init__(self, config, onChange)
self._limitFiles = config.getInt(
['dataset limit files', 'dataset limit urls'],
-1,
onChange=onChange)
def __init__(self, config):
DataProcessor.__init__(self, config)
self._ignoreURLs = config.getList(
['dataset ignore urls', 'dataset ignore files'], [])
def __init__(self, config): DataProcessor.__init__(self, config) self._limitEntries = config.getInt(['dataset limit events', 'dataset limit entries'], -1, onChange = DataProcessor.triggerDataResync)
def __init__(self, config, datasource_name):
DataProcessor.__init__(self, config, datasource_name)
self._limit_entries = config.get_int(
self._get_dproc_opt(['limit events', 'limit entries']), -1)
