def __init__(self, config, datasource_name, dataset_expr, dataset_nick, provider_list):
for provider in provider_list:
provider.disable_stream_singletons()
DataProvider.__init__(self, config, datasource_name, dataset_expr, dataset_nick)
self._stats = DataProcessor.create_instance('SimpleStatsDataProcessor', config,
'dataset', self._log, 'Summary: Running over ')
self._provider_list = provider_list
def _parse_filter(self, config, value):
dataset_proc = DataProcessor.create_instance('NullDataProcessor')
def _get_filter_entries():
for pat in value.split():
if ':' not in pat.lstrip(':'):
yield pat
else:
block_iter = DataProvider.iter_blocks_from_expr(
config,
':%s' % pat.lstrip(':'),
dataset_proc=dataset_proc)
for block in block_iter:
for fi in block[DataProvider.FileList]:
yield fi[DataProvider.URL]
return str.join('\n', _get_filter_entries())
def __init__(self,
config,
datasource_name,
dataset_expr,
dataset_nick=None,
dataset_proc=None):
ConfigurablePlugin.__init__(self, config)
self._log = logging.getLogger('%s.provider' % datasource_name)
(self._datasource_name, self._dataset_expr) = (datasource_name,
dataset_expr)
self._dataset_nick_override = dataset_nick
(self._cache_block, self._cache_dataset) = (None, None)
self._dataset_query_interval = config.get_time(
'%s default query interval' % datasource_name, 60, on_change=None)
self._stats = dataset_proc or DataProcessor.create_instance(
'SimpleStatsDataProcessor', config, datasource_name, self._log,
' * Dataset %s:\n\tcontains ' % repr(dataset_nick or dataset_expr))
dataset_config = config.change_view(
default_on_change=TriggerResync(['datasets', 'parameters']))
self._nick_producer = dataset_config.get_plugin(
['nickname source',
'%s nickname source' % datasource_name],
'SimpleNickNameProducer',
cls=DataProcessor,
pargs=(datasource_name, ))
self._dataset_processor = dataset_proc or dataset_config.get_composited_plugin(
'%s processor' % datasource_name,
'NickNameConsistencyProcessor EntriesConsistencyDataProcessor URLDataProcessor '
+
'URLCountDataProcessor EntriesCountDataProcessor EmptyDataProcessor UniqueDataProcessor '
+ 'LocationDataProcessor',
'MultiDataProcessor',
cls=DataProcessor,
pargs=(datasource_name, ))
