def __init__(self, config):
ConfigurablePlugin.__init__(self, config)
self.setState(src=None, protocol={})
# Resync settings:
self._interactive = config.getBool('resync interactive',
False,
onChange=None)
# behaviour in case of event size changes
self._mode_removed = config.getEnum('resync mode removed',
ResyncMode,
ResyncMode.complete,
subset=ResyncMode.noChanged)
self._mode_expanded = config.getEnum('resync mode expand', ResyncMode,
ResyncMode.changed)
self._mode_shrunken = config.getEnum('resync mode shrink', ResyncMode,
ResyncMode.changed)
self._mode_new = config.getEnum(
'resync mode new',
ResyncMode,
ResyncMode.complete,
subset=[ResyncMode.complete, ResyncMode.ignore])
# behaviour in case of metadata changes
self._metaOpts = {}
for meta in config.getList('resync metadata', [], onChange=None):
self._metaOpts[meta] = config.getEnum('resync mode %s' % meta,
ResyncMode,
ResyncMode.complete,
subset=ResyncMode.noChanged)
# behaviour in case of job changes - disable changed jobs, preserve job number of changed jobs or reorder
self._resyncOrder = config.getEnum('resync jobs', ResyncOrder,
ResyncOrder.append)
self._initConfig(config)
def __init__(self, config, workflow):
ConfigurablePlugin.__init__(self, config)
self._workflow = workflow
self._reportOpts = config.get('report options', '', onChange = None)
self._report = config.getCompositePlugin('report', 'BasicReport', 'MultiReport',
cls = Report, onChange = None, pargs = (workflow.jobManager.jobDB,
workflow.task), pkwargs = {'configString': self._reportOpts})
def __init__(self, config, datasetExpr, datasetNick=None):
ConfigurablePlugin.__init__(self, config)
self._log = logging.getLogger('dataset.provider')
(self._datasetExpr, self._datasetNick) = (datasetExpr, datasetNick)
(self._cache_block, self._cache_dataset) = (None, None)
self._dataset_query_interval = config.getTime(
'dataset default query interval', 60, onChange=None)
triggerDataResync = triggerResync(['datasets', 'parameters'])
self._stats = DataProcessor.createInstance(
'SimpleStatsDataProcessor', config, triggerDataResync, self._log,
' * Dataset %s:\n\tcontains ' % repr(datasetNick or datasetExpr))
self._nickProducer = config.getPlugin('nickname source',
'SimpleNickNameProducer',
cls=DataProcessor,
pargs=(triggerDataResync, ),
onChange=triggerDataResync)
self._datasetProcessor = config.getCompositePlugin(
'dataset processor',
'NickNameConsistencyProcessor EntriesConsistencyDataProcessor URLDataProcessor URLCountDataProcessor '
+
'EntriesCountDataProcessor EmptyDataProcessor UniqueDataProcessor LocationDataProcessor',
'MultiDataProcessor',
cls=DataProcessor,
pargs=(triggerDataResync, ),
onChange=triggerDataResync)
def __init__(self, config, jobLimit=-1, jobSelector=None):
ConfigurablePlugin.__init__(self, config)
self._log = logging.getLogger('jobs')
self._dbPath = config.getWorkPath('jobs')
self._jobMap = self.readJobs(jobLimit)
if jobLimit < 0 and len(self._jobMap) > 0:
jobLimit = max(self._jobMap) + 1
(self.jobLimit, self.alwaysSelector) = (jobLimit, jobSelector)
def __init__(self, config, option_prefix, case_override = None, **kwargs):
ConfigurablePlugin.__init__(self, config)
self._case = case_override
if case_override is None:
self._case = config.getBool(appendOption(option_prefix, 'case sensitive'), default = True, **kwargs)
self._log = logging.getLogger('matcher.%s' % option_prefix)
if not self._log.isEnabledFor(logging.DEBUG1):
self._log = None
def __init__(self, config, jobLimit = -1, jobSelector = None):
ConfigurablePlugin.__init__(self, config)
self._log = logging.getLogger('jobs')
self._dbPath = config.getWorkPath('jobs')
self._jobMap = self.readJobs(jobLimit)
if jobLimit < 0 and len(self._jobMap) > 0:
jobLimit = max(self._jobMap) + 1
(self.jobLimit, self.alwaysSelector) = (jobLimit, jobSelector)
def __init__(self, config, datasource_name):
ConfigurablePlugin.__init__(self, config)
self._datasource_name = datasource_name
self._log = logging.getLogger('%s.provider.processor' % datasource_name)
self._log_debug = None
if self._log.isEnabledFor(logging.DEBUG):
self._log_debug = self._log
self._disabled = False
def __init__(self, config, datasource_name):
ConfigurablePlugin.__init__(self, config)
self._datasource_name = datasource_name
self._log = logging.getLogger('%s.provider.processor' %
datasource_name)
self._log_debug = None
if self._log.isEnabledFor(logging.DEBUG):
self._log_debug = self._log
self._disabled = False
def __init__(self, config, datasource_name, resync_handler_name='DefaultPartitionResyncHandler'): ConfigurablePlugin.__init__(self, config) self._datasource_name = datasource_name self._partition_source = None self._resync_handler = PartitionResyncHandler.create_instance(resync_handler_name, config) self._dp_ds_prop_list = [] for prop in ['Dataset', 'BlockName', 'Nickname', 'Locations']: self._dp_ds_prop_list.append((getattr(DataProvider, prop), getattr(DataSplitter, prop)))
def __init__(self, config, option_prefix, case_override=None, **kwargs):
ConfigurablePlugin.__init__(self, config)
self._case = case_override
if case_override is None:
self._case = config.get_bool(join_config_locations(option_prefix, 'case sensitive'),
default=True, **kwargs)
self._log = logging.getLogger('matcher.%s.%s' % (self.__class__.__name__,
option_prefix[0].replace(' ', '_')))
if not self._log.isEnabledFor(logging.DEBUG1):
self._log = None
def __init__(self, config, option_prefix, case_override=None, **kwargs):
ConfigurablePlugin.__init__(self, config)
self._case = case_override
if case_override is None:
self._case = config.getBool(appendOption(option_prefix,
'case sensitive'),
default=True,
**kwargs)
self._log = logging.getLogger('matcher.%s' % option_prefix)
if not self._log.isEnabledFor(logging.DEBUG1):
self._log = None
def __init__(self, config, datasetExpr, datasetNick = None, datasetID = 0):
ConfigurablePlugin.__init__(self, config)
self._log = logging.getLogger('user.dataprovider')
(self._datasetExpr, self._datasetNick, self._datasetID) = (datasetExpr, datasetNick, datasetID)
(self._cache_block, self._cache_dataset, self._passthrough) = (None, None, False)
self._stats = DataProcessor.createInstance('StatsDataProcessor', config)
self._nickProducer = config.getPlugin('nickname source', 'SimpleNickNameProducer', cls = DataProcessor)
self._datasetProcessor = config.getCompositePlugin('dataset processor',
'EntriesConsistencyDataProcessor URLDataProcessor URLCountDataProcessor ' +
'EntriesCountDataProcessor EmptyDataProcessor UniqueDataProcessor LocationDataProcessor',
'MultiDataProcessor', cls = DataProcessor)
def __init__(self, config, datasetExpr, datasetNick = None, datasetID = 0):
ConfigurablePlugin.__init__(self, config)
self._log = logging.getLogger('user.dataprovider')
(self._datasetExpr, self._datasetNick, self._datasetID) = (datasetExpr, datasetNick, datasetID)
(self._cache_block, self._cache_dataset, self._passthrough) = (None, None, False)
self._stats = DataProcessor.createInstance('StatsDataProcessor', config)
self._nickProducer = config.getPlugin('nickname source', 'SimpleNickNameProducer', cls = DataProcessor)
self._datasetProcessor = config.getCompositePlugin('dataset processor',
'EntriesConsistencyDataProcessor URLDataProcessor URLCountDataProcessor ' +
'EntriesCountDataProcessor EmptyDataProcessor UniqueDataProcessor LocationDataProcessor',
'MultiDataProcessor', cls = DataProcessor, onChange = triggerResync(['datasets', 'parameters']))
def __init__(self, config, option_prefix, case_override=None, **kwargs):
ConfigurablePlugin.__init__(self, config)
self._case = case_override
if case_override is None:
self._case = config.get_bool(join_config_locations(
option_prefix, 'case sensitive'),
default=True,
**kwargs)
self._log = logging.getLogger(
'matcher.%s.%s' %
(self.__class__.__name__, option_prefix[0].replace(' ', '_')))
if not self._log.isEnabledFor(logging.DEBUG1):
self._log = None
def __init__(self, config, datasetExpr, datasetNick = None):
ConfigurablePlugin.__init__(self, config)
self._log = logging.getLogger('dataset.provider')
(self._datasetExpr, self._datasetNick) = (datasetExpr, datasetNick)
(self._cache_block, self._cache_dataset) = (None, None)
self._dataset_query_interval = config.getTime('dataset default query interval', 60, onChange = None)
triggerDataResync = triggerResync(['datasets', 'parameters'])
self._stats = DataProcessor.createInstance('SimpleStatsDataProcessor', config, triggerDataResync, self._log,
' * Dataset %s:\n\tcontains ' % repr(datasetNick or datasetExpr))
self._nickProducer = config.getPlugin('nickname source', 'SimpleNickNameProducer',
cls = DataProcessor, pargs = (triggerDataResync,), onChange = triggerDataResync)
self._datasetProcessor = config.getCompositePlugin('dataset processor',
'NickNameConsistencyProcessor EntriesConsistencyDataProcessor URLDataProcessor URLCountDataProcessor ' +
'EntriesCountDataProcessor EmptyDataProcessor UniqueDataProcessor LocationDataProcessor', 'MultiDataProcessor',
cls = DataProcessor, pargs = (triggerDataResync,), onChange = triggerDataResync)
def __init__(self, config):
ConfigurablePlugin.__init__(self, config)
self.setState(src = None, protocol = {})
# Resync settings:
self._interactive = config.isInteractive('partition resync', False)
# behaviour in case of event size changes
self._mode_removed = config.getEnum('resync mode removed', ResyncMode, ResyncMode.complete, subset = ResyncMode.noChanged)
self._mode_expanded = config.getEnum('resync mode expand', ResyncMode, ResyncMode.changed)
self._mode_shrunken = config.getEnum('resync mode shrink', ResyncMode, ResyncMode.changed)
self._mode_new = config.getEnum('resync mode new', ResyncMode, ResyncMode.complete, subset = [ResyncMode.complete, ResyncMode.ignore])
# behaviour in case of metadata changes
self._metaOpts = {}
for meta in config.getList('resync metadata', [], onChange = None):
self._metaOpts[meta] = config.getEnum('resync mode %s' % meta, ResyncMode, ResyncMode.complete, subset = ResyncMode.noChanged)
# behaviour in case of job changes - disable changed jobs, preserve job number of changed jobs or reorder
self._resyncOrder = config.getEnum('resync jobs', ResyncOrder, ResyncOrder.append)
self._initConfig(config)
def __init__(self,
config,
datasource_name,
dataset_expr,
dataset_nick=None,
dataset_proc=None):
ConfigurablePlugin.__init__(self, config)
self._log = logging.getLogger('%s.provider' % datasource_name)
(self._datasource_name, self._dataset_expr) = (datasource_name,
dataset_expr)
self._dataset_nick_override = dataset_nick
(self._cache_block, self._cache_dataset) = (None, None)
self._dataset_query_interval = config.get_time(
'%s default query interval' % datasource_name, 60, on_change=None)
self._stats = dataset_proc or DataProcessor.create_instance(
'SimpleStatsDataProcessor', config, datasource_name, self._log,
' * Dataset %s:\n\tcontains ' % repr(dataset_nick or dataset_expr))
dataset_config = config.change_view(
default_on_change=TriggerResync(['datasets', 'parameters']))
self._nick_producer = dataset_config.get_plugin(
['nickname source',
'%s nickname source' % datasource_name],
'SimpleNickNameProducer',
cls=DataProcessor,
pargs=(datasource_name, ))
self._dataset_processor = dataset_proc or dataset_config.get_composited_plugin(
'%s processor' % datasource_name,
'NickNameConsistencyProcessor EntriesConsistencyDataProcessor URLDataProcessor '
+
'URLCountDataProcessor EntriesCountDataProcessor EmptyDataProcessor UniqueDataProcessor '
+ 'LocationDataProcessor',
'MultiDataProcessor',
cls=DataProcessor,
pargs=(datasource_name, ))
def __init__(self, config, job_limit=-1, job_selector=None):
ConfigurablePlugin.__init__(self, config)
self._log = logging.getLogger('jobs.db')
(self._job_limit, self._always_selector, self._default_job_obj) = (job_limit, job_selector, Job())
def __init__(self, config, datasource_name):
ConfigurablePlugin.__init__(self, config)
self._datasource_name = datasource_name
self._log = logging.getLogger('%s.partition.processor' % datasource_name)
def __init__(self, config, task): ConfigurablePlugin.__init__(self, config) self._task = task
def __init__(self, config):
ConfigurablePlugin.__init__(self, config)
self._log = logging.getLogger('dataset.provider.infoscanner')
def __init__(self, config, workflow): ConfigurablePlugin.__init__(self, config)
def __init__(self, config, datasource_name):
ConfigurablePlugin.__init__(self, config)
self._datasource_name = datasource_name
self._log = logging.getLogger('%s.partition.processor' %
datasource_name)
def __init__(self, config, job_limit=-1, job_selector=None):
ConfigurablePlugin.__init__(self, config)
self._log = logging.getLogger('jobs.db')
(self._job_limit, self._always_selector,
self._default_job_obj) = (job_limit, job_selector, Job())
def __init__(self, config, repository):
ConfigurablePlugin.__init__(self, config)
self._repository = repository
def __init__(self, config):
ConfigurablePlugin.__init__(self, config)
self._log = logging.getLogger('dataset.partition.processor')
def __init__(self, config, datasource_name):
ConfigurablePlugin.__init__(self, config)
self._log = logging.getLogger('%s.provider.infoscanner' % datasource_name)
self._datasource_name = datasource_name
def __init__(self, config):
ConfigurablePlugin.__init__(self, config)
self._log = logging.getLogger('dataset.provider.infoscanner')
def __init__(self, config, jobLimit=-1, jobSelector=None):
ConfigurablePlugin.__init__(self, config)
self._log = logging.getLogger('jobs.db')
(self._jobLimit, self._alwaysSelector) = (jobLimit, jobSelector)
(self._defaultJob, self._workPath) = (Job(), config.getWorkPath())
def __init__(self, config, stream, register_callback=False):
ConfigurablePlugin.__init__(self, config)
self._msg_len_max = config.get_int('activity max length',
75,
on_change=None)
(self._stream, self._register_cb) = (stream, register_callback)
def __init__(self, config, onChange):
ConfigurablePlugin.__init__(self, config)
self._log = logging.getLogger('dataset.provider.processor')
self._log_debug = None
if self._log.isEnabledFor(logging.DEBUG):
self._log_debug = self._log
def __init__(self, config, source):
ConfigurablePlugin.__init__(self, config)
self._log = logging.getLogger('padapter')
self._source = source
def __init__(self, config, datasource_name):
ConfigurablePlugin.__init__(self, config)
self._log = logging.getLogger('%s.provider.infoscanner' % datasource_name)
self._datasource_name = datasource_name
def __init__(self, config, task):
ConfigurablePlugin.__init__(self, config)
self._task = task
def __init__(self, config): ConfigurablePlugin.__init__(self, config) self._log = None
def __init__(self, config, option_prefix, *kwargs): ConfigurablePlugin.__init__(self, config) self._case = config.getBool(appendOption(option_prefix, 'case sensitive'), default = True)
def __init__(self, config, onChange):
ConfigurablePlugin.__init__(self, config)
self._log = logging.getLogger('dataset.provider.processor')
self._log_debug = None
if self._log.isEnabledFor(logging.DEBUG):
self._log_debug = self._log
def __init__(self, config):
ConfigurablePlugin.__init__(self, config)
self._log = logging.getLogger('infoscanner')
def __init__(self, config):
ConfigurablePlugin.__init__(self, config)
self._log = logging.getLogger('dataset.partition.processor')
def __init__(self, config, option_prefix, *kwargs):
ConfigurablePlugin.__init__(self, config)
self._case = config.getBool(appendOption(option_prefix,
'case sensitive'),
default=True)
def __init__(self, config):
ConfigurablePlugin.__init__(self, config)
self._log = logging.getLogger('partproc')
def __init__(self, config, workflow):
ConfigurablePlugin.__init__(self, config)
def __init__(self, config, stream, register_callback=False):
ConfigurablePlugin.__init__(self, config)
self._msg_len_max = config.get_int('activity max length', 75, on_change=None)
(self._stream, self._register_cb) = (stream, register_callback)