def __init__(self, config, datasource_name):
		DataProcessor.__init__(self, config, datasource_name)
		self._target_jobs = config.get_int(
			join_config_locations(['', datasource_name], 'target partitions'), -1)
		self._target_jobs_ds = config.get_int(
			join_config_locations(['', datasource_name], 'target partitions per nickname'), -1)
		self._entries = {None: 0}
		self._files = {None: 0}
		self._config = None
		if self.enabled():
			self._config = config
Esempio n. 2
0
 def __init__(self, config, datasource_name):
     DataProcessor.__init__(self, config, datasource_name)
     self._target_jobs = config.get_int(
         join_config_locations(['', datasource_name], 'target partitions'),
         -1)
     self._target_jobs_ds = config.get_int(
         join_config_locations(['', datasource_name],
                               'target partitions per nickname'), -1)
     self._entries = {None: 0}
     self._files = {None: 0}
     self._config = None
     if self.enabled():
         self._config = config
Esempio n. 3
0
 def _get_dproc_opt(self, *args):
     return join_config_locations(self._datasource_name, *args)
Esempio n. 4
0
	def _get_part_opt(self, *args):
		return join_config_locations(['', self._datasource_name], *args)
Esempio n. 5
0
		def _get_active_hash_input(prefix, guard_entry_idx):
			hash_input_list_user = scanner_config.get_list(join_config_locations(prefix, 'hash keys'), [])
			hash_input_list_guard = scanner_config.get_list(join_config_locations(prefix, 'guard override'),
				lchain(imap(lambda scanner: scanner.get_guard_keysets()[guard_entry_idx], self._scanner_list)))
			return hash_input_list_user + hash_input_list_guard
Esempio n. 6
0
		def _setup(prefix):
			selected_hash_list = scanner_config.get_list(join_config_locations(prefix, 'key select'), [])
			name = scanner_config.get(join_config_locations(prefix, 'name pattern'), '')
			return (selected_hash_list, name)
Esempio n. 7
0
	def _get_pproc_opt(self, opt):
		return join_config_locations(['', self._datasource_name], 'partition', opt)
Esempio n. 8
0
	def _get_dproc_opt(self, *args):
		return join_config_locations(self._datasource_name, *args)
Esempio n. 9
0
	def __init__(self, config, option_prefix, executor, def_chunk_size=5, def_chunk_interval=5):
		ForwardingExecutor.__init__(self, config, executor)
		self._chunk_size = config.get_int(join_config_locations(option_prefix, 'chunk size'),
			def_chunk_size, on_change=None)
		self._chunk_interval = config.get_int(join_config_locations(option_prefix, 'chunk interval'),
			def_chunk_interval, on_change=None)
Esempio n. 10
0
 def _get_pproc_opt(self, opt):
     return join_config_locations(['', self._datasource_name], 'partition',
                                  opt)
Esempio n. 11
0
	def __init__(self, config, datasource_name):
		NickNameProducer.__init__(self, config, datasource_name)
		self._full_name = config.get_bool(
			join_config_locations(['', datasource_name], 'nickname full name'), True)
Esempio n. 12
0
	def __init__(self, config, datasource_name):
		NickNameProducer.__init__(self, config, datasource_name)
		self._expr = config.get(
			join_config_locations(['', datasource_name], 'nickname expr'), 'current_nickname')
Esempio n. 13
0
 def __init__(self, config, datasource_name):
     NickNameProducer.__init__(self, config, datasource_name)
     self._full_name = config.get_bool(
         join_config_locations(['', datasource_name], 'nickname full name'),
         True)
Esempio n. 14
0
 def __init__(self, config, datasource_name):
     NickNameProducer.__init__(self, config, datasource_name)
     self._expr = config.get(
         join_config_locations(['', datasource_name], 'nickname expr'),
         'current_nickname')