def __init__(self, node):
self.meta_session = node.meta_session.clone()
self.commands = {}
self.cmd_progress = {}
self.active_hosts = []
# self.pending_hosts = None
# self.ready = False
self.__tq = timed_queue.TimedQueue()
self.__tq.add_task_in(self.MAKE_IOLOOP, 0,
self._make_tq_thread_ioloop)
self.__tq.add_task_in(self.STATE_FETCH,
5, self._fetch_states)
self.minion_headers = ({'X-Auth': config['minions']['authkey']}
if MINIONS_CFG.get('authkey') else
None)
self.minion_port = MINIONS_CFG.get('port', 8081)
self.__commands_lock = threading.Lock()
self.__active_hosts_lock = threading.Lock()
def __init__(self,
node,
job_finder,
couple_record_finder=None,
prepare_namespaces_states=False,
prepare_flow_stats=False,
statistics=None):
logger.info("Created NodeInfoUpdater")
self.__node = node
self.statistics = statistics
self.job_finder = job_finder
self.couple_record_finder = couple_record_finder
self._namespaces_states = CachedGzipResponse()
self._flow_stats = {}
self.__tq = timed_queue.TimedQueue()
self.__session = elliptics.Session(self.__node)
wait_timeout = config.get('elliptics',
{}).get('wait_timeout') or config.get(
'wait_timeout', 5)
self.__session.set_timeout(wait_timeout)
self.__nodeUpdateTimestamps = (time.time(), time.time())
self.__cluster_update_lock = threading.Lock()
if prepare_namespaces_states and statistics is None:
raise AssertionError(
'Statistics is required for namespaces states calculation')
if prepare_flow_stats and statistics is None:
raise AssertionError(
'Statistics is required for flow stats calculation')
self._prepare_namespaces_states = prepare_namespaces_states
self._prepare_flow_stats = prepare_flow_stats
def __init__(self, node, niu, job_processor, db):
self.node = node
self.niu = niu
self.session = elliptics.Session(self.node)
wait_timeout = config.get('elliptics', {}).get('wait_timeout', 5)
self.session.set_timeout(wait_timeout)
self.service_metakey = str(
self.session.transform(keys.SYMMETRIC_GROUPS_KEY))
try:
keys_db_uri = config['metadata']['cache']['db']
except KeyError:
logger.error('Config parameter metadata.cache.db is required '
'for cache manager')
raise
self.keys_db = Collection(db[keys_db_uri], 'keys')
self.distributor = CacheDistributor(
self.node, self.keys_db, job_processor)
self.top_keys = {}
self.__tq = timed_queue.TimedQueue()
self.nodes_update()
self.update_cache_groups()
self.top_update_timer = periodic_timer(
seconds=CACHE_CFG.get('top_update_period', 1800))
self.__tq.add_task_at(
CacheManager.MONITOR_TOP_STATS,
self.top_update_timer.next(),
self.monitor_top_stats)
def setup(self, session, index_prefix):
self.__session = session
self.__index_prefix = index_prefix
self.__tq = timed_queue.TimedQueue()
self.__tq.start()
self.__tq.add_task_in('cache_status_update', 10, self.cache_status_update)
self.__tq.add_task_in('cache_list_update', 15, self.update_cache_list)
self.enabled = True
def __init__(self, node, job_finder):
logger.info("Created NodeInfoUpdater")
self.__node = node
self.job_finder = job_finder
self.__tq = timed_queue.TimedQueue()
self.__session = elliptics.Session(self.__node)
wait_timeout = config.get('elliptics', {}).get(
'wait_timeout', None) or config.get('wait_timeout', 5)
self.__session.set_timeout(wait_timeout)
self.__nodeUpdateTimestamps = (time.time(), time.time())
self.__cluster_update_lock = threading.Lock()
def __init__(self, logging, node):
logging.info("Created NodeInfoUpdater")
self.__logging = logging
self.__node = node
self.__tq = timed_queue.TimedQueue()
self.__tq.start()
self.__session = elliptics.Session(self.__node)
self.__session.set_timeout(config.get('wait_timeout', 5))
self.__nodeUpdateTimestamps = (time.time(), time.time())
self.__cache = Service('cache')
self.loadNodes(delayed=False)
def __init__(self):
# actual init happens in 'init' method
# TODO: return node back to constructor after wrapping
# all the code in a 'mastermind' package
self.node = None
self.meta_session = None
self.cache = None
self._sync_ts = int(time.time())
self._groups_to_update = set()
self._groups_to_update_lock = threading.Lock()
self.__tq = timed_queue.TimedQueue()
def __init__(self):
# actual init happens in 'init' method
# TODO: return node back to constructor after wrapping
# all the code in a 'mastermind' package
self.node = None
self.meta_session = None
self.state = {}
self.__state_lock = threading.Lock()
self.sync_ts = None
self.state_valid_time = config.get('infrastructure_state_valid_time',
120)
self.__tq = timed_queue.TimedQueue()
self.__tq.start()
def __init__(self, job_finder, node, db, niu, minions):
logger.info('Starting JobProcessor')
self.job_finder = job_finder
self.session = elliptics.Session(node)
wait_timeout = config.get('elliptics', {}).get('wait_timeout', None) or \
config.get('wait_timeout', 5)
self.session.set_timeout(wait_timeout)
self.meta_session = node.meta_session
self.minions = minions
self.node_info_updater = niu
self.planner = None
self.__tq = timed_queue.TimedQueue()
self.jobs_timer = periodic_timer(
seconds=JOB_CONFIG.get('execute_period', 60))
self.downtimes = Collection(db[config['metadata']['jobs']['db']],
'downtimes')
self.__tq.add_task_at(self.JOBS_EXECUTE, self.jobs_timer.next(),
self._execute_jobs)
def __init__(self, node):
self.node = node
self.meta_session = self.node.meta_session
self.commands = {}
self.history = {}
self.active_hosts = []
self.__tq = timed_queue.TimedQueue()
self.__tq.start()
self.__tq.add_task_in(self.MAKE_IOLOOP, 0, self._make_tq_thread_ioloop)
self.__tq.add_task_in(self.HISTORY_FETCH, 5, self._fetch_history)
self.__tq.add_task_in(self.STATE_FETCH, 5, self._fetch_states)
self.minion_headers = ({
'X-Auth': config['minions']['authkey']
} if config.get('minions', {}).get('authkey') else None)
self.minion_port = config.get('minions', {}).get('port', 8081)
self.__commands_lock = threading.Lock()
self.__active_hosts_lock = threading.Lock()
def __init__(self, meta_session):
self.meta_session = meta_session
self.__tq = timed_queue.TimedQueue()
cache.init(self.meta_session, self.__tq)
