def health_check(self):
amp_health_repo = repo.AmphoraHealthRepository()
with futures.ThreadPoolExecutor(max_workers=self.threads) as executor:
# Don't start checking immediately, as the health manager may
# have been down for a while and amphorae not able to check in.
LOG.debug("Pausing before starting health check")
time.sleep(CONF.health_manager.heartbeat_timeout)
while True:
session = db_api.get_session()
LOG.debug("Starting amphora health check")
failover_count = 0
while True:
amp = amp_health_repo.get_stale_amphora(session)
if amp is None:
break
failover_count += 1
LOG.info("Stale amphora's id is: %s",
amp.amphora_id)
executor.submit(self.cw.failover_amphora,
amp.amphora_id)
if failover_count > 0:
LOG.info("Failed over %s amphora",
failover_count)
time.sleep(CONF.health_manager.health_check_interval)
def __init__(self):
self._amphora_flows = amphora_flows.AmphoraFlows()
self._health_monitor_flows = health_monitor_flows.HealthMonitorFlows()
self._lb_flows = load_balancer_flows.LoadBalancerFlows()
self._listener_flows = listener_flows.ListenerFlows()
self._member_flows = member_flows.MemberFlows()
self._pool_flows = pool_flows.PoolFlows()
self._l7policy_flows = l7policy_flows.L7PolicyFlows()
self._l7rule_flows = l7rule_flows.L7RuleFlows()
self._amphora_repo = repo.AmphoraRepository()
self._amphora_health_repo = repo.AmphoraHealthRepository()
self._health_mon_repo = repo.HealthMonitorRepository()
self._lb_repo = repo.LoadBalancerRepository()
self._listener_repo = repo.ListenerRepository()
self._member_repo = repo.MemberRepository()
self._pool_repo = repo.PoolRepository()
self._l7policy_repo = repo.L7PolicyRepository()
self._l7rule_repo = repo.L7RuleRepository()
self._exclude_result_logging_tasks = (
constants.ROLE_STANDALONE + '-' +
constants.CREATE_AMP_FOR_LB_SUBFLOW + '-' +
constants.GENERATE_SERVER_PEM,
constants.ROLE_BACKUP + '-' + constants.CREATE_AMP_FOR_LB_SUBFLOW +
'-' + constants.GENERATE_SERVER_PEM, constants.ROLE_MASTER + '-' +
constants.CREATE_AMP_FOR_LB_SUBFLOW + '-' +
constants.GENERATE_SERVER_PEM, constants.GENERATE_SERVER_PEM_TASK)
super(ControllerWorker, self).__init__()
def __init__(self):
LOG.info('Health Manager starting.')
self.seq = 0
self.amp_repo = repo.AmphoraRepository()
self.listener_repo = repo.ListenerRepository()
self.amp_health_repo = repo.AmphoraHealthRepository()
self.lb_repo = repo.LoadBalancerRepository()
self.health_executor = futurist.ThreadPoolExecutor(
max_workers=CONF.health_manager.health_update_threads)
self.stats_executor = futurist.ThreadPoolExecutor(
max_workers=CONF.health_manager.stats_update_threads)
self.bigips = [bigip for bigip in self.initialize_bigips()]
# Cache reachability of every bigip
self.bigip_status = {bigip.hostname: False for bigip in self.bigips}
self._active_bigip = None
self._last_failover_check = 0
self._last_cleanup_check = 0
# Create RPC Client
topic = cfg.CONF.oslo_messaging.topic
self.target = messaging.Target(
namespace=o_const.RPC_NAMESPACE_CONTROLLER_AGENT,
topic=topic,
version="1.0",
fanout=False)
self.client = rpc.get_client(self.target)
if cfg.CONF.f5_agent.prometheus:
prometheus_port = CONF.f5_agent.prometheus_port
LOG.info('Starting Prometheus HTTP server on port {}'.format(
prometheus_port))
prometheus.start_http_server(prometheus_port)
def __init__(self, exit_event):
self.cw = cw.ControllerWorker()
self.threads = CONF.health_manager.failover_threads
self.executor = futures.ThreadPoolExecutor(max_workers=self.threads)
self.amp_repo = repo.AmphoraRepository()
self.amp_health_repo = repo.AmphoraHealthRepository()
self.lb_repo = repo.LoadBalancerRepository()
self.dead = exit_event
def __init__(self):
super().__init__()
# first setup repo for amphora, listener,member(nodes),pool repo
self.amphora_repo = repo.AmphoraRepository()
self.amphora_health_repo = repo.AmphoraHealthRepository()
self.listener_repo = repo.ListenerRepository()
self.loadbalancer_repo = repo.LoadBalancerRepository()
self.member_repo = repo.MemberRepository()
self.pool_repo = repo.PoolRepository()
def __init__(self, exit_event):
if CONF.api_settings.default_provider_driver == constants.AMPHORAV2:
self.cw = cw2.ControllerWorker()
else:
self.cw = cw1.ControllerWorker()
self.threads = CONF.health_manager.failover_threads
self.executor = futures.ThreadPoolExecutor(max_workers=self.threads)
self.amp_repo = repo.AmphoraRepository()
self.amp_health_repo = repo.AmphoraHealthRepository()
self.lb_repo = repo.LoadBalancerRepository()
self.dead = exit_event
def __init__(self, **kwargs):
self.amphora_repo = repo.AmphoraRepository()
self.health_mon_repo = repo.HealthMonitorRepository()
self.listener_repo = repo.ListenerRepository()
self.loadbalancer_repo = repo.LoadBalancerRepository()
self.member_repo = repo.MemberRepository()
self.pool_repo = repo.PoolRepository()
self.amp_health_repo = repo.AmphoraHealthRepository()
self.l7policy_repo = repo.L7PolicyRepository()
self.l7rule_repo = repo.L7RuleRepository()
super(TaskUtils, self).__init__(**kwargs)
def __init__(self):
super(UpdateHealthDb, self).__init__()
# first setup repo for amphora, listener,member(nodes),pool repo
self.event_streamer = stevedore_driver.DriverManager(
namespace='octavia.controller.queues',
name=CONF.health_manager.event_streamer_driver,
invoke_on_load=True).driver
self.amphora_repo = repo.AmphoraRepository()
self.amphora_health_repo = repo.AmphoraHealthRepository()
self.listener_repo = repo.ListenerRepository()
self.loadbalancer_repo = repo.LoadBalancerRepository()
self.member_repo = repo.MemberRepository()
self.pool_repo = repo.PoolRepository()
def health_check(self):
amp_health_repo = repo.AmphoraHealthRepository()
with futures.ThreadPoolExecutor(max_workers=self.threads) as executor:
# Don't start checking immediately, as the health manager may
# have been down for a while and amphorae not able to check in.
LOG.debug("Pausing before starting health check")
time.sleep(CONF.health_manager.heartbeat_timeout)
while True:
LOG.debug("Starting amphora health check")
failover_count = 0
while True:
lock_session = db_api.get_session(autocommit=False)
amp = None
try:
amp = amp_health_repo.get_stale_amphora(lock_session)
lock_session.commit()
except db_exc.DBDeadlock:
LOG.debug('Database reports deadlock. Skipping.')
try:
lock_session.rollback()
except Exception:
pass
except db_exc.RetryRequest:
LOG.debug('Database is requesting a retry. Skipping.')
try:
lock_session.rollback()
except Exception:
pass
except Exception:
with excutils.save_and_reraise_exception():
try:
lock_session.rollback()
except Exception:
pass
if amp is None:
break
failover_count += 1
LOG.info("Stale amphora's id is: %s",
amp.amphora_id)
executor.submit(self.cw.failover_amphora,
amp.amphora_id)
if failover_count > 0:
LOG.info("Failed over %s amphora",
failover_count)
time.sleep(CONF.health_manager.health_check_interval)
def __init__(self):
super(StatusManager,
self).__init__(bigip_urls=CONF.f5_agent.bigip_urls,
enable_verify=CONF.f5_agent.bigip_verify,
enable_token=CONF.f5_agent.bigip_token)
self.amphora_id = None
self.seq = 0
LOG.info('Health Manager Sender starting.')
self.amp_repo = repo.AmphoraRepository()
self.amp_health_repo = repo.AmphoraHealthRepository()
self.lb_repo = repo.LoadBalancerRepository()
self.health_executor = futurist.ThreadPoolExecutor(
max_workers=CONF.health_manager.health_update_threads)
self.stats_executor = futurist.ThreadPoolExecutor(
max_workers=CONF.health_manager.stats_update_threads)
if cfg.CONF.f5_agent.prometheus:
prometheus_port = CONF.f5_agent.prometheus_port
LOG.info('Starting Prometheus HTTP server on port {}'.format(
prometheus_port))
prometheus.start_http_server(prometheus_port)
def __init__(self):
self._amphora_repo = repo.AmphoraRepository()
self._amphora_health_repo = repo.AmphoraHealthRepository()
self._health_mon_repo = repo.HealthMonitorRepository()
self._lb_repo = repo.LoadBalancerRepository()
self._listener_repo = repo.ListenerRepository()
self._member_repo = repo.MemberRepository()
self._pool_repo = repo.PoolRepository()
self._l7policy_repo = repo.L7PolicyRepository()
self._l7rule_repo = repo.L7RuleRepository()
self._flavor_repo = repo.FlavorRepository()
self._az_repo = repo.AvailabilityZoneRepository()
persistence = tsk_driver.MysqlPersistenceDriver()
self.jobboard_driver = stevedore_driver.DriverManager(
namespace='octavia.worker.jobboard_driver',
name=CONF.task_flow.jobboard_backend_driver,
invoke_args=(persistence, ),
invoke_on_load=True).driver
def __init__(self):
self._amphora_flows = amphora_flows.AmphoraFlows()
self._health_monitor_flows = health_monitor_flows.HealthMonitorFlows()
self._lb_flows = load_balancer_flows.LoadBalancerFlows()
self._listener_flows = listener_flows.ListenerFlows()
self._member_flows = member_flows.MemberFlows()
self._pool_flows = pool_flows.PoolFlows()
self._l7policy_flows = l7policy_flows.L7PolicyFlows()
self._l7rule_flows = l7rule_flows.L7RuleFlows()
self._amphora_repo = repo.AmphoraRepository()
self._amphora_health_repo = repo.AmphoraHealthRepository()
self._health_mon_repo = repo.HealthMonitorRepository()
self._lb_repo = repo.LoadBalancerRepository()
self._listener_repo = repo.ListenerRepository()
self._member_repo = repo.MemberRepository()
self._pool_repo = repo.PoolRepository()
self._l7policy_repo = repo.L7PolicyRepository()
self._l7rule_repo = repo.L7RuleRepository()
self._flavor_repo = repo.FlavorRepository()
super(ControllerWorker, self).__init__()
def health_check(self):
amp_health_repo = repo.AmphoraHealthRepository()
with futures.ThreadPoolExecutor(max_workers=self.threads) as executor:
try:
while True:
time.sleep(CONF.health_manager.health_check_interval)
session = db_api.get_session()
LOG.debug("Starting amphora health check")
failover_count = 0
while True:
amp = amp_health_repo.get_stale_amphora(session)
if amp is None:
break
failover_count += 1
LOG.info(_LI("Stale amphora's id is: %s"),
amp.amphora_id)
executor.submit(self.cw.failover_amphora,
amp.amphora_id)
if failover_count > 0:
LOG.info(_LI("Failed over %s amphora"), failover_count)
finally:
executor.shutdown(wait=True)
def __init__(self):
self.amp_repo = repo.AmphoraRepository()
self.amp_health_repo = repo.AmphoraHealthRepository()
self.lb_repo = repo.LoadBalancerRepository()
lb_amp_dict[amp.load_balancer_id] = amp.id
mgmt_port = get_vm_mgmt_port(net_driver, amp.compute_id,
amp.lb_network_ip)
disable_port(net_driver, mgmt_port)
return lb_amp_dict
if __name__ == '__main__':
raw_input('First, please stop octavia house_keeping process, then press '
'any key to continue...')
service.prepare_service(sys.argv)
session = db_api.get_session()
amp_repo = repo.AmphoraRepository()
amphealth_repo = repo.AmphoraHealthRepository()
nova_manager = nova_driver.VirtualMachineManager()
net_driver = allowed_address_pairs.AllowedAddressPairsDriver()
# Delete all the amp vms in spare pool and corresponding db records.
amps = get_spare_amphoras(session)
LOG.debug("Step 1: Clean up %d amphorae vms in spare pool.", len(amps))
for amp in amps:
delete_amp(nova_manager, session, amp_repo, amphealth_repo, amp)
raw_input('Now, please start octavia house_keeping process, then press '
'any key to continue...')
# Wait for filling up the spare pool.
LOG.debug("Step 2: Waiting for spare pool to fill up again...")
def __init__(self):
self.amp_repo = repo.AmphoraRepository()
self.amp_health_repo = repo.AmphoraHealthRepository()
