def process_service_instance(self, service_instance):
bounce_timers = self.setup_timers(service_instance)
self.log.info(f"{self.name} processing {service_instance.service}.{service_instance.instance}")
bounce_timers.setup_marathon.start()
return_code, bounce_again_in_seconds = deploy_marathon_service(
service=service_instance.service,
instance=service_instance.instance,
clients=self.marathon_clients,
soa_dir=marathon_tools.DEFAULT_SOA_DIR,
marathon_apps_with_clients=None,
)
bounce_timers.setup_marathon.stop()
self.log.info("setup marathon completed with exit code {} for {}.{}".format(
return_code,
service_instance.service,
service_instance.instance,
))
if bounce_again_in_seconds:
bounce_timers.processed_by_worker.start()
self.log.info("{}.{} not in steady state so bouncing again in {} "
"seconds".format(
service_instance.service,
service_instance.instance,
bounce_again_in_seconds,
))
else:
bounce_timers.bounce_length.stop()
self.log.info("{}.{} in steady state".format(
service_instance.service,
service_instance.instance,
))
return BounceResults(bounce_again_in_seconds, return_code, bounce_timers)
def process_service_instance(self, service_instance):
bounce_timers = self.setup_timers(service_instance)
human_bounce_by = humanize.naturaldelta(
datetime.timedelta(seconds=(time.time() -
service_instance.bounce_by)))
self.log.info(
f"{self.name} processing {service_instance.service}.{service_instance.instance} (bounce_by {human_bounce_by} ago)"
) # noqa E501
bounce_timers.setup_marathon.start()
return_code, bounce_again_in_seconds = deploy_marathon_service(
service=service_instance.service,
instance=service_instance.instance,
clients=self.marathon_clients,
soa_dir=marathon_tools.DEFAULT_SOA_DIR,
marathon_apps_with_clients=None,
)
bounce_timers.setup_marathon.stop()
self.log.info(
f"{self.name} setup marathon completed with exit code {return_code} for {service_instance.service}.{service_instance.instance}"
) # noqa E501
if bounce_again_in_seconds:
bounce_timers.processed_by_worker.start()
self.log.info(
f"{self.name} {service_instance.service}.{service_instance.instance} not in steady state so bouncing again in {bounce_again_in_seconds}"
) # noqa E501
else:
self.log.info(
f"{self.name} {service_instance.service}.{service_instance.instance} in steady state"
)
if service_instance.processed_count > 0:
bounce_timers.bounce_length.stop()
return BounceResults(bounce_again_in_seconds, return_code,
bounce_timers)
def run(self):
self.log.info("{} starting up".format(self.name))
while True:
service_instance = self.bounce_q.get()
failures = service_instance.failures
bounce_timers = self.setup_timers(service_instance)
self.log.info("{} processing {}.{}".format(
self.name, service_instance.service,
service_instance.instance))
marathon_apps = marathon_tools.get_all_marathon_apps(
self.marathon_client, embed_failures=True)
bounce_timers.setup_marathon.start()
try:
return_code, bounce_again_in_seconds = deploy_marathon_service(
service=service_instance.service,
instance=service_instance.instance,
client=self.marathon_client,
soa_dir=marathon_tools.DEFAULT_SOA_DIR,
marathon_config=self.marathon_config,
marathon_apps=marathon_apps)
except Exception as e:
self.log.warning(
"deploy_marathon_service caused exception: {}".format(e))
return_code = -2
if return_code != 0:
failures += 1
bounce_again_in_seconds = exponential_back_off(
failures=failures,
factor=self.config.
get_deployd_worker_failure_backoff_factor(),
base=2,
max_time=6000)
bounce_timers.setup_marathon.stop()
self.log.info(
"setup marathon completed with exit code {} for {}.{}".format(
return_code, service_instance.service,
service_instance.instance))
if bounce_again_in_seconds:
bounce_timers.processed_by_worker.start()
self.log.info(
"{}.{} not in steady state so bouncing again in {} "
"seconds".format(service_instance.service,
service_instance.instance,
bounce_again_in_seconds))
service_instance = ServiceInstance(
service=service_instance.service,
instance=service_instance.instance,
bounce_by=int(time.time()) + bounce_again_in_seconds,
watcher=self.name,
bounce_timers=bounce_timers,
failures=failures)
self.inbox_q.put(service_instance)
else:
bounce_timers.bounce_length.stop()
self.log.info("{}.{} in steady state".format(
service_instance.service, service_instance.instance))
time.sleep(0.1)
def process_service_instance(self, service_instance):
bounce_timers = self.setup_timers(service_instance)
self.log.info("{} processing {}.{}".format(self.name,
service_instance.service,
service_instance.instance))
# TODO: change this to use get_all_clients_for_service() instead.
unique_clients = self.marathon_clients.get_all_clients()
marathon_apps_with_clients = marathon_tools.get_marathon_apps_with_clients(
unique_clients, embed_tasks=True)
bounce_timers.setup_marathon.start()
return_code, bounce_again_in_seconds = deploy_marathon_service(
service=service_instance.service,
instance=service_instance.instance,
clients=self.marathon_clients,
soa_dir=marathon_tools.DEFAULT_SOA_DIR,
marathon_apps_with_clients=marathon_apps_with_clients,
)
bounce_timers.setup_marathon.stop()
self.log.info(
"setup marathon completed with exit code {} for {}.{}".format(
return_code,
service_instance.service,
service_instance.instance,
))
if bounce_again_in_seconds:
bounce_timers.processed_by_worker.start()
self.log.info("{}.{} not in steady state so bouncing again in {} "
"seconds".format(
service_instance.service,
service_instance.instance,
bounce_again_in_seconds,
))
else:
bounce_timers.bounce_length.stop()
self.log.info("{}.{} in steady state".format(
service_instance.service,
service_instance.instance,
))
return BounceResults(bounce_again_in_seconds, return_code,
bounce_timers)
def run(self):
self.log.info("{} starting up".format(self.name))
while True:
service_instance = self.bounce_q.get()
bounce_timers = self.setup_timers(service_instance)
self.log.info("{} processing {}.{}".format(
self.name, service_instance.service,
service_instance.instance))
marathon_apps = marathon_tools.get_all_marathon_apps(
self.marathon_client, embed_failures=True)
bounce_timers.setup_marathon.start()
return_code, bounce_again_in_seconds = deploy_marathon_service(
service=service_instance.service,
instance=service_instance.instance,
client=self.marathon_client,
soa_dir=marathon_tools.DEFAULT_SOA_DIR,
marathon_config=self.marathon_config,
marathon_apps=marathon_apps)
bounce_timers.setup_marathon.stop()
self.log.info(
"setup marathon completed with exit code {} for {}.{}".format(
return_code, service_instance.service,
service_instance.instance))
if bounce_again_in_seconds:
bounce_timers.processed_by_worker.start()
self.log.info(
"{}.{} not in steady state so bouncing again in {} "
"seconds".format(service_instance.service,
service_instance.instance,
bounce_again_in_seconds))
service_instance = ServiceInstance(
service=service_instance.service,
instance=service_instance.instance,
bounce_by=int(time.time()) + bounce_again_in_seconds,
watcher=self.name,
bounce_timers=bounce_timers)
self.inbox_q.put(service_instance)
else:
bounce_timers.bounce_length.stop()
self.log.info("{}.{} in steady state".format(
service_instance.service, service_instance.instance))
time.sleep(0.1)