def check_for_celery():
celery, created = ExternalToolStatus.objects.get_or_create(name=ExternalToolStatus.CELERY_TOOL_NAME)
if celery.is_active:
try:
array = ping() #pings celery to see if it's alive
if len(array) == 0:
celery.status = ExternalToolStatus.FAILURE_STATUS #celery is gone, get error thrown
else:
celery.status = ExternalToolStatus.SUCCESS_STATUS #celery is alive
except IOError:
logger.info("core.tasks.check_for_celery: Celeryd could not connect to the broker (e.g. RabbitMQ). Please restart it.")
celery.status = ExternalToolStatus.FAILURE_STATUS #quit with error
except AMQPConnectionException:
logger.info("core.tasks.check_for_celery: Celeryd could not connect to the broker (e.g. RabbitMQ). Please restart it.")
celery.status = ExternalToolStatus.FAILURE_STATUS #quit with error
except AMQPChannelException:
logger.error("AMQPChannelException raised by ping(). Is your broker (e.g. RabbitMQ) available?")
celery.status = ExternalToolStatus.SUCCESS_STATUS
except socket.error as e:
logger.info("core.tasks.check_for_celery: Celeryd could not connect to the broker (e.g. RabbitMQ). Please restart it.")
celery.status = ExternalToolStatus.FAILURE_STATUS #quit with error
except:
logger.info("core.tasks.check_for_celery: Something went wrong, check the stack trace below for what")
celery.status = ExternalToolStatus.FAILURE_STATUS
#set last time check to now
celery.last_time_check = datetime.now()
#save status
celery.save()
def checkWorkersState():
celery_config = Loader().read_configuration()
registered_workers = celery_config["WORKERS"]
active_workers = [ping_answer.keys().pop() for ping_answer in ping()]
for worker in registered_workers:
if worker not in active_workers:
#TODO e-mail извещение
logging.error("WORKER {0} IS NOT AVAILABLE".format(worker))
def test_celeries_running(self):
"""
Ensure the expected number of celeryd instances are running.
"""
pongs = control.ping()
self.assertEqual(len(pongs), len(CeleryQueuesTestCase.celeryd_hosts),
"Number of hosts responding to ping should match " +
"number of test hosts.")
for pong in pongs:
for hostname in pong.keys():
self.assertTrue(hostname in CeleryQueuesTestCase.celeryd_hosts)
def health_details():
try:
worker_name = f"celery@{_namespace()}"
workers = control.ping(destination=[worker_name])
if not workers:
return (
"fail",
{
"celery:ping": {
"componentType": "component",
"status": "fail",
"time": datetime.utcnow().isoformat(),
"output": f"No {worker_name} workers could be found.",
}
},
)
return (
"pass",
{
"celery:ping": {
"componentType": "component",
"observedValue": workers,
"status": "pass",
"time": datetime.utcnow().isoformat(),
}
},
)
except Exception as e:
return (
"fail",
{
"celery:ping": {
"componentType": "component",
"status": "fail",
"time": datetime.utcnow().isoformat(),
"output": str(e),
}
},
)
def check_for_celery():
celery, created = ExternalToolStatus.objects.get_or_create(
name=ExternalToolStatus.CELERY_TOOL_NAME)
if celery.is_active:
try:
array = ping() #pings celery to see if it's alive
if len(array) == 0:
celery.status = ExternalToolStatus.FAILURE_STATUS #celery is gone, get error thrown
else:
celery.status = ExternalToolStatus.SUCCESS_STATUS #celery is alive
except IOError:
logger.info(
"core.tasks.check_for_celery: Celeryd could not connect to the broker (e.g. RabbitMQ). Please restart it."
)
celery.status = ExternalToolStatus.FAILURE_STATUS #quit with error
except AMQPConnectionException:
logger.info(
"core.tasks.check_for_celery: Celeryd could not connect to the broker (e.g. RabbitMQ). Please restart it."
)
celery.status = ExternalToolStatus.FAILURE_STATUS #quit with error
except AMQPChannelException:
logger.error(
"AMQPChannelException raised by ping(). Is your broker (e.g. RabbitMQ) available?"
)
celery.status = ExternalToolStatus.SUCCESS_STATUS
except socket.error as e:
logger.info(
"core.tasks.check_for_celery: Celeryd could not connect to the broker (e.g. RabbitMQ). Please restart it."
)
celery.status = ExternalToolStatus.FAILURE_STATUS #quit with error
except:
logger.info(
"core.tasks.check_for_celery: Something went wrong, check the stack trace below for what"
)
celery.status = ExternalToolStatus.FAILURE_STATUS
#set last time check to now
celery.last_time_check = datetime.now()
#save status
celery.save()
def is_alive(self, timeout=1):
r = ping(destination=[self.hostname], timeout=timeout)
return self.hostname in flatten_reply(r)
def is_alive(self, timeout=1):
r = ping(destination=[self.hostname],
timeout=timeout)
return self.hostname in flatten_reply(r)
def test_ping(self):
control.ping()
self.assertIn("ping", MockBroadcastPublisher.sent)
def list_workers(timeout=0.5):
return [item.keys()[0] for item in control.ping(timeout=timeout)]
def get_ping():
r = ping(app=ce)
current_app.logger.debug("Ping {0} ".format(r))
return r