def _keep_alive_thread():
last_update = None
proc_stat = None
settings.local.host_ping_timestamp = utils.now()
while True:
try:
timestamp = utils.now()
timestamp -= datetime.timedelta(
microseconds=timestamp.microsecond,
seconds=timestamp.second,
)
if timestamp != last_update:
last_update = timestamp
last_proc_stat = proc_stat
proc_stat = host.usage_utils.get_proc_stat()
if last_proc_stat and proc_stat:
cpu_usage = host.usage_utils.calc_cpu_usage(
last_proc_stat, proc_stat)
mem_usage = host.usage_utils.get_mem_usage()
settings.local.host.usage.add_period(
timestamp, cpu_usage, mem_usage)
yield interrupter_sleep(settings.app.host_ttl - 10)
ping_timestamp = utils.now()
settings.local.host.collection.update(
{
'_id': settings.local.host.id,
}, {
'$set': {
'status': ONLINE,
'ping_timestamp': utils.now(),
'auto_public_address': settings.local.public_ip,
}
})
if settings.local.host.auto_public_address != \
settings.local.public_ip:
settings.local.host.auto_public_address = \
settings.local.public_ip
event.Event(type=HOSTS_UPDATED)
settings.local.host_ping_timestamp = ping_timestamp
except GeneratorExit:
host.deinit()
raise
except:
logger.exception(
'Error in host keep alive update',
'runners',
host_id=settings.local.host.id,
host_name=settings.local.host.name,
)
time.sleep(0.5)
def _keep_alive_thread():
last_update = None
proc_stat = None
settings.local.host_ping_timestamp = utils.now()
while True:
try:
timestamp = utils.now()
timestamp -= datetime.timedelta(
microseconds=timestamp.microsecond,
seconds=timestamp.second,
)
if timestamp != last_update:
last_update = timestamp
last_proc_stat = proc_stat
proc_stat = host.usage_utils.get_proc_stat()
if last_proc_stat and proc_stat:
cpu_usage = host.usage_utils.calc_cpu_usage(
last_proc_stat, proc_stat)
mem_usage = host.usage_utils.get_mem_usage()
settings.local.host.usage.add_period(timestamp,
cpu_usage, mem_usage)
yield interrupter_sleep(settings.app.host_ttl - 10)
ping_timestamp = utils.now()
settings.local.host.collection.update({
'_id': settings.local.host.id,
}, {'$set': {
'status': ONLINE,
'ping_timestamp': utils.now(),
'auto_public_address': settings.local.public_ip,
'auto_public_address6': settings.local.public_ip6,
}})
if settings.local.host.auto_public_address != \
settings.local.public_ip or \
settings.local.host.auto_public_address6 != \
settings.local.public_ip6:
settings.local.host.auto_public_address = \
settings.local.public_ip
settings.local.host.auto_public_address6 = \
settings.local.public_ip6
event.Event(type=HOSTS_UPDATED)
settings.local.host_ping_timestamp = ping_timestamp
except GeneratorExit:
host.deinit()
raise
except:
logger.exception('Error in host keep alive update', 'runners',
host_id=settings.local.host.id,
host_name=settings.local.host.name,
)
time.sleep(0.5)
def _keep_alive_thread():
host_event = False
last_update = None
proc_stat = None
settings.local.host_ping_timestamp = utils.now()
cur_public_ip = None
cur_public_ip6 = None
cur_host_name = settings.local.host.name
cur_route53_region = settings.app.route53_region
cur_route53_zone = settings.app.route53_zone
auto_public_host = settings.local.host.auto_public_host
auto_public_host6 = settings.local.host.auto_public_host6
while True:
try:
timestamp = utils.now()
timestamp -= datetime.timedelta(
microseconds=timestamp.microsecond,
seconds=timestamp.second,
)
if timestamp != last_update:
last_update = timestamp
last_proc_stat = proc_stat
proc_stat = host.usage_utils.get_proc_stat()
if last_proc_stat and proc_stat:
cpu_usage = host.usage_utils.calc_cpu_usage(
last_proc_stat, proc_stat)
mem_usage = host.usage_utils.get_mem_usage()
settings.local.host.usage.add_period(timestamp,
cpu_usage, mem_usage)
yield interrupter_sleep(settings.app.host_ping)
ping_timestamp = utils.now()
try:
open_file_count = len(os.listdir('/proc/self/fd'))
except:
open_file_count = 0
cpu_usage = None
mem_usage = None
thread_count = threading.active_count()
server_count = len(host.global_servers)
device_count = host.global_clients.count({})
try:
cpu_usage, mem_usage = utils.get_process_cpu_mem()
except:
logger.exception('Failed to get process cpu and mem usage',
'runners',
host_id=settings.local.host.id,
host_name=settings.local.host.name,
)
host_name = settings.local.host.name
route53_region = settings.app.route53_region
route53_zone = settings.app.route53_zone
if route53_region and route53_zone:
if cur_public_ip != settings.local.public_ip or \
cur_public_ip6 != settings.local.public_ip6 or \
cur_host_name != host_name or \
cur_route53_region != route53_region or \
cur_route53_zone != route53_zone:
cur_host_name = host_name
cur_public_ip = settings.local.public_ip
cur_public_ip6 = settings.local.public_ip6
cur_route53_region = route53_region
cur_route53_zone = route53_zone
auto_public_host, auto_public_host6 = \
utils.set_zone_record(
route53_region,
route53_zone,
host_name,
cur_public_ip,
cur_public_ip6,
)
settings.local.host.auto_public_host = auto_public_host
settings.local.host.auto_public_host6 = auto_public_host6
host_event = True
else:
auto_public_host = None
auto_public_host6 = None
if settings.local.host.auto_public_address != \
settings.local.public_ip or \
settings.local.host.auto_public_address6 != \
settings.local.public_ip6:
settings.local.host.auto_public_address = \
settings.local.public_ip
settings.local.host.auto_public_address6 = \
settings.local.public_ip6
host_event = True
settings.local.host.collection.update({
'_id': settings.local.host.id,
}, {'$set': {
'server_count': server_count,
'device_count': device_count,
'cpu_usage': cpu_usage,
'mem_usage': mem_usage,
'thread_count': thread_count,
'open_file_count': open_file_count,
'status': ONLINE,
'ping_timestamp': utils.now(),
'auto_public_address': settings.local.public_ip,
'auto_public_address6': settings.local.public_ip6,
'auto_public_host': auto_public_host,
'auto_public_host6': auto_public_host6,
}})
if host_event:
host_event = False
event.Event(type=HOSTS_UPDATED)
monitoring.insert_point('system', {
'host': settings.local.host.name,
}, {
'cpu_usage': cpu_usage,
'mem_usage': mem_usage,
'thread_count': thread_count,
'open_file_count': open_file_count,
})
settings.local.host_ping_timestamp = ping_timestamp
except GeneratorExit:
host.deinit()
raise
except:
logger.exception('Error in host keep alive update', 'runners',
host_id=settings.local.host.id,
host_name=settings.local.host.name,
)
time.sleep(0.5)
def _keep_alive_thread():
last_update = None
proc_stat = None
settings.local.host_ping_timestamp = utils.now()
while True:
try:
timestamp = utils.now()
timestamp -= datetime.timedelta(microseconds=timestamp.microsecond, seconds=timestamp.second)
if timestamp != last_update:
last_update = timestamp
last_proc_stat = proc_stat
proc_stat = host.usage_utils.get_proc_stat()
if last_proc_stat and proc_stat:
cpu_usage = host.usage_utils.calc_cpu_usage(last_proc_stat, proc_stat)
mem_usage = host.usage_utils.get_mem_usage()
settings.local.host.usage.add_period(timestamp, cpu_usage, mem_usage)
yield interrupter_sleep(settings.app.host_ping)
ping_timestamp = utils.now()
cpu_usage = None
mem_usage = None
try:
cpu_usage, mem_usage = utils.get_process_cpu_mem()
except:
logger.exception(
"Failed to get process cpu and mem usage",
"runners",
host_id=settings.local.host.id,
host_name=settings.local.host.name,
)
settings.local.host.collection.update(
{"_id": settings.local.host.id},
{
"$set": {
"server_count": len(host.global_servers),
"device_count": host.global_clients.count({}),
"cpu_usage": cpu_usage,
"mem_usage": mem_usage,
"thread_count": threading.active_count(),
"status": ONLINE,
"ping_timestamp": utils.now(),
"auto_public_address": settings.local.public_ip,
"auto_public_address6": settings.local.public_ip6,
}
},
)
if (
settings.local.host.auto_public_address != settings.local.public_ip
or settings.local.host.auto_public_address6 != settings.local.public_ip6
):
settings.local.host.auto_public_address = settings.local.public_ip
settings.local.host.auto_public_address6 = settings.local.public_ip6
event.Event(type=HOSTS_UPDATED)
settings.local.host_ping_timestamp = ping_timestamp
except GeneratorExit:
host.deinit()
raise
except:
logger.exception(
"Error in host keep alive update",
"runners",
host_id=settings.local.host.id,
host_name=settings.local.host.name,
)
time.sleep(0.5)
def _keep_alive_thread():
last_update = None
proc_stat = None
settings.local.host_ping_timestamp = utils.now()
while True:
try:
timestamp = utils.now()
timestamp -= datetime.timedelta(
microseconds=timestamp.microsecond,
seconds=timestamp.second,
)
if timestamp != last_update:
last_update = timestamp
last_proc_stat = proc_stat
proc_stat = host.usage_utils.get_proc_stat()
if last_proc_stat and proc_stat:
cpu_usage = host.usage_utils.calc_cpu_usage(
last_proc_stat, proc_stat)
mem_usage = host.usage_utils.get_mem_usage()
settings.local.host.usage.add_period(
timestamp, cpu_usage, mem_usage)
yield interrupter_sleep(settings.app.host_ping)
ping_timestamp = utils.now()
cpu_usage = None
mem_usage = None
try:
cpu_usage, mem_usage = utils.get_process_cpu_mem()
except:
logger.exception(
'Failed to get process cpu and mem usage',
'runners',
host_id=settings.local.host.id,
host_name=settings.local.host.name,
)
settings.local.host.collection.update(
{
'_id': settings.local.host.id,
}, {
'$set': {
'server_count': len(host.global_servers),
'device_count': host.global_clients.count({}),
'cpu_usage': cpu_usage,
'mem_usage': mem_usage,
'thread_count': threading.active_count(),
'status': ONLINE,
'ping_timestamp': utils.now(),
'auto_public_address': settings.local.public_ip,
'auto_public_address6': settings.local.public_ip6,
}
})
if settings.local.host.auto_public_address != \
settings.local.public_ip or \
settings.local.host.auto_public_address6 != \
settings.local.public_ip6:
settings.local.host.auto_public_address = \
settings.local.public_ip
settings.local.host.auto_public_address6 = \
settings.local.public_ip6
event.Event(type=HOSTS_UPDATED)
settings.local.host_ping_timestamp = ping_timestamp
except GeneratorExit:
host.deinit()
raise
except:
logger.exception(
'Error in host keep alive update',
'runners',
host_id=settings.local.host.id,
host_name=settings.local.host.name,
)
time.sleep(0.5)
