def _poll_queue(sqs_config, queue, table, asg_name):
"""
Poll SQS queue.
:param sqs_config: SQS daemon configuration
:param queue: SQS Queue object connected to the cluster queue
:param table: DB table resource object
"""
scheduler_module = load_module("sqswatcher.plugins." +
sqs_config.scheduler)
scheduler_module.init()
max_cluster_size = None
instance_type = None
cluster_properties_refresh_timer = 0
while True:
start_time = datetime.now()
force_cluster_update = False
# dynamically retrieve max_cluster_size and compute_instance_type
if (not max_cluster_size or not instance_type
or cluster_properties_refresh_timer >=
CLUSTER_PROPERTIES_REFRESH_INTERVAL):
cluster_properties_refresh_timer = 0
logging.info("Refreshing cluster properties")
new_max_cluster_size = retrieve_max_cluster_size(
sqs_config.region,
sqs_config.proxy_config,
asg_name,
fallback=max_cluster_size)
new_instance_type = get_compute_instance_type(
sqs_config.region,
sqs_config.proxy_config,
sqs_config.stack_name,
fallback=instance_type)
force_cluster_update = new_max_cluster_size != max_cluster_size or new_instance_type != instance_type
if new_instance_type != instance_type:
instance_type = new_instance_type
instance_properties = get_instance_properties(
sqs_config.region, sqs_config.proxy_config, instance_type)
max_cluster_size = new_max_cluster_size
cluster_properties_refresh_timer += LOOP_TIME
messages = _retrieve_all_sqs_messages(
queue, sqs_config.max_processed_messages)
update_events = _parse_sqs_messages(sqs_config.region,
sqs_config.proxy_config, messages,
table, queue)
_process_sqs_messages(
update_events,
scheduler_module,
sqs_config,
table,
queue,
max_cluster_size,
instance_properties,
force_cluster_update,
)
sleep_remaining_loop_time(LOOP_TIME, start_time)
def main():
logging.basicConfig(
level=logging.INFO,
format="%(asctime)s %(levelname)s [%(module)s:%(funcName)s] %(message)s"
)
log.info("nodewatcher startup")
try:
config = _get_config()
scheduler_module = load_module("nodewatcher.plugins." +
config.scheduler)
instance_id = _get_metadata("instance-id")
hostname = _get_metadata("local-hostname")
instance_type = _get_metadata("instance-type")
log.info("Instance id is %s, hostname is %s, instance type is %s",
instance_id, hostname, instance_type)
asg_name = get_asg_name(config.stack_name, config.region,
config.proxy_config)
_poll_instance_status(config, scheduler_module, asg_name, hostname,
instance_id, instance_type)
except Exception as e:
log.exception("An unexpected error occurred: %s", e)
raise
def _poll_queue(sqs_config, queue, table, asg_name):
"""
Poll SQS queue.
:param sqs_config: SQS daemon configuration
:param queue: SQS Queue object connected to the cluster queue
:param table: DB table resource object
"""
scheduler_module = load_module("sqswatcher.plugins." +
sqs_config.scheduler)
max_cluster_size = sqs_config.max_queue_size
while True:
new_max_cluster_size = _retrieve_max_cluster_size(
sqs_config, asg_name, max_cluster_size)
messages = _retrieve_all_sqs_messages(queue)
update_events = _parse_sqs_messages(messages, table)
_process_sqs_messages(
update_events,
scheduler_module,
sqs_config,
table,
queue,
new_max_cluster_size,
new_max_cluster_size != max_cluster_size,
)
max_cluster_size = new_max_cluster_size
time.sleep(30)
def main():
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s [%(module)s:%(funcName)s] %(message)s")
log.info("jobwatcher startup")
try:
config = _get_config()
asg_name = get_asg_name(config.stack_name, config.region, config.proxy_config, log)
instance_properties = _get_instance_properties(config)
scheduler_module = load_module("jobwatcher.plugins." + config.scheduler)
_poll_scheduler_status(config, asg_name, scheduler_module, instance_properties)
except Exception as e:
log.critical("An unexpected error occurred: %s", e)
raise
def main():
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s [%(module)s:%(funcName)s] %(message)s")
log.info("nodewatcher startup")
try:
config = _get_config()
scheduler_module = load_module("nodewatcher.plugins." + config.scheduler)
instance_id = _get_metadata("instance-id")
hostname = _get_metadata("local-hostname")
log.info("Instance id is %s, hostname is %s", instance_id, hostname)
asg_name = get_asg_name(config.stack_name, config.region, config.proxy_config, log)
_poll_instance_status(config, scheduler_module, asg_name, hostname, instance_id)
except Exception as e:
log.critical("An unexpected error occurred: %s", e)
raise
def main():
logging.basicConfig(
level=logging.INFO,
format="%(asctime)s %(levelname)s [%(module)s:%(funcName)s] %(message)s"
)
log.info("jobwatcher startup")
try:
config = _get_config()
asg_name = get_asg_name(config.stack_name, config.region,
config.proxy_config)
scheduler_module = load_module("jobwatcher.plugins." +
config.scheduler)
_poll_scheduler_status(config, asg_name, scheduler_module)
except Exception as e:
log.critical("An unexpected error occurred: %s", e)
raise
