def _poll_queue(sqs_config, queue, table, asg_name): """ Poll SQS queue. :param sqs_config: SQS daemon configuration :param queue: SQS Queue object connected to the cluster queue :param table: DB table resource object """ scheduler_module = load_module("sqswatcher.plugins." + sqs_config.scheduler) scheduler_module.init() max_cluster_size = None instance_type = None cluster_properties_refresh_timer = 0 while True: start_time = datetime.now() force_cluster_update = False # dynamically retrieve max_cluster_size and compute_instance_type if (not max_cluster_size or not instance_type or cluster_properties_refresh_timer >= CLUSTER_PROPERTIES_REFRESH_INTERVAL): cluster_properties_refresh_timer = 0 logging.info("Refreshing cluster properties") new_max_cluster_size = retrieve_max_cluster_size( sqs_config.region, sqs_config.proxy_config, asg_name, fallback=max_cluster_size) new_instance_type = get_compute_instance_type( sqs_config.region, sqs_config.proxy_config, sqs_config.stack_name, fallback=instance_type) force_cluster_update = new_max_cluster_size != max_cluster_size or new_instance_type != instance_type if new_instance_type != instance_type: instance_type = new_instance_type instance_properties = get_instance_properties( sqs_config.region, sqs_config.proxy_config, instance_type) max_cluster_size = new_max_cluster_size cluster_properties_refresh_timer += LOOP_TIME messages = _retrieve_all_sqs_messages( queue, sqs_config.max_processed_messages) update_events = _parse_sqs_messages(sqs_config.region, sqs_config.proxy_config, messages, table, queue) _process_sqs_messages( update_events, scheduler_module, sqs_config, table, queue, max_cluster_size, instance_properties, force_cluster_update, ) sleep_remaining_loop_time(LOOP_TIME, start_time)
def main(): logging.basicConfig( level=logging.INFO, format="%(asctime)s %(levelname)s [%(module)s:%(funcName)s] %(message)s" ) log.info("nodewatcher startup") try: config = _get_config() scheduler_module = load_module("nodewatcher.plugins." + config.scheduler) instance_id = _get_metadata("instance-id") hostname = _get_metadata("local-hostname") instance_type = _get_metadata("instance-type") log.info("Instance id is %s, hostname is %s, instance type is %s", instance_id, hostname, instance_type) asg_name = get_asg_name(config.stack_name, config.region, config.proxy_config) _poll_instance_status(config, scheduler_module, asg_name, hostname, instance_id, instance_type) except Exception as e: log.exception("An unexpected error occurred: %s", e) raise
def _poll_queue(sqs_config, queue, table, asg_name): """ Poll SQS queue. :param sqs_config: SQS daemon configuration :param queue: SQS Queue object connected to the cluster queue :param table: DB table resource object """ scheduler_module = load_module("sqswatcher.plugins." + sqs_config.scheduler) max_cluster_size = sqs_config.max_queue_size while True: new_max_cluster_size = _retrieve_max_cluster_size( sqs_config, asg_name, max_cluster_size) messages = _retrieve_all_sqs_messages(queue) update_events = _parse_sqs_messages(messages, table) _process_sqs_messages( update_events, scheduler_module, sqs_config, table, queue, new_max_cluster_size, new_max_cluster_size != max_cluster_size, ) max_cluster_size = new_max_cluster_size time.sleep(30)
def main(): logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s [%(module)s:%(funcName)s] %(message)s") log.info("jobwatcher startup") try: config = _get_config() asg_name = get_asg_name(config.stack_name, config.region, config.proxy_config, log) instance_properties = _get_instance_properties(config) scheduler_module = load_module("jobwatcher.plugins." + config.scheduler) _poll_scheduler_status(config, asg_name, scheduler_module, instance_properties) except Exception as e: log.critical("An unexpected error occurred: %s", e) raise
def main(): logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s [%(module)s:%(funcName)s] %(message)s") log.info("nodewatcher startup") try: config = _get_config() scheduler_module = load_module("nodewatcher.plugins." + config.scheduler) instance_id = _get_metadata("instance-id") hostname = _get_metadata("local-hostname") log.info("Instance id is %s, hostname is %s", instance_id, hostname) asg_name = get_asg_name(config.stack_name, config.region, config.proxy_config, log) _poll_instance_status(config, scheduler_module, asg_name, hostname, instance_id) except Exception as e: log.critical("An unexpected error occurred: %s", e) raise
def main(): logging.basicConfig( level=logging.INFO, format="%(asctime)s %(levelname)s [%(module)s:%(funcName)s] %(message)s" ) log.info("jobwatcher startup") try: config = _get_config() asg_name = get_asg_name(config.stack_name, config.region, config.proxy_config) scheduler_module = load_module("jobwatcher.plugins." + config.scheduler) _poll_scheduler_status(config, asg_name, scheduler_module) except Exception as e: log.critical("An unexpected error occurred: %s", e) raise