def do_housekeeping(): is_leader = is_cluster_leader() if is_leader: JBoxAPI.log_info("I am the cluster leader") terminating = False else: try: terminating = JBoxAsyncJob.sync_is_terminating() if terminating['code'] == 0: terminating = terminating['data'] else: JBoxAPI.log_error( "Error checking if instance is terminating. Assuming False." ) terminating = False except: JBoxAPI.log_error( "Exception checking if instance is terminating. Assuming False." ) terminating = False if terminating: JBoxAPI.log_warn("terminating to scale down") else: APIContainer.maintain() JBoxAsyncJob.async_plugin_maintenance(is_leader)
def do_housekeeping(): terminating = False server_delete_timeout = JBoxCfg.get('interactive.expire') inactive_timeout = JBoxCfg.get('interactive.inactivity_timeout') SessContainer.maintain(max_timeout=server_delete_timeout, inactive_timeout=inactive_timeout) is_leader = is_cluster_leader() if is_leader: terminating = False else: try: terminating = JBoxAsyncJob.sync_is_terminating() if terminating['code'] == 0: terminating = terminating['data'] else: JBox.log_error("Error checking if instance is terminating. Assuming False.") terminating = False except: JBox.log_error("Exception checking if instance is terminating. Assuming False.") terminating = False if is_leader: JBox.log_info("I am the cluster leader") JBox.update_juliabox_status() JBox.monitor_registrations() if not JBoxDynConfig.is_stat_collected_within(Compute.get_install_id(), 1): JBoxAsyncJob.async_collect_stats() if terminating: JBox.log_warn("terminating to scale down") else: JBox.do_update_user_home_image() JBoxAsyncJob.async_plugin_maintenance(is_leader)
def __init__(self): LoggerMixin.configure() db.configure() Compute.configure() APIContainer.configure() JBoxAsyncJob.configure() JBoxAsyncJob.init(JBoxAsyncJob.MODE_PUB) self.application = tornado.web.Application(handlers=[ (r"^/", APIInfoHandler), (r"^/.*/.*", APIHandler) ]) self.application.settings["cookie_secret"] = JBoxCfg.get('sesskey') self.application.listen(JBoxCfg.get('api.manager_port'), address=socket.gethostname()) self.application.listen(JBoxCfg.get('api.manager_port'), address='localhost') self.ioloop = ioloop.IOLoop.instance() # run container maintainence every 5 minutes run_interval = 5 * 60 * 1000 self.log_info("Container maintenance every " + str(run_interval / (60 * 1000)) + " minutes") self.ct = ioloop.PeriodicCallback(JBoxAPI.do_housekeeping, run_interval, self.ioloop) self.sigct = ioloop.PeriodicCallback(JBoxAPI.do_signals, 1000, self.ioloop)
def __init__(self): LoggerMixin.configure() db.configure() Compute.configure() APIContainer.configure() JBoxAsyncJob.configure() JBoxAsyncJob.init(JBoxAsyncJob.MODE_PUB) self.application = tornado.web.Application( handlers=[(r"^/", APIInfoHandler), (r"^/.*/.*", APIHandler)]) self.application.settings["cookie_secret"] = JBoxCfg.get('sesskey') self.application.listen(JBoxCfg.get('api.manager_port'), address=socket.gethostname()) self.application.listen(JBoxCfg.get('api.manager_port'), address='localhost') self.ioloop = ioloop.IOLoop.instance() # run container maintainence every 5 minutes run_interval = 5 * 60 * 1000 self.log_info("Container maintenance every " + str(run_interval / (60 * 1000)) + " minutes") self.ct = ioloop.PeriodicCallback(JBoxAPI.do_housekeeping, run_interval, self.ioloop) self.sigct = ioloop.PeriodicCallback(JBoxAPI.do_signals, 1000, self.ioloop)
def __init__(self): LoggerMixin.configure() db.configure() Compute.configure() SessContainer.configure() VolMgr.configure() JBoxAsyncJob.configure() JBoxAsyncJob.init(JBoxAsyncJob.MODE_PUB) self.application = tornado.web.Application(handlers=[ (r"/", MainHandler), (r"/jboxadmin/", AdminHandler), (r"/jboxping/", PingHandler), (r"/jboxcors/", CorsHandler) ]) JBPluginHandler.add_plugin_handlers(self.application) JBPluginUI.create_include_files() # cookie_secret = ''.join(random.choice(string.ascii_uppercase + string.digits) for x in xrange(32)) # use sesskey as cookie secret to be able to span multiple tornado servers self.application.settings["cookie_secret"] = JBoxCfg.get('sesskey') self.application.settings["plugin_features"] = JBox.get_pluggedin_features() self.application.listen(JBoxCfg.get('interactive.manager_port'), address=socket.gethostname()) self.application.listen(JBoxCfg.get('interactive.manager_port'), address='localhost') self.ioloop = tornado.ioloop.IOLoop.instance() # run container maintainence every 5 minutes run_interval = 5 * 60 * 1000 self.log_info("Container maintenance every " + str(run_interval / (60 * 1000)) + " minutes") self.ct = tornado.ioloop.PeriodicCallback(JBox.do_housekeeping, run_interval, self.ioloop) self.sigct = tornado.ioloop.PeriodicCallback(JBox.do_signals, 1000, self.ioloop)
def run(self): JBox.do_update_user_home_image() JBoxAsyncJob.async_refresh_disks() JBox.log_debug("Setting up signal handlers") signal.signal(signal.SIGINT, JBox.signal_handler) signal.signal(signal.SIGTERM, JBox.signal_handler) JBox.log_debug("Starting ioloops") self.ct.start() self.sigct.start() self.ioloop.start() JBox.log_info("Stopped.")
def __init__(self): dckr = docker.Client() cfg = JBox.cfg = read_config() cloud_cfg = cfg['cloud_host'] LoggerMixin.setup_logger(level=cfg['root_log_level']) LoggerMixin.DEFAULT_LEVEL = cfg['jbox_log_level'] JBoxHandler.configure(cfg) db.configure_db(cfg) CloudHost.configure(has_s3=cloud_cfg['s3'], has_dynamodb=cloud_cfg['dynamodb'], has_cloudwatch=cloud_cfg['cloudwatch'], has_autoscale=cloud_cfg['autoscale'], has_route53=cloud_cfg['route53'], has_ebs=cloud_cfg['ebs'], has_ses=cloud_cfg['ses'], scale_up_at_load=cloud_cfg['scale_up_at_load'], scale_up_policy=cloud_cfg['scale_up_policy'], autoscale_group=cloud_cfg['autoscale_group'], route53_domain=cloud_cfg['route53_domain'], region=cloud_cfg['region'], install_id=cloud_cfg['install_id']) VolMgr.configure(dckr, cfg) JBoxAsyncJob.configure(cfg) JBoxContainer.configure(dckr, cfg['docker_image'], cfg['mem_limit'], cfg['cpu_limit'], cfg['numlocalmax'], cfg['async_job_ports']) self.application = tornado.web.Application([ (r"/", MainHandler), (r"/hostlaunchipnb/", AuthHandler), (r"/hostadmin/", AdminHandler), (r"/ping/", PingHandler), (r"/cors/", CorsHandler) ]) cookie_secret = ''.join( random.choice(string.ascii_uppercase + string.digits) for x in xrange(32)) self.application.settings["cookie_secret"] = cookie_secret self.application.settings["google_oauth"] = cfg["google_oauth"] self.application.listen(cfg["port"]) self.ioloop = tornado.ioloop.IOLoop.instance() # run container maintainence every 5 minutes run_interval = 5 * 60 * 1000 self.log_info("Container maintenance every " + str(run_interval / (60 * 1000)) + " minutes") self.ct = tornado.ioloop.PeriodicCallback(JBox.do_housekeeping, run_interval, self.ioloop)
def __init__(self): dckr = docker.Client() cfg = JBox.cfg = read_config() cloud_cfg = cfg['cloud_host'] LoggerMixin.setup_logger(level=cfg['root_log_level']) LoggerMixin.DEFAULT_LEVEL = cfg['jbox_log_level'] JBoxHandler.configure(cfg) db.configure_db(cfg) CloudHost.configure(has_s3=cloud_cfg['s3'], has_dynamodb=cloud_cfg['dynamodb'], has_cloudwatch=cloud_cfg['cloudwatch'], has_autoscale=cloud_cfg['autoscale'], has_route53=cloud_cfg['route53'], has_ebs=cloud_cfg['ebs'], has_ses=cloud_cfg['ses'], scale_up_at_load=cloud_cfg['scale_up_at_load'], scale_up_policy=cloud_cfg['scale_up_policy'], autoscale_group=cloud_cfg['autoscale_group'], route53_domain=cloud_cfg['route53_domain'], region=cloud_cfg['region'], install_id=cloud_cfg['install_id']) VolMgr.configure(dckr, cfg) JBoxAsyncJob.configure(cfg) JBoxContainer.configure(dckr, cfg['docker_image'], cfg['mem_limit'], cfg['cpu_limit'], cfg['numlocalmax'], cfg['async_job_ports']) self.application = tornado.web.Application([ (r"/", MainHandler), (r"/hostlaunchipnb/", AuthHandler), (r"/hostadmin/", AdminHandler), (r"/ping/", PingHandler), (r"/cors/", CorsHandler), (r"/hw/", HomeworkHandler) ]) cookie_secret = ''.join(random.choice(string.ascii_uppercase + string.digits) for x in xrange(32)) self.application.settings["cookie_secret"] = cookie_secret self.application.settings["google_oauth"] = cfg["google_oauth"] self.application.listen(cfg["port"]) self.ioloop = tornado.ioloop.IOLoop.instance() # run container maintainence every 5 minutes run_interval = 5 * 60 * 1000 self.log_info("Container maintenance every " + str(run_interval / (60 * 1000)) + " minutes") self.ct = tornado.ioloop.PeriodicCallback(JBox.do_housekeeping, run_interval, self.ioloop)
def update_juliabox_status(): instances = Compute.get_all_instances() in_error = 0 HTML = "<html><body><center><pre>\nJuliaBox is Up.\n\nLast updated: " + datetime.datetime.now().isoformat() + " UTC\n\nLoads: " for inst in instances: try: status = JBoxAsyncJob.sync_api_status(inst)['data'] HTML += (str(status['load']) + '% ') except: in_error += 1 pass HTML += ("\n\nErrors: " + str(in_error) + "\n\nAWS Status: <a href='http://status.aws.amazon.com/'>status.aws.amazon.com</a></pre></center></body></html>") plugin = JBPluginCloud.jbox_get_plugin(JBPluginCloud.JBP_BUCKETSTORE) bkt = JBoxCfg.get("cloud_host.status_bucket") if plugin is not None and bkt is not None: try: f = open("/tmp/index.html", "w") f.write(HTML) f.close() plugin.push(bkt, "/tmp/index.html") finally: os.remove("/tmp/index.html") else: JBox.log_debug("Status: %s", HTML) return None
def monitor_registrations(): max_rate = JBoxDynConfig.get_registration_hourly_rate(Compute.get_install_id()) rate = JBoxUserV2.count_created(1) reg_allowed = JBoxDynConfig.get_allow_registration(Compute.get_install_id()) JBox.log_debug("registration allowed: %r, rate: %d, max allowed: %d", reg_allowed, rate, max_rate) if (reg_allowed and (rate > max_rate*1.1)) or ((not reg_allowed) and (rate < max_rate*0.9)): reg_allowed = not reg_allowed JBox.log_warn("Changing registration allowed to %r", reg_allowed) JBoxDynConfig.set_allow_registration(Compute.get_install_id(), reg_allowed) if reg_allowed: num_pending_activations = JBoxUserV2.count_pending_activations() if num_pending_activations > 0: JBox.log_info("scheduling activations for %d pending activations", num_pending_activations) JBoxAsyncJob.async_schedule_activations()
def __init__(self): dckr = docker.Client() cfg = read_config() cloud_cfg = cfg['cloud_host'] user_activation_cfg = cfg['user_activation'] LoggerMixin.setup_logger(level=cfg['root_log_level']) LoggerMixin.DEFAULT_LEVEL = cfg['jbox_log_level'] db.configure_db(cfg) CloudHost.configure(has_s3=cloud_cfg['s3'], has_dynamodb=cloud_cfg['dynamodb'], has_cloudwatch=cloud_cfg['cloudwatch'], has_autoscale=cloud_cfg['autoscale'], has_route53=cloud_cfg['route53'], has_ebs=cloud_cfg['ebs'], has_ses=cloud_cfg['ses'], scale_up_at_load=cloud_cfg['scale_up_at_load'], scale_up_policy=cloud_cfg['scale_up_policy'], autoscale_group=cloud_cfg['autoscale_group'], route53_domain=cloud_cfg['route53_domain'], region=cloud_cfg['region'], install_id=cloud_cfg['install_id']) VolMgr.configure(dckr, cfg) JBoxAsyncJob.configure(cfg) JBoxContainer.configure(dckr, cfg['docker_image'], cfg['mem_limit'], cfg['cpu_limit'], cfg['numlocalmax'], cfg['async_job_ports'], async_mode=JBoxAsyncJob.MODE_SUB) self.log_debug("Backup daemon listening on ports: %s", repr(cfg['async_job_ports'])) JBoxd.QUEUE = JBoxContainer.ASYNC_JOB JBoxd.MAX_ACTIVATIONS_PER_SEC = user_activation_cfg[ 'max_activations_per_sec'] JBoxd.MAX_AUTO_ACTIVATIONS_PER_RUN = user_activation_cfg[ 'max_activations_per_run'] JBoxd.ACTIVATION_SUBJECT = user_activation_cfg['mail_subject'] JBoxd.ACTIVATION_BODY = user_activation_cfg['mail_body'] JBoxd.ACTIVATION_SENDER = user_activation_cfg['sender']
def __init__(self): LoggerMixin.configure() db.configure() Compute.configure() SessContainer.configure() APIContainer.configure() VolMgr.configure() JBoxAsyncJob.configure() JBoxAsyncJob.init(JBoxAsyncJob.MODE_SUB) self.log_debug("Container manager listening on ports: %s", repr(JBoxCfg.get('container_manager_ports'))) JBoxd.QUEUE = JBoxAsyncJob.get() JBoxd.MAX_ACTIVATIONS_PER_SEC = JBoxCfg.get('user_activation.max_activations_per_sec') JBoxd.MAX_AUTO_ACTIVATIONS_PER_RUN = JBoxCfg.get('user_activation.max_activations_per_run') JBoxd.ACTIVATION_SUBJECT = JBoxCfg.get('user_activation.mail_subject') JBoxd.ACTIVATION_BODY = JBoxCfg.get('user_activation.mail_body') JBoxd.ACTIVATION_SENDER = JBoxCfg.get('user_activation.sender')
def __init__(self): LoggerMixin.configure() db.configure() Compute.configure() SessContainer.configure() VolMgr.configure() JBoxAsyncJob.configure() JBoxAsyncJob.init(JBoxAsyncJob.MODE_PUB) self.application = tornado.web.Application( handlers=[(r"/", MainHandler), ( r"/jboxadmin/", AdminHandler), (r"/jboxping/", PingHandler), (r"/jboxcors/", CorsHandler)]) JBPluginHandler.add_plugin_handlers(self.application) JBPluginUI.create_include_files() # cookie_secret = ''.join(random.choice(string.ascii_uppercase + string.digits) for x in xrange(32)) # use sesskey as cookie secret to be able to span multiple tornado servers self.application.settings["cookie_secret"] = JBoxCfg.get('sesskey') self.application.settings[ "plugin_features"] = JBox.get_pluggedin_features() self.application.listen(JBoxCfg.get('interactive.manager_port'), address=socket.gethostname()) self.application.listen(JBoxCfg.get('interactive.manager_port'), address='localhost') self.ioloop = tornado.ioloop.IOLoop.instance() # run container maintainence every 5 minutes run_interval = 5 * 60 * 1000 self.log_info("Container maintenance every " + str(run_interval / (60 * 1000)) + " minutes") self.ct = tornado.ioloop.PeriodicCallback(JBox.do_housekeeping, run_interval, self.ioloop) self.sigct = tornado.ioloop.PeriodicCallback(JBox.do_signals, 1000, self.ioloop) # or configure cacerts AsyncHTTPClient.configure(None, defaults=dict(validate_cert=None))
def configure(dckr, image, mem_limit, cpu_limit, max_containers, async_job_ports, async_mode=JBoxAsyncJob.MODE_PUB): JBoxContainer.DCKR = dckr JBoxContainer.DCKR_IMAGE = image JBoxContainer.MEM_LIMIT = mem_limit JBoxContainer.CPU_LIMIT = cpu_limit JBoxContainer.MAX_CONTAINERS = max_containers JBoxContainer.ASYNC_JOB = JBoxAsyncJob(async_job_ports, async_mode)
def do_housekeeping(): is_leader = is_cluster_leader() if is_leader: JBoxAPI.log_info("I am the cluster leader") terminating = False else: try: terminating = JBoxAsyncJob.sync_is_terminating() if terminating['code'] == 0: terminating = terminating['data'] else: JBoxAPI.log_error("Error checking if instance is terminating. Assuming False.") terminating = False except: JBoxAPI.log_error("Exception checking if instance is terminating. Assuming False.") terminating = False if terminating: JBoxAPI.log_warn("terminating to scale down") else: APIContainer.maintain() JBoxAsyncJob.async_plugin_maintenance(is_leader)
def do_housekeeping(): terminating = False server_delete_timeout = JBoxCfg.get('interactive.expire') inactive_timeout = JBoxCfg.get('interactive.inactivity_timeout') SessContainer.maintain(max_timeout=server_delete_timeout, inactive_timeout=inactive_timeout) is_leader = is_cluster_leader() if is_leader: terminating = False else: try: terminating = JBoxAsyncJob.sync_is_terminating() if terminating['code'] == 0: terminating = terminating['data'] else: JBox.log_error( "Error checking if instance is terminating. Assuming False." ) terminating = False except: JBox.log_error( "Exception checking if instance is terminating. Assuming False." ) terminating = False if is_leader: JBox.log_info("I am the cluster leader") JBox.update_juliabox_status() JBox.monitor_registrations() if not JBoxDynConfig.is_stat_collected_within( Compute.get_install_id(), 1): JBoxAsyncJob.async_collect_stats() if terminating: JBox.log_warn("terminating to scale down") else: JBox.do_update_user_home_image() JBoxAsyncJob.async_plugin_maintenance(is_leader)
def monitor_registrations(): max_rate = JBoxDynConfig.get_registration_hourly_rate( Compute.get_install_id()) rate = JBoxUserV2.count_created(1) reg_allowed = JBoxDynConfig.get_allow_registration( Compute.get_install_id()) JBox.log_debug("registration allowed: %r, rate: %d, max allowed: %d", reg_allowed, rate, max_rate) if (reg_allowed and (rate > max_rate * 1.1)) or ((not reg_allowed) and (rate < max_rate * 0.9)): reg_allowed = not reg_allowed JBox.log_warn("Changing registration allowed to %r", reg_allowed) JBoxDynConfig.set_allow_registration(Compute.get_install_id(), reg_allowed) if reg_allowed: num_pending_activations = JBoxUserV2.count_pending_activations() if num_pending_activations > 0: JBox.log_info( "scheduling activations for %d pending activations", num_pending_activations) JBoxAsyncJob.async_schedule_activations()
def do_update_user_home_image(): if VolMgr.has_update_for_user_home_image(): if not VolMgr.update_user_home_image(fetch=False): JBoxAsyncJob.async_update_user_home_image()