コード例 #1
0
ファイル: srvr_jbapi.py プロジェクト: jngod2011/JuliaBox-1
    def do_housekeeping():
        is_leader = is_cluster_leader()
        if is_leader:
            JBoxAPI.log_info("I am the cluster leader")
            terminating = False
        else:
            try:
                terminating = JBoxAsyncJob.sync_is_terminating()
                if terminating['code'] == 0:
                    terminating = terminating['data']
                else:
                    JBoxAPI.log_error(
                        "Error checking if instance is terminating. Assuming False."
                    )
                    terminating = False
            except:
                JBoxAPI.log_error(
                    "Exception checking if instance is terminating. Assuming False."
                )
                terminating = False

        if terminating:
            JBoxAPI.log_warn("terminating to scale down")
        else:
            APIContainer.maintain()
            JBoxAsyncJob.async_plugin_maintenance(is_leader)
コード例 #2
0
ファイル: srvr_jbox.py プロジェクト: JuliaLang/JuliaBox
    def do_housekeeping():
        terminating = False
        server_delete_timeout = JBoxCfg.get('interactive.expire')
        inactive_timeout = JBoxCfg.get('interactive.inactivity_timeout')
        SessContainer.maintain(max_timeout=server_delete_timeout, inactive_timeout=inactive_timeout)
        is_leader = is_cluster_leader()

        if is_leader:
            terminating = False
        else:
            try:
                terminating = JBoxAsyncJob.sync_is_terminating()
                if terminating['code'] == 0:
                    terminating = terminating['data']
                else:
                    JBox.log_error("Error checking if instance is terminating. Assuming False.")
                    terminating = False
            except:
                JBox.log_error("Exception checking if instance is terminating. Assuming False.")
                terminating = False

        if is_leader:
            JBox.log_info("I am the cluster leader")
            JBox.update_juliabox_status()
            JBox.monitor_registrations()
            if not JBoxDynConfig.is_stat_collected_within(Compute.get_install_id(), 1):
                JBoxAsyncJob.async_collect_stats()

        if terminating:
            JBox.log_warn("terminating to scale down")
        else:
            JBox.do_update_user_home_image()
            JBoxAsyncJob.async_plugin_maintenance(is_leader)
コード例 #3
0
ファイル: srvr_jbapi.py プロジェクト: JuliaLang/JuliaBox
    def __init__(self):
        LoggerMixin.configure()
        db.configure()
        Compute.configure()
        APIContainer.configure()

        JBoxAsyncJob.configure()
        JBoxAsyncJob.init(JBoxAsyncJob.MODE_PUB)

        self.application = tornado.web.Application(handlers=[
            (r"^/", APIInfoHandler),
            (r"^/.*/.*", APIHandler)
        ])

        self.application.settings["cookie_secret"] = JBoxCfg.get('sesskey')
        self.application.listen(JBoxCfg.get('api.manager_port'), address=socket.gethostname())
        self.application.listen(JBoxCfg.get('api.manager_port'), address='localhost')

        self.ioloop = ioloop.IOLoop.instance()

        # run container maintainence every 5 minutes
        run_interval = 5 * 60 * 1000
        self.log_info("Container maintenance every " + str(run_interval / (60 * 1000)) + " minutes")
        self.ct = ioloop.PeriodicCallback(JBoxAPI.do_housekeeping, run_interval, self.ioloop)
        self.sigct = ioloop.PeriodicCallback(JBoxAPI.do_signals, 1000, self.ioloop)
コード例 #4
0
ファイル: srvr_jbapi.py プロジェクト: jngod2011/JuliaBox-1
    def __init__(self):
        LoggerMixin.configure()
        db.configure()
        Compute.configure()
        APIContainer.configure()

        JBoxAsyncJob.configure()
        JBoxAsyncJob.init(JBoxAsyncJob.MODE_PUB)

        self.application = tornado.web.Application(
            handlers=[(r"^/", APIInfoHandler), (r"^/.*/.*", APIHandler)])

        self.application.settings["cookie_secret"] = JBoxCfg.get('sesskey')
        self.application.listen(JBoxCfg.get('api.manager_port'),
                                address=socket.gethostname())
        self.application.listen(JBoxCfg.get('api.manager_port'),
                                address='localhost')

        self.ioloop = ioloop.IOLoop.instance()

        # run container maintainence every 5 minutes
        run_interval = 5 * 60 * 1000
        self.log_info("Container maintenance every " +
                      str(run_interval / (60 * 1000)) + " minutes")
        self.ct = ioloop.PeriodicCallback(JBoxAPI.do_housekeeping,
                                          run_interval, self.ioloop)
        self.sigct = ioloop.PeriodicCallback(JBoxAPI.do_signals, 1000,
                                             self.ioloop)
コード例 #5
0
ファイル: srvr_jbox.py プロジェクト: vk1975/JuliaBox
    def __init__(self):
        LoggerMixin.configure()
        db.configure()
        Compute.configure()
        SessContainer.configure()
        VolMgr.configure()

        JBoxAsyncJob.configure()
        JBoxAsyncJob.init(JBoxAsyncJob.MODE_PUB)

        self.application = tornado.web.Application(handlers=[
            (r"/", MainHandler),
            (r"/jboxadmin/", AdminHandler),
            (r"/jboxping/", PingHandler),
            (r"/jboxcors/", CorsHandler)
        ])
        JBPluginHandler.add_plugin_handlers(self.application)
        JBPluginUI.create_include_files()

        # cookie_secret = ''.join(random.choice(string.ascii_uppercase + string.digits) for x in xrange(32))
        # use sesskey as cookie secret to be able to span multiple tornado servers
        self.application.settings["cookie_secret"] = JBoxCfg.get('sesskey')
        self.application.settings["plugin_features"] = JBox.get_pluggedin_features()
        self.application.listen(JBoxCfg.get('interactive.manager_port'), address=socket.gethostname())
        self.application.listen(JBoxCfg.get('interactive.manager_port'), address='localhost')

        self.ioloop = tornado.ioloop.IOLoop.instance()

        # run container maintainence every 5 minutes
        run_interval = 5 * 60 * 1000
        self.log_info("Container maintenance every " + str(run_interval / (60 * 1000)) + " minutes")
        self.ct = tornado.ioloop.PeriodicCallback(JBox.do_housekeeping, run_interval, self.ioloop)
        self.sigct = tornado.ioloop.PeriodicCallback(JBox.do_signals, 1000, self.ioloop)
コード例 #6
0
ファイル: srvr_jbox.py プロジェクト: JuliaLang/JuliaBox
    def run(self):
        JBox.do_update_user_home_image()
        JBoxAsyncJob.async_refresh_disks()

        JBox.log_debug("Setting up signal handlers")
        signal.signal(signal.SIGINT, JBox.signal_handler)
        signal.signal(signal.SIGTERM, JBox.signal_handler)

        JBox.log_debug("Starting ioloops")
        self.ct.start()
        self.sigct.start()
        self.ioloop.start()
        JBox.log_info("Stopped.")
コード例 #7
0
    def run(self):
        JBox.do_update_user_home_image()
        JBoxAsyncJob.async_refresh_disks()

        JBox.log_debug("Setting up signal handlers")
        signal.signal(signal.SIGINT, JBox.signal_handler)
        signal.signal(signal.SIGTERM, JBox.signal_handler)

        JBox.log_debug("Starting ioloops")
        self.ct.start()
        self.sigct.start()
        self.ioloop.start()
        JBox.log_info("Stopped.")
コード例 #8
0
    def __init__(self):
        dckr = docker.Client()
        cfg = JBox.cfg = read_config()
        cloud_cfg = cfg['cloud_host']

        LoggerMixin.setup_logger(level=cfg['root_log_level'])
        LoggerMixin.DEFAULT_LEVEL = cfg['jbox_log_level']

        JBoxHandler.configure(cfg)
        db.configure_db(cfg)

        CloudHost.configure(has_s3=cloud_cfg['s3'],
                            has_dynamodb=cloud_cfg['dynamodb'],
                            has_cloudwatch=cloud_cfg['cloudwatch'],
                            has_autoscale=cloud_cfg['autoscale'],
                            has_route53=cloud_cfg['route53'],
                            has_ebs=cloud_cfg['ebs'],
                            has_ses=cloud_cfg['ses'],
                            scale_up_at_load=cloud_cfg['scale_up_at_load'],
                            scale_up_policy=cloud_cfg['scale_up_policy'],
                            autoscale_group=cloud_cfg['autoscale_group'],
                            route53_domain=cloud_cfg['route53_domain'],
                            region=cloud_cfg['region'],
                            install_id=cloud_cfg['install_id'])

        VolMgr.configure(dckr, cfg)
        JBoxAsyncJob.configure(cfg)
        JBoxContainer.configure(dckr, cfg['docker_image'], cfg['mem_limit'],
                                cfg['cpu_limit'], cfg['numlocalmax'],
                                cfg['async_job_ports'])

        self.application = tornado.web.Application([
            (r"/", MainHandler), (r"/hostlaunchipnb/", AuthHandler),
            (r"/hostadmin/", AdminHandler), (r"/ping/", PingHandler),
            (r"/cors/", CorsHandler)
        ])
        cookie_secret = ''.join(
            random.choice(string.ascii_uppercase + string.digits)
            for x in xrange(32))
        self.application.settings["cookie_secret"] = cookie_secret
        self.application.settings["google_oauth"] = cfg["google_oauth"]
        self.application.listen(cfg["port"])

        self.ioloop = tornado.ioloop.IOLoop.instance()

        # run container maintainence every 5 minutes
        run_interval = 5 * 60 * 1000
        self.log_info("Container maintenance every " +
                      str(run_interval / (60 * 1000)) + " minutes")
        self.ct = tornado.ioloop.PeriodicCallback(JBox.do_housekeeping,
                                                  run_interval, self.ioloop)
コード例 #9
0
ファイル: jbox.py プロジェクト: barrygolden/JuliaBox
    def __init__(self):
        dckr = docker.Client()
        cfg = JBox.cfg = read_config()
        cloud_cfg = cfg['cloud_host']

        LoggerMixin.setup_logger(level=cfg['root_log_level'])
        LoggerMixin.DEFAULT_LEVEL = cfg['jbox_log_level']

        JBoxHandler.configure(cfg)
        db.configure_db(cfg)

        CloudHost.configure(has_s3=cloud_cfg['s3'],
                            has_dynamodb=cloud_cfg['dynamodb'],
                            has_cloudwatch=cloud_cfg['cloudwatch'],
                            has_autoscale=cloud_cfg['autoscale'],
                            has_route53=cloud_cfg['route53'],
                            has_ebs=cloud_cfg['ebs'],
                            has_ses=cloud_cfg['ses'],
                            scale_up_at_load=cloud_cfg['scale_up_at_load'],
                            scale_up_policy=cloud_cfg['scale_up_policy'],
                            autoscale_group=cloud_cfg['autoscale_group'],
                            route53_domain=cloud_cfg['route53_domain'],
                            region=cloud_cfg['region'],
                            install_id=cloud_cfg['install_id'])

        VolMgr.configure(dckr, cfg)
        JBoxAsyncJob.configure(cfg)
        JBoxContainer.configure(dckr, cfg['docker_image'], cfg['mem_limit'], cfg['cpu_limit'],
                                cfg['numlocalmax'], cfg['async_job_ports'])

        self.application = tornado.web.Application([
            (r"/", MainHandler),
            (r"/hostlaunchipnb/", AuthHandler),
            (r"/hostadmin/", AdminHandler),
            (r"/ping/", PingHandler),
            (r"/cors/", CorsHandler),
            (r"/hw/", HomeworkHandler)
        ])
        cookie_secret = ''.join(random.choice(string.ascii_uppercase + string.digits) for x in xrange(32))
        self.application.settings["cookie_secret"] = cookie_secret
        self.application.settings["google_oauth"] = cfg["google_oauth"]
        self.application.listen(cfg["port"])

        self.ioloop = tornado.ioloop.IOLoop.instance()

        # run container maintainence every 5 minutes
        run_interval = 5 * 60 * 1000
        self.log_info("Container maintenance every " + str(run_interval / (60 * 1000)) + " minutes")
        self.ct = tornado.ioloop.PeriodicCallback(JBox.do_housekeeping, run_interval, self.ioloop)
コード例 #10
0
ファイル: srvr_jbox.py プロジェクト: vk1975/JuliaBox
    def update_juliabox_status():
        instances = Compute.get_all_instances()

        in_error = 0
        HTML = "<html><body><center><pre>\nJuliaBox is Up.\n\nLast updated: " + datetime.datetime.now().isoformat() + " UTC\n\nLoads: "

        for inst in instances:
            try:
                status = JBoxAsyncJob.sync_api_status(inst)['data']
                HTML += (str(status['load']) + '% ')
            except:
                in_error += 1
                pass

        HTML += ("\n\nErrors: " + str(in_error) + "\n\nAWS Status: <a href='http://status.aws.amazon.com/'>status.aws.amazon.com</a></pre></center></body></html>")

        plugin = JBPluginCloud.jbox_get_plugin(JBPluginCloud.JBP_BUCKETSTORE)
        bkt = JBoxCfg.get("cloud_host.status_bucket")
        if plugin is not None and bkt is not None:
            try:
                f = open("/tmp/index.html", "w")
                f.write(HTML)
                f.close()
                plugin.push(bkt, "/tmp/index.html")
            finally:
                os.remove("/tmp/index.html")
        else:
            JBox.log_debug("Status: %s", HTML)

        return None
コード例 #11
0
ファイル: srvr_jbox.py プロジェクト: JuliaLang/JuliaBox
    def monitor_registrations():
        max_rate = JBoxDynConfig.get_registration_hourly_rate(Compute.get_install_id())
        rate = JBoxUserV2.count_created(1)
        reg_allowed = JBoxDynConfig.get_allow_registration(Compute.get_install_id())
        JBox.log_debug("registration allowed: %r, rate: %d, max allowed: %d", reg_allowed, rate, max_rate)

        if (reg_allowed and (rate > max_rate*1.1)) or ((not reg_allowed) and (rate < max_rate*0.9)):
            reg_allowed = not reg_allowed
            JBox.log_warn("Changing registration allowed to %r", reg_allowed)
            JBoxDynConfig.set_allow_registration(Compute.get_install_id(), reg_allowed)

        if reg_allowed:
            num_pending_activations = JBoxUserV2.count_pending_activations()
            if num_pending_activations > 0:
                JBox.log_info("scheduling activations for %d pending activations", num_pending_activations)
                JBoxAsyncJob.async_schedule_activations()
コード例 #12
0
    def __init__(self):
        dckr = docker.Client()
        cfg = read_config()
        cloud_cfg = cfg['cloud_host']
        user_activation_cfg = cfg['user_activation']

        LoggerMixin.setup_logger(level=cfg['root_log_level'])
        LoggerMixin.DEFAULT_LEVEL = cfg['jbox_log_level']

        db.configure_db(cfg)

        CloudHost.configure(has_s3=cloud_cfg['s3'],
                            has_dynamodb=cloud_cfg['dynamodb'],
                            has_cloudwatch=cloud_cfg['cloudwatch'],
                            has_autoscale=cloud_cfg['autoscale'],
                            has_route53=cloud_cfg['route53'],
                            has_ebs=cloud_cfg['ebs'],
                            has_ses=cloud_cfg['ses'],
                            scale_up_at_load=cloud_cfg['scale_up_at_load'],
                            scale_up_policy=cloud_cfg['scale_up_policy'],
                            autoscale_group=cloud_cfg['autoscale_group'],
                            route53_domain=cloud_cfg['route53_domain'],
                            region=cloud_cfg['region'],
                            install_id=cloud_cfg['install_id'])
        VolMgr.configure(dckr, cfg)
        JBoxAsyncJob.configure(cfg)
        JBoxContainer.configure(dckr,
                                cfg['docker_image'],
                                cfg['mem_limit'],
                                cfg['cpu_limit'],
                                cfg['numlocalmax'],
                                cfg['async_job_ports'],
                                async_mode=JBoxAsyncJob.MODE_SUB)
        self.log_debug("Backup daemon listening on ports: %s",
                       repr(cfg['async_job_ports']))
        JBoxd.QUEUE = JBoxContainer.ASYNC_JOB

        JBoxd.MAX_ACTIVATIONS_PER_SEC = user_activation_cfg[
            'max_activations_per_sec']
        JBoxd.MAX_AUTO_ACTIVATIONS_PER_RUN = user_activation_cfg[
            'max_activations_per_run']
        JBoxd.ACTIVATION_SUBJECT = user_activation_cfg['mail_subject']
        JBoxd.ACTIVATION_BODY = user_activation_cfg['mail_body']
        JBoxd.ACTIVATION_SENDER = user_activation_cfg['sender']
コード例 #13
0
ファイル: srvr_jboxd.py プロジェクト: Emram/JuliaBox
    def __init__(self):
        LoggerMixin.configure()
        db.configure()
        Compute.configure()
        SessContainer.configure()
        APIContainer.configure()
        VolMgr.configure()

        JBoxAsyncJob.configure()
        JBoxAsyncJob.init(JBoxAsyncJob.MODE_SUB)

        self.log_debug("Container manager listening on ports: %s", repr(JBoxCfg.get('container_manager_ports')))
        JBoxd.QUEUE = JBoxAsyncJob.get()

        JBoxd.MAX_ACTIVATIONS_PER_SEC = JBoxCfg.get('user_activation.max_activations_per_sec')
        JBoxd.MAX_AUTO_ACTIVATIONS_PER_RUN = JBoxCfg.get('user_activation.max_activations_per_run')
        JBoxd.ACTIVATION_SUBJECT = JBoxCfg.get('user_activation.mail_subject')
        JBoxd.ACTIVATION_BODY = JBoxCfg.get('user_activation.mail_body')
        JBoxd.ACTIVATION_SENDER = JBoxCfg.get('user_activation.sender')
コード例 #14
0
    def __init__(self):
        LoggerMixin.configure()
        db.configure()
        Compute.configure()
        SessContainer.configure()
        VolMgr.configure()

        JBoxAsyncJob.configure()
        JBoxAsyncJob.init(JBoxAsyncJob.MODE_PUB)

        self.application = tornado.web.Application(
            handlers=[(r"/", MainHandler), (
                r"/jboxadmin/",
                AdminHandler), (r"/jboxping/",
                                PingHandler), (r"/jboxcors/", CorsHandler)])
        JBPluginHandler.add_plugin_handlers(self.application)
        JBPluginUI.create_include_files()

        # cookie_secret = ''.join(random.choice(string.ascii_uppercase + string.digits) for x in xrange(32))
        # use sesskey as cookie secret to be able to span multiple tornado servers
        self.application.settings["cookie_secret"] = JBoxCfg.get('sesskey')
        self.application.settings[
            "plugin_features"] = JBox.get_pluggedin_features()
        self.application.listen(JBoxCfg.get('interactive.manager_port'),
                                address=socket.gethostname())
        self.application.listen(JBoxCfg.get('interactive.manager_port'),
                                address='localhost')

        self.ioloop = tornado.ioloop.IOLoop.instance()

        # run container maintainence every 5 minutes
        run_interval = 5 * 60 * 1000
        self.log_info("Container maintenance every " +
                      str(run_interval / (60 * 1000)) + " minutes")
        self.ct = tornado.ioloop.PeriodicCallback(JBox.do_housekeeping,
                                                  run_interval, self.ioloop)
        self.sigct = tornado.ioloop.PeriodicCallback(JBox.do_signals, 1000,
                                                     self.ioloop)

        # or configure cacerts
        AsyncHTTPClient.configure(None, defaults=dict(validate_cert=None))
コード例 #15
0
 def configure(dckr,
               image,
               mem_limit,
               cpu_limit,
               max_containers,
               async_job_ports,
               async_mode=JBoxAsyncJob.MODE_PUB):
     JBoxContainer.DCKR = dckr
     JBoxContainer.DCKR_IMAGE = image
     JBoxContainer.MEM_LIMIT = mem_limit
     JBoxContainer.CPU_LIMIT = cpu_limit
     JBoxContainer.MAX_CONTAINERS = max_containers
     JBoxContainer.ASYNC_JOB = JBoxAsyncJob(async_job_ports, async_mode)
コード例 #16
0
ファイル: srvr_jbapi.py プロジェクト: JuliaLang/JuliaBox
    def do_housekeeping():
        is_leader = is_cluster_leader()
        if is_leader:
            JBoxAPI.log_info("I am the cluster leader")
            terminating = False
        else:
            try:
                terminating = JBoxAsyncJob.sync_is_terminating()
                if terminating['code'] == 0:
                    terminating = terminating['data']
                else:
                    JBoxAPI.log_error("Error checking if instance is terminating. Assuming False.")
                    terminating = False
            except:
                JBoxAPI.log_error("Exception checking if instance is terminating. Assuming False.")
                terminating = False

        if terminating:
            JBoxAPI.log_warn("terminating to scale down")
        else:
            APIContainer.maintain()
            JBoxAsyncJob.async_plugin_maintenance(is_leader)
コード例 #17
0
    def do_housekeeping():
        terminating = False
        server_delete_timeout = JBoxCfg.get('interactive.expire')
        inactive_timeout = JBoxCfg.get('interactive.inactivity_timeout')
        SessContainer.maintain(max_timeout=server_delete_timeout,
                               inactive_timeout=inactive_timeout)
        is_leader = is_cluster_leader()

        if is_leader:
            terminating = False
        else:
            try:
                terminating = JBoxAsyncJob.sync_is_terminating()
                if terminating['code'] == 0:
                    terminating = terminating['data']
                else:
                    JBox.log_error(
                        "Error checking if instance is terminating. Assuming False."
                    )
                    terminating = False
            except:
                JBox.log_error(
                    "Exception checking if instance is terminating. Assuming False."
                )
                terminating = False

        if is_leader:
            JBox.log_info("I am the cluster leader")
            JBox.update_juliabox_status()
            JBox.monitor_registrations()
            if not JBoxDynConfig.is_stat_collected_within(
                    Compute.get_install_id(), 1):
                JBoxAsyncJob.async_collect_stats()

        if terminating:
            JBox.log_warn("terminating to scale down")
        else:
            JBox.do_update_user_home_image()
            JBoxAsyncJob.async_plugin_maintenance(is_leader)
コード例 #18
0
    def monitor_registrations():
        max_rate = JBoxDynConfig.get_registration_hourly_rate(
            Compute.get_install_id())
        rate = JBoxUserV2.count_created(1)
        reg_allowed = JBoxDynConfig.get_allow_registration(
            Compute.get_install_id())
        JBox.log_debug("registration allowed: %r, rate: %d, max allowed: %d",
                       reg_allowed, rate, max_rate)

        if (reg_allowed and
            (rate > max_rate * 1.1)) or ((not reg_allowed) and
                                         (rate < max_rate * 0.9)):
            reg_allowed = not reg_allowed
            JBox.log_warn("Changing registration allowed to %r", reg_allowed)
            JBoxDynConfig.set_allow_registration(Compute.get_install_id(),
                                                 reg_allowed)

        if reg_allowed:
            num_pending_activations = JBoxUserV2.count_pending_activations()
            if num_pending_activations > 0:
                JBox.log_info(
                    "scheduling activations for %d pending activations",
                    num_pending_activations)
                JBoxAsyncJob.async_schedule_activations()
コード例 #19
0
 def do_update_user_home_image():
     if VolMgr.has_update_for_user_home_image():
         if not VolMgr.update_user_home_image(fetch=False):
             JBoxAsyncJob.async_update_user_home_image()
コード例 #20
0
ファイル: srvr_jbox.py プロジェクト: JuliaLang/JuliaBox
 def do_update_user_home_image():
     if VolMgr.has_update_for_user_home_image():
         if not VolMgr.update_user_home_image(fetch=False):
             JBoxAsyncJob.async_update_user_home_image()