Example #1
0
    def do_housekeeping():
        terminating = False
        server_delete_timeout = JBoxCfg.get('interactive.expire')
        inactive_timeout = JBoxCfg.get('interactive.inactivity_timeout')
        SessContainer.maintain(max_timeout=server_delete_timeout, inactive_timeout=inactive_timeout)
        is_leader = is_cluster_leader()

        if is_leader:
            terminating = False
        else:
            try:
                terminating = JBoxAsyncJob.sync_is_terminating()
                if terminating['code'] == 0:
                    terminating = terminating['data']
                else:
                    JBox.log_error("Error checking if instance is terminating. Assuming False.")
                    terminating = False
            except:
                JBox.log_error("Exception checking if instance is terminating. Assuming False.")
                terminating = False

        if is_leader:
            JBox.log_info("I am the cluster leader")
            JBox.update_juliabox_status()
            JBox.monitor_registrations()
            if not JBoxDynConfig.is_stat_collected_within(Compute.get_install_id(), 1):
                JBoxAsyncJob.async_collect_stats()

        if terminating:
            JBox.log_warn("terminating to scale down")
        else:
            JBox.do_update_user_home_image()
            JBoxAsyncJob.async_plugin_maintenance(is_leader)
Example #2
0
    def __init__(self):
        LoggerMixin.configure()
        db.configure()
        Compute.configure()
        SessContainer.configure()
        VolMgr.configure()

        JBoxAsyncJob.configure()
        JBoxAsyncJob.init(JBoxAsyncJob.MODE_PUB)

        self.application = tornado.web.Application(handlers=[
            (r"/", MainHandler),
            (r"/jboxadmin/", AdminHandler),
            (r"/jboxping/", PingHandler),
            (r"/jboxcors/", CorsHandler)
        ])
        JBPluginHandler.add_plugin_handlers(self.application)
        JBPluginUI.create_include_files()

        # cookie_secret = ''.join(random.choice(string.ascii_uppercase + string.digits) for x in xrange(32))
        # use sesskey as cookie secret to be able to span multiple tornado servers
        self.application.settings["cookie_secret"] = JBoxCfg.get('sesskey')
        self.application.settings["plugin_features"] = JBox.get_pluggedin_features()
        self.application.listen(JBoxCfg.get('interactive.manager_port'), address=socket.gethostname())
        self.application.listen(JBoxCfg.get('interactive.manager_port'), address='localhost')

        self.ioloop = tornado.ioloop.IOLoop.instance()

        # run container maintainence every 5 minutes
        run_interval = 5 * 60 * 1000
        self.log_info("Container maintenance every " + str(run_interval / (60 * 1000)) + " minutes")
        self.ct = tornado.ioloop.PeriodicCallback(JBox.do_housekeeping, run_interval, self.ioloop)
        self.sigct = tornado.ioloop.PeriodicCallback(JBox.do_signals, 1000, self.ioloop)
Example #3
0
 def publish_sessions():
     iid = Compute.get_instance_id()
     for c in SessContainer.session_containers(allcontainers=True):
         if ('Names' in c) and (c['Names'] is not None):
             sessname = SessContainer(c['Id']).get_name()
             if sessname:
                 JBoxSessionProps.attach_instance(Compute.get_install_id(),
                                                  sessname, iid, c["Status"])
Example #4
0
 def handle_if_logout(self, cont):
     logout = self.get_argument('logout', False)
     if logout == 'me':
         SessContainer.invalidate_container(cont.get_name())
         JBoxAsyncJob.async_backup_and_cleanup(cont.dockid)
         response = {'code': 0, 'data': ''}
         self.write(response)
         return True
     return False
Example #5
0
 def get(self):
     valid_req = self.is_valid_req()
     sessname = self.get_session_id(validate=False)
     if valid_req:
         SessContainer.record_ping("/" + sessname)
         self.set_status(status_code=204)
         self.finish()
     else:
         self.log_warn("Invalid ping request for " + sessname)
         self.send_error(status_code=403)
Example #6
0
 def publish_sessions():
     iid = Compute.get_instance_id()
     for c in SessContainer.session_containers(allcontainers=True):
         if ("Names" in c) and (c["Names"] is not None):
             sessname = SessContainer(c["Id"]).get_name()
             if sessname:
                 JBoxSessionProps.attach_instance(Compute.get_install_id(), sessname, iid, c["Status"])
Example #7
0
    def post(self):
        self.log_debug("Homework handler got POST request")
        sessname = self.get_session_id()
        user_id = self.get_user_id()
        if (sessname is None) or (user_id is None):
            self.log_info("Homework handler got invalid sessname[%r] or user_id[%r]", sessname, user_id)
            self.send_error()
            return

        user = JBoxUserV2(user_id)
        is_admin = sessname in JBoxCfg.get("admin_sessnames", []) or user.has_role(JBoxUserV2.ROLE_SUPER)
        course_owner = is_admin or user.has_role(JBoxUserV2.ROLE_OFFER_COURSES)
        cont = SessContainer.get_by_name(sessname)
        self.log_info("user_id[%r], is_admin[%r], course_owner[%r]", user_id, is_admin, course_owner)

        if cont is None:
            self.log_info("user_id[%r] container not found", user_id)
            self.send_error()
            return

        courses_offered = user.get_courses_offered()

        if self.handle_if_check(user_id):
            return
        if self.handle_create_course(user_id):
            return
        if self.handle_get_metadata(is_admin, courses_offered):
            return
        if self.handle_if_report(user_id, is_admin, courses_offered):
            return

        self.log_error("no handlers found")
        # only AJAX requests responded to
        self.send_error()
Example #8
0
    def refresh_disk_use_status(container_id_list=None):
        JBoxLoopbackVol.LOCK.acquire()
        try:
            nfree = 0
            for idx in range(0, JBoxLoopbackVol.MAX_DISKS):
                if JBoxLoopbackVol._is_reserved(idx):
                    JBoxLoopbackVol.DISK_USE_STATUS[idx] = True
                else:
                    JBoxLoopbackVol.DISK_USE_STATUS[idx] = False
                    nfree += 1

            if container_id_list is None:
                container_id_list = [
                    cdesc['Id'] for cdesc in SessContainer.session_containers(
                        allcontainers=True)
                ]

            for cid in container_id_list:
                disk_ids = JBoxLoopbackVol._get_disk_ids_used(cid)
                for disk_id in disk_ids:
                    JBoxLoopbackVol._mark_disk_used(disk_id)
                    nfree -= 1
            JBoxLoopbackVol.log_info("Loopback Disk free: " + str(nfree) +
                                     "/" + str(JBoxLoopbackVol.MAX_DISKS))
        finally:
            JBoxLoopbackVol.LOCK.release()
Example #9
0
 def backup_and_cleanup(dockid):
     cont = SessContainer(dockid)
     cont.stop()
     cont.delete(backup=True)
     JBoxSessionProps.detach_instance(Compute.get_install_id(), cont.get_name(), Compute.get_instance_id())
     JBoxd.publish_perf_counters()
     JBoxd.publish_anticipated_load()
Example #10
0
    def try_launch_container(cls, user_id, max_hop=False):
        sessname = unique_sessname(user_id)
        cont = SessContainer.get_by_name(sessname)
        cls.log_debug("have existing container for %s: %r", sessname,
                      None != cont)
        if cont is not None:
            cls.log_debug("container running: %r", cont.is_running())

        if max_hop:
            self_load = Compute.get_instance_stats(Compute.get_instance_id(),
                                                   'Load')
            if self_load < 100:
                SessContainer.invalidate_container(sessname)
                JBoxAsyncJob.async_launch_by_name(sessname, user_id, True)
                return True

        is_leader = is_proposed_cluster_leader()
        if ((cont is None) or (not cont.is_running())) and (
                not Compute.should_accept_session(is_leader)):
            if cont is not None:
                SessContainer.invalidate_container(cont.get_name())
                JBoxAsyncJob.async_backup_and_cleanup(cont.dockid)
            return False

        SessContainer.invalidate_container(sessname)
        JBoxAsyncJob.async_launch_by_name(sessname, user_id, True)
        return True
Example #11
0
    def do_task(plugin_type, data):
        if plugin_type != JBPluginTask.JBP_CMD_ASYNC:
            return
        mode = data['action']
        user_id = data['user_id']
        sessname = data['sessname']

        user = JBoxUserV2(user_id)
        is_allowed = user.has_resource_profile(
            JBoxUserV2.RES_PROF_DISK_EBS_10G)
        if not is_allowed:
            JBoxEBSVolAsyncTask.log_error(
                "Data volume access not allowed for user")
            return

        cont = SessContainer.get_by_name(sessname)
        if cont is None:
            return

        vol = JBoxEBSVol.get_disk_from_container(sessname)
        disk_state = None
        try:
            disk_state = JBoxDiskState(cluster_id=CompEC2.INSTALL_ID,
                                       region_id=CompEC2.REGION,
                                       user_id=user_id)
        except:
            pass

        JBoxEBSVolAsyncTask.log_debug("Data volume request %s for %s", mode,
                                      cont.debug_str())

        if mode == 'attach':
            if vol is None:
                vol = JBoxEBSVol.get_disk_for_user(user_id)
                JBoxEBSVol.mount_host_device(vol.disk_path, cont.dockid,
                                             JBoxVol.DATA_MOUNT_POINT)
                disk_state = JBoxDiskState(cluster_id=CompEC2.INSTALL_ID,
                                           region_id=CompEC2.REGION,
                                           user_id=user_id)
                if disk_state.get_state() != JBoxDiskState.STATE_ATTACHED:
                    disk_state.set_state(JBoxDiskState.STATE_ATTACHED)
                    disk_state.save()
        elif mode == 'detach':
            if cont is not None and cont.is_running():
                if vol is not None:
                    # unmount from container first
                    JBoxEBSVol.unmount_host_device(vol.disk_path, cont.dockid)
                elif disk_state is not None:
                    # no volume attached. ensure disk state is updated
                    if disk_state.get_state() != JBoxDiskState.STATE_DETACHED:
                        disk_state.set_state(JBoxDiskState.STATE_DETACHED)
                        disk_state.save()
            if vol is not None:
                vol.release(backup=True)

        JBoxEBSVolAsyncTask.log_debug(
            "Data volume request %s completed for %s", mode, cont.debug_str())
Example #12
0
    def __init__(self):
        LoggerMixin.configure()
        db.configure()
        Compute.configure()
        SessContainer.configure()
        APIContainer.configure()
        VolMgr.configure()

        JBoxAsyncJob.configure()
        JBoxAsyncJob.init(JBoxAsyncJob.MODE_SUB)

        self.log_debug("Container manager listening on ports: %s", repr(JBoxCfg.get('container_manager_ports')))
        JBoxd.QUEUE = JBoxAsyncJob.get()

        JBoxd.MAX_ACTIVATIONS_PER_SEC = JBoxCfg.get('user_activation.max_activations_per_sec')
        JBoxd.MAX_AUTO_ACTIVATIONS_PER_RUN = JBoxCfg.get('user_activation.max_activations_per_run')
        JBoxd.ACTIVATION_SUBJECT = JBoxCfg.get('user_activation.mail_subject')
        JBoxd.ACTIVATION_BODY = JBoxCfg.get('user_activation.mail_body')
        JBoxd.ACTIVATION_SENDER = JBoxCfg.get('user_activation.sender')
Example #13
0
    def get(self):
        sessname = self.get_session_id()
        user_id = self.get_user_id()
        if (sessname is None) or (user_id is None):
            self.send_error()
            return

        user = JBoxUserV2(user_id)
        is_admin = sessname in JBoxCfg.get("admin_sessnames", [])
        manage_containers = is_admin or user.has_role(JBoxUserV2.ROLE_MANAGE_CONTAINERS)
        show_report = is_admin or user.has_role(JBoxUserV2.ROLE_ACCESS_STATS)
        cont = SessContainer.get_by_name(sessname)

        if cont is None:
            self.send_error()
            return

        if self.handle_if_logout(cont):
            return
        if self.handle_if_stats(is_admin or show_report):
            return
        if self.handle_if_show_cfg(is_admin):
            return
        if self.handle_if_instance_info(is_admin):
            return
        if self.handle_switch_julia_img(user):
            return
        if self.handle_if_open_port(sessname, user_id):
            return

        juliaboxver, _upgrade_available = self.get_upgrade_available(cont)

        jimg_type = 0
        if user.has_resource_profile(JBoxUserV2.RES_PROF_JULIA_PKG_PRECOMP):
            jimg_type = JBoxUserV2.RES_PROF_JULIA_PKG_PRECOMP

        expire = JBoxCfg.get('interactive.expire')
        d = dict(
            manage_containers=manage_containers,
            show_report=show_report,
            sessname=sessname,
            user_id=user_id,
            created=isodate.datetime_isoformat(cont.time_created()),
            started=isodate.datetime_isoformat(cont.time_started()),
            allowed_till=isodate.datetime_isoformat((cont.time_started() + timedelta(seconds=expire))),
            mem=cont.get_memory_allocated(),
            cpu=cont.get_cpu_allocated(),
            disk=cont.get_disk_allocated(),
            expire=expire,
            juliaboxver=juliaboxver,
            jimg_type=jimg_type
        )

        self.rendertpl("ipnbadmin.tpl", d=d)
Example #14
0
    def __init__(self):
        LoggerMixin.configure()
        db.configure()
        Compute.configure()
        SessContainer.configure()
        VolMgr.configure()

        JBoxAsyncJob.configure()
        JBoxAsyncJob.init(JBoxAsyncJob.MODE_PUB)

        self.application = tornado.web.Application(
            handlers=[(r"/", MainHandler), (
                r"/jboxadmin/",
                AdminHandler), (r"/jboxping/",
                                PingHandler), (r"/jboxcors/", CorsHandler)])
        JBPluginHandler.add_plugin_handlers(self.application)
        JBPluginUI.create_include_files()

        # cookie_secret = ''.join(random.choice(string.ascii_uppercase + string.digits) for x in xrange(32))
        # use sesskey as cookie secret to be able to span multiple tornado servers
        self.application.settings["cookie_secret"] = JBoxCfg.get('sesskey')
        self.application.settings[
            "plugin_features"] = JBox.get_pluggedin_features()
        self.application.listen(JBoxCfg.get('interactive.manager_port'),
                                address=socket.gethostname())
        self.application.listen(JBoxCfg.get('interactive.manager_port'),
                                address='localhost')

        self.ioloop = tornado.ioloop.IOLoop.instance()

        # run container maintainence every 5 minutes
        run_interval = 5 * 60 * 1000
        self.log_info("Container maintenance every " +
                      str(run_interval / (60 * 1000)) + " minutes")
        self.ct = tornado.ioloop.PeriodicCallback(JBox.do_housekeeping,
                                                  run_interval, self.ioloop)
        self.sigct = tornado.ioloop.PeriodicCallback(JBox.do_signals, 1000,
                                                     self.ioloop)

        # or configure cacerts
        AsyncHTTPClient.configure(None, defaults=dict(validate_cert=None))
Example #15
0
 def do_periodic_task(_mode):
     active_clusters = UserCluster.list_all_groupids()
     ParallelHousekeep.log_info("%d active clusters", len(active_clusters))
     if len(active_clusters) == 0:
         return
     active_sessions = SessContainer.get_active_sessions()
     for cluster_id in active_clusters:
         sess_id = "/" + UserCluster.sessname_for_cluster(cluster_id)
         if sess_id not in active_sessions:
             ParallelHousekeep.log_info("Session (%s) corresponding to cluster (%s) not found. Terminating cluster.",
                                        sess_id, cluster_id)
             ParallelHousekeep.terminate_or_delete_cluster(cluster_id)
Example #16
0
 def backup_and_cleanup(dockid):
     cont = SessContainer(dockid)
     cont.stop()
     cont.delete(backup=True)
     JBoxSessionProps.detach_instance(Compute.get_install_id(), cont.get_name(), Compute.get_instance_id())
     JBoxd.publish_perf_counters()
     JBoxd.publish_anticipated_load()
Example #17
0
    def do_housekeeping():
        terminating = False
        server_delete_timeout = JBoxCfg.get('interactive.expire')
        inactive_timeout = JBoxCfg.get('interactive.inactivity_timeout')
        SessContainer.maintain(max_timeout=server_delete_timeout,
                               inactive_timeout=inactive_timeout)
        is_leader = is_cluster_leader()

        if is_leader:
            terminating = False
        else:
            try:
                terminating = JBoxAsyncJob.sync_is_terminating()
                if terminating['code'] == 0:
                    terminating = terminating['data']
                else:
                    JBox.log_error(
                        "Error checking if instance is terminating. Assuming False."
                    )
                    terminating = False
            except:
                JBox.log_error(
                    "Exception checking if instance is terminating. Assuming False."
                )
                terminating = False

        if is_leader:
            JBox.log_info("I am the cluster leader")
            JBox.update_juliabox_status()
            JBox.monitor_registrations()
            if not JBoxDynConfig.is_stat_collected_within(
                    Compute.get_install_id(), 1):
                JBoxAsyncJob.async_collect_stats()

        if terminating:
            JBox.log_warn("terminating to scale down")
        else:
            JBox.do_update_user_home_image()
            JBoxAsyncJob.async_plugin_maintenance(is_leader)
Example #18
0
 def do_periodic_task(_mode):
     active_clusters = UserCluster.list_all_groupids()
     ParallelHousekeep.log_info("%d active clusters", len(active_clusters))
     if len(active_clusters) == 0:
         return
     active_sessions = SessContainer.get_active_sessions()
     for cluster_id in active_clusters:
         sess_id = "/" + UserCluster.sessname_for_cluster(cluster_id)
         if sess_id not in active_sessions:
             ParallelHousekeep.log_info(
                 "Session (%s) corresponding to cluster (%s) not found. Terminating cluster.",
                 sess_id, cluster_id)
             ParallelHousekeep.terminate_or_delete_cluster(cluster_id)
Example #19
0
    def is_valid_req(self):
        sessname = self.get_session_id()
        if sessname is None:
            return False

        ports = self.get_ports()
        container_ports = (ports[JBoxCookies.COOKIE_PORT_SHELL],
                           ports[JBoxCookies.COOKIE_PORT_UPL],
                           ports[JBoxCookies.COOKIE_PORT_IPNB])
        if not SessContainer.is_valid_container("/" + sessname, container_ports):
            self.log_info('not valid req. container deleted or ports not matching')
            return False

        return True
Example #20
0
    def refresh_disk_use_status(container_id_list=None):
        JBoxDefaultPackagesVol.LOCK.acquire()
        bundles = set()
        try:
            if container_id_list is None:
                container_id_list = [cdesc['Id'] for cdesc in SessContainer.session_containers(allcontainers=True)]

            for cid in container_id_list:
                mount_points = JBoxDefaultPackagesVol._get_package_mounts_used(cid)
                bundles.update(mount_points)

            JBoxDefaultPackagesVol.BUNDLES_IN_USE = bundles
            JBoxDefaultPackagesVol.log_info("Packages in use: %r", bundles)
        finally:
            JBoxDefaultPackagesVol.LOCK.release()
Example #21
0
    def refresh_disk_use_status(container_id_list=None):
        JBoxDefaultPackagesVol.LOCK.acquire()
        bundles = set()
        try:
            if container_id_list is None:
                container_id_list = [cdesc['Id'] for cdesc in SessContainer.session_containers(allcontainers=True)]

            for cid in container_id_list:
                mount_points = JBoxDefaultPackagesVol._get_package_mounts_used(cid)
                bundles.update(mount_points)

            JBoxDefaultPackagesVol.BUNDLES_IN_USE = bundles
            JBoxDefaultPackagesVol.log_info("Packages in use: %r", bundles)
        finally:
            JBoxDefaultPackagesVol.LOCK.release()
Example #22
0
    def do_task(plugin_type, data):
        if plugin_type != JBPluginTask.JBP_CMD_ASYNC:
            return
        mode = data["action"]
        user_id = data["user_id"]
        sessname = data["sessname"]

        user = JBoxUserV2(user_id)
        is_allowed = user.has_resource_profile(JBoxUserV2.RES_PROF_DISK_EBS_10G)
        if not is_allowed:
            JBoxEBSVolAsyncTask.log_error("Data volume access not allowed for user")
            return

        cont = SessContainer.get_by_name(sessname)
        if cont is None:
            return

        vol = JBoxEBSVol.get_disk_from_container(sessname)
        disk_state = None
        try:
            disk_state = JBoxDiskState(cluster_id=CompEC2.INSTALL_ID, region_id=CompEC2.REGION, user_id=user_id)
        except:
            pass

        JBoxEBSVolAsyncTask.log_debug("Data volume request %s for %s", mode, cont.debug_str())

        if mode == "attach":
            if vol is None:
                vol = JBoxEBSVol.get_disk_for_user(user_id)
                JBoxEBSVol.mount_host_device(vol.disk_path, cont.dockid, JBoxVol.DATA_MOUNT_POINT)
                disk_state = JBoxDiskState(cluster_id=CompEC2.INSTALL_ID, region_id=CompEC2.REGION, user_id=user_id)
                if disk_state.get_state() != JBoxDiskState.STATE_ATTACHED:
                    disk_state.set_state(JBoxDiskState.STATE_ATTACHED)
                    disk_state.save()
        elif mode == "detach":
            if cont is not None and cont.is_running():
                if vol is not None:
                    # unmount from container first
                    JBoxEBSVol.unmount_host_device(vol.disk_path, cont.dockid)
                elif disk_state is not None:
                    # no volume attached. ensure disk state is updated
                    if disk_state.get_state() != JBoxDiskState.STATE_DETACHED:
                        disk_state.set_state(JBoxDiskState.STATE_DETACHED)
                        disk_state.save()
            if vol is not None:
                vol.release(backup=True)

        JBoxEBSVolAsyncTask.log_debug("Data volume request %s completed for %s", mode, cont.debug_str())
Example #23
0
    def do_monitor_loading_ajax(self, user_id):
        sessname = unique_sessname(user_id)
        self.log_debug("AJAX monitoring loading of session [%s] user[%s]...", sessname, user_id)
        cont = SessContainer.get_by_name(sessname)
        if (cont is None) or (not cont.is_running()):
            loading_step = int(self.get_loading_state(), 0)
            if loading_step > 90:
                self.log_error("Could not start instance. Session [%s] for user [%s] didn't load.", sessname, user_id)
                self.write({'code': -1})
                return

            loading_step += 1
            self.set_loading_state(loading_step)
            self.write({'code': 0})
        else:
            self.write({'code': 1})
Example #24
0
    def do_monitor_loading_ajax(self, user_id):
        sessname = unique_sessname(user_id)
        self.log_debug("AJAX monitoring loading of session [%s] user[%s]...", sessname, user_id)
        cont = SessContainer.get_by_name(sessname)
        if (cont is None) or (not cont.is_running()):
            loading_step = int(self.get_loading_state(), 0)
            if loading_step > 60:
                self.log_error("Could not start instance. Session [%s] for user [%s] didn't load.", sessname, user_id)
                self.write({'code': -1})
                return

            loading_step += 1
            self.set_loading_state(loading_step)
            self.write({'code': 0})
        else:
            self.write({'code': 1})
Example #25
0
    def is_valid_req(self):
        sessname = self.get_session_id()
        if sessname is None:
            return False

        ports = self.get_ports()
        container_ports = (ports[JBoxCookies.COOKIE_PORT_SHELL],
                           ports[JBoxCookies.COOKIE_PORT_UPL],
                           ports[JBoxCookies.COOKIE_PORT_IPNB])
        if not SessContainer.is_valid_container("/" + sessname,
                                                container_ports):
            self.log_info(
                'not valid req. container deleted or ports not matching')
            return False

        return True
Example #26
0
 def _launch_session(name, email, reuse):
     cont = SessContainer.launch_by_name(name, email, reuse=reuse)
     JBoxd.publish_perf_counters()
     if not JBoxd._wait_for_container_start(cont):
         JBoxd.log_error("did not start: %s", cont.debug_str())
         BaseContainer.DCKR.kill(cont.dockid)
         return False
     # wait for services to start
     for port in cont.get_host_ports():
         if not JBoxd._wait_for_port(port):
             JBoxd.log_error("port %s did not start: %s", port, cont.debug_str())
             BaseContainer.DCKR.kill(cont.dockid)
             return False
         else:
             JBoxd.log_debug("port %s active: %s", port, cont.debug_str())
     JBoxd.log_info("passed connectivity check: %s", cont.debug_str())
     return True
Example #27
0
 def _launch_session(name, email, reuse):
     cont = SessContainer.launch_by_name(name, email, reuse=reuse)
     JBoxd.publish_perf_counters()
     if not JBoxd._wait_for_container_start(cont):
         JBoxd.log_error("did not start: %s", cont.debug_str())
         BaseContainer.DCKR.kill(cont.dockid)
         return False
     # wait for services to start
     for port in cont.get_host_ports():
         if not JBoxd._wait_for_port(port):
             JBoxd.log_error("port %s did not start: %s", port, cont.debug_str())
             BaseContainer.DCKR.kill(cont.dockid)
             return False
         else:
             JBoxd.log_debug("port %s active: %s", port, cont.debug_str())
     JBoxd.log_info("passed connectivity check: %s", cont.debug_str())
     return True
Example #28
0
    def do_monitor_loading(self, user_id):
        sessname = unique_sessname(user_id)
        self.log_debug("Monitoring loading of session [%s] user[%s]...",
                       sessname, user_id)
        cont = SessContainer.get_by_name(sessname)
        if (cont is None) or (not cont.is_running()):
            loading_step = int(self.get_loading_state(), 0)
            if loading_step > 30:
                self.log_error(
                    "Could not start instance. Session [%s] for user [%s] didn't load.",
                    sessname, user_id)
                self.clear_container()
                self.rendertpl(
                    "index.tpl",
                    cfg=JBoxCfg.nv,
                    state=self.state(
                        error=
                        'Could not start your instance! Please try again.',
                        pending_activation=False,
                        user_id=user_id))
                return
            else:
                loading_step += 1

            self.set_loading_state(loading_step)
            self.rendertpl("loading.tpl",
                           user_id=user_id,
                           cfg=JBoxCfg.nv,
                           js_includes=JBPluginHandler.PLUGIN_JAVASCRIPTS)
        else:
            (shellport, uplport, ipnbport) = cont.get_host_ports()

            self.set_container_ports({
                JBoxHandler.COOKIE_PORT_SHELL: shellport,
                JBoxHandler.COOKIE_PORT_UPL: uplport,
                JBoxHandler.COOKIE_PORT_IPNB: ipnbport
            })
            self.clear_loading()

            self.rendertpl("ipnbsess.tpl",
                           sessname=sessname,
                           cfg=JBoxCfg.nv,
                           user_id=user_id,
                           plugin_features=json.dumps(
                               self.application.settings["plugin_features"]),
                           js_includes=JBPluginHandler.PLUGIN_JAVASCRIPTS)
Example #29
0
 def do_node_housekeeping():
     JBoxEBSHousekeep.log_debug("starting node housekeeping")
     for device, vol in JBoxEBSVol.get_mapped_volumes().iteritems():
         deviceid = os.path.basename(device)
         vol_id = vol.volume_id
         vol = EBSVol.get_volume(vol_id)
         user_id = vol.tags["Name"] if "Name" in vol.tags else None
         if user_id is None:
             continue
         sessname = unique_sessname(user_id)
         cont = SessContainer.get_by_name(sessname)
         if cont is not None:
             continue
         JBoxEBSHousekeep.log_debug("Found orphaned volume %s for %s, %s", vol_id, user_id, sessname)
         ebsvol = JBoxEBSVol(deviceid, sessname=sessname)
         ebsvol.release(backup=True)
     JBoxEBSHousekeep.log_debug("finished node housekeeping")
Example #30
0
 def do_node_housekeeping():
     JBoxEBSHousekeep.log_debug("starting node housekeeping")
     for device, vol in JBoxEBSVol.get_mapped_volumes().iteritems():
         deviceid = os.path.basename(device)
         vol_id = vol.volume_id
         vol = EBSVol.get_volume(vol_id)
         user_id = vol.tags['Name'] if 'Name' in vol.tags else None
         if user_id is None:
             continue
         sessname = unique_sessname(user_id)
         cont = SessContainer.get_by_name(sessname)
         if cont is not None:
             continue
         JBoxEBSHousekeep.log_debug("Found orphaned volume %s for %s, %s", vol_id, user_id, sessname)
         ebsvol = JBoxEBSVol(deviceid, sessname=sessname)
         ebsvol.release(backup=True)
     JBoxEBSHousekeep.log_debug("finished node housekeeping")
Example #31
0
    def refresh_disk_use_status(container_id_list=None):
        JBoxLoopbackVol.LOCK.acquire()
        try:
            nfree = 0
            for idx in range(0, JBoxLoopbackVol.MAX_DISKS):
                if JBoxLoopbackVol._is_reserved(idx):
                    JBoxLoopbackVol.DISK_USE_STATUS[idx] = True
                else:
                    JBoxLoopbackVol.DISK_USE_STATUS[idx] = False
                    nfree += 1

            if container_id_list is None:
                container_id_list = [cdesc['Id'] for cdesc in SessContainer.session_containers(allcontainers=True)]

            for cid in container_id_list:
                disk_ids = JBoxLoopbackVol._get_disk_ids_used(cid)
                for disk_id in disk_ids:
                    JBoxLoopbackVol._mark_disk_used(disk_id)
                    nfree -= 1
            JBoxLoopbackVol.log_info("Loopback Disk free: " + str(nfree) + "/" + str(JBoxLoopbackVol.MAX_DISKS))
        finally:
            JBoxLoopbackVol.LOCK.release()
Example #32
0
    def handle_if_open_port(self, sessname, user_id):
        port = self.get_argument('open_port', None)
        if port is None:
            return False

        portname = self.get_argument('port_name', "", strip=True)
        if re.match(r"^[a-zA-Z0-9]{1,20}$", portname) is None:
            response = {'code': -1, 'data': 'Port name must be alpha numeric only.'}
        elif portname in ['shell', 'nb', 'file']:
            response = {'code': -1, 'data': 'Port names "shell", "nb" and "file" are reserved for use by JuliaBox.'}
        else:
            port = int(port)
            if port < 8050 or port > 8052:
                response = {'code': -1, 'data': 'Only ports in the range 8050-8052 can be used.'}
            else:
                cont = SessContainer.get_by_name(sessname)
                hostport = cont._get_host_ports([port])[0]
                self.set_container_ports({
                    portname: hostport
                })
                response = {'code': 0, 'data': ''}
        self.write(response)
        return True
Example #33
0
    def is_valid_req(self):
        sessname = self.get_session_id()
        if sessname is None:
            return False

        ports = self.get_ports()
        isvalid = True
        if not ports or \
           any(not ports.has_key(k) for k in [JBoxCookies.COOKIE_PORT_SHELL,
                                              JBoxCookies.COOKIE_PORT_UPL,
                                              JBoxCookies.COOKIE_PORT_IPNB]):
            isvalid = False
        else:
            container_ports = (ports[JBoxCookies.COOKIE_PORT_SHELL],
                               ports[JBoxCookies.COOKIE_PORT_UPL],
                               ports[JBoxCookies.COOKIE_PORT_IPNB])
            if not SessContainer.is_valid_container("/" + sessname, container_ports):
                isvalid = False

        if not isvalid:
            self.log_info('Not valid request. Container deleted or ports not matching.')
            return False

        return True
Example #34
0
    def do_monitor_loading(self, user_id):
        sessname = unique_sessname(user_id)
        self.log_debug("Monitoring loading of session [%s] user[%s]...", sessname, user_id)
        cont = SessContainer.get_by_name(sessname)
        if (cont is None) or (not cont.is_running()):
            loading_step = int(self.get_loading_state(), 0)
            if loading_step > 30:
                self.log_error("Could not start instance. Session [%s] for user [%s] didn't load.", sessname, user_id)
                self.clear_container()
                self.rendertpl("index.tpl", cfg=JBoxCfg.nv,
                               state=self.state(
                                   error='Could not start your instance! Please try again.',
                                   pending_activation=False,
                                   user_id=user_id))
                return
            else:
                loading_step += 1

            self.set_loading_state(loading_step)
            self.rendertpl("loading.tpl",
                           user_id=user_id,
                           cfg=JBoxCfg.nv,
                           js_includes=JBPluginHandler.PLUGIN_JAVASCRIPTS)
        else:
            (shellport, uplport, ipnbport) = cont.get_host_ports()

            self.set_container_ports({
                JBoxHandler.COOKIE_PORT_SHELL: shellport,
                JBoxHandler.COOKIE_PORT_UPL: uplport,
                JBoxHandler.COOKIE_PORT_IPNB: ipnbport
            })
            self.clear_loading()

            self.rendertpl("ipnbsess.tpl",  sessname=sessname, cfg=JBoxCfg.nv, user_id=user_id,
                           plugin_features=json.dumps(self.application.settings["plugin_features"]),
                           js_includes=JBPluginHandler.PLUGIN_JAVASCRIPTS)
Example #35
0
    def try_launch_container(cls, user_id, max_hop=False):
        sessname = unique_sessname(user_id)
        cont = SessContainer.get_by_name(sessname)
        cls.log_debug("have existing container for %s: %r", sessname, None != cont)
        if cont is not None:
            cls.log_debug("container running: %r", cont.is_running())

        if max_hop:
            self_load = Compute.get_instance_stats(Compute.get_instance_id(), 'Load')
            if self_load < 100:
                SessContainer.invalidate_container(sessname)
                JBoxAsyncJob.async_launch_by_name(sessname, user_id, True)
                return True

        is_leader = is_proposed_cluster_leader()
        if ((cont is None) or (not cont.is_running())) and (not Compute.should_accept_session(is_leader)):
            if cont is not None:
                SessContainer.invalidate_container(cont.get_name())
                JBoxAsyncJob.async_backup_and_cleanup(cont.dockid)
            return False

        SessContainer.invalidate_container(sessname)
        JBoxAsyncJob.async_launch_by_name(sessname, user_id, True)
        return True
Example #36
0
 def publish_sessions():
     iid = Compute.get_instance_id()
     for c in SessContainer.session_containers(allcontainers=True):
         if ('Names' in c) and (c['Names'] is not None):
             JBoxSessionProps.attach_instance(Compute.get_install_id(), SessContainer(c['Id']).get_name(), iid,
                                              c["Status"])
Example #37
0
    def post(self):
        sessname = self.get_session_id()
        user_id = self.get_user_id()
        if (sessname is None) or (user_id is None):
            self.send_error()
            return

        mode = self.get_argument('cluster', False)
        if mode is False:
            ParallelHandler.log_error("Unknown mode for parallel handler")
            self.send_error()
            return

        user = JBoxUserV2(user_id)
        is_allowed = user.has_resource_profile(JBoxUserV2.RES_PROF_CLUSTER)
        if not is_allowed:
            ParallelHandler.log_error("Cluster access not allowed for user")
            response = {'code': -1, 'data': 'You do not have permissions to use any clusters'}
            self.write(response)
            return True

        cont = SessContainer.get_by_name(sessname)
        if cont is None:
            self.send_error()
            return

        ParallelHandler.log_debug("Parallel request %s for %s", mode, cont.debug_str())

        try:
            max_cores = user.get_max_cluster_cores()
            balance = user.get_balance()
            uc = UserCluster(user.get_user_id())
            if mode == 'status':
                status = uc.status()
                status['limits'] = {
                    'max_cores': max_cores,
                    'credits': balance
                }
                self.write_machinefile(cont, uc)
                response = {'code': 0, 'data': status}
            elif mode == 'terminate':
                action = 'terminate' if uc.isactive() else 'delete'
                uc.terminate_or_delete()
                response = {'code': 0, 'data': action}
            elif mode == 'create':
                ninsts = int(self.get_argument('ninsts', 0))
                avzone = self.get_argument('avzone', '')
                spot_price = float(self.get_argument('spot_price', 0.0))
                if ninsts > (max_cores / UserCluster.INSTANCE_CORES):
                    response = {'code': -1, 'data': 'You are allowed a maximum of ' + str(max_cores) + ' cores.'}
                elif (spot_price > UserCluster.INSTANCE_COST) or (spot_price < 0):
                    response = {
                        'code': -1,
                        'data': 'Bid price must be between $0 - $' + str(UserCluster.INSTANCE_COST) + '.'
                    }
                else:
                    uc.delete()
                    user_data = ParallelHandler.create_user_script(cont)
                    uc.create(ninsts, avzone, user_data, spot_price=spot_price)
                    uc.start()
                    response = {'code': 0, 'data': ''}
            else:
                response = {'code': -1, 'data': 'Unknown cluster operation ' + mode}
        except Exception as ex:
            ParallelHandler.log_error("exception in cluster operation")
            ParallelHandler._get_logger().exception("exception in cluster operation")
            response = {'code': -1, 'data': ex.message}

        self.write(response)
Example #38
0
 def backup_and_cleanup(dockid):
     cont = SessContainer(dockid)
     cont.stop()
     cont.delete(backup=True)
Example #39
0
        dname, fname = os.path.split(f)
        copyname = expanduser(os.path.join("~", fname))
        shutil.copyfile(f, copyname)


if __name__ == "__main__":
    conf_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), '../../engine/conf'))
    conf_file = os.path.join(conf_dir, 'tornado.conf')
    user_conf_file = os.path.join('/jboxengine/conf', 'jbox.user')

    JBoxCfg.read(conf_file, user_conf_file)
    JBoxCfg.dckr = docker.Client()

    LoggerMixin.configure()
    db.configure()
    SessContainer.configure()
    VolMgr.configure()

    plugin = JBPluginCloud.jbox_get_plugin(JBPluginCloud.JBP_BUCKETSTORE)
    if plugin is None:
        VolMgr.log_error("No plugin found for bucketstore")
        exit(1)

    ts = JBoxVol._get_user_home_timestamp()
    tsstr = ts.strftime("%Y%m%d_%H%M")
    VolMgr.log_debug("user_home_timestamp: %s", tsstr)

    imgf, pkgf = copy_for_upload(tsstr)
    copy_for_boot()

    bucket = 'juliabox-user-home-templates'
Example #40
0
 def launch_session(name, email, reuse=True):
     JBoxd._wait_for_session_backup(name)
     VolMgr.refresh_disk_use_status()
     SessContainer.launch_by_name(name, email, reuse=reuse)
     JBoxd.publish_perf_counters()
Example #41
0
 def _wait_for_session_backup(sessname):
     cont = SessContainer.get_by_name(sessname)
     if (cont is not None) and JBoxd._is_scheduled(JBoxAsyncJob.CMD_BACKUP_CLEANUP, (cont.dockid,)):
         JBoxd.log_debug("Waiting for backup of session %s", sessname)
         return False
     return True