def _create_new(name, email): home_disk = VolMgr.get_disk_for_user(email) pkgs_disk = VolMgr.get_pkg_mount_for_user(email) vols = { home_disk.disk_path: { 'bind': SessContainer.VOLUMES[0], 'ro': False }, pkgs_disk.disk_path: { 'bind': SessContainer.VOLUMES[1], 'ro': True } } port_bindings = {p: ('127.0.0.1', ) for p in SessContainer.PORTS} hostcfg = docker.utils.create_host_config( binds=vols, port_bindings=port_bindings, mem_limit=SessContainer.MEM_LIMIT) jsonobj = BaseContainer.DCKR.create_container( SessContainer.DCKR_IMAGE, detach=True, host_config=hostcfg, cpu_shares=SessContainer.CPU_LIMIT, ports=SessContainer.PORTS, volumes=SessContainer.VOLUMES, hostname='juliabox', name=name) dockid = jsonobj["Id"] cont = SessContainer(dockid) SessContainer.log_info( "Created %s with hostcfg %r, cpu_limit: %r, volumes: %r", cont.debug_str(), hostcfg, SessContainer.CPU_LIMIT, vols) return cont
def _create_new(name, email): home_disk = VolMgr.get_disk_for_user(email) pkgs_disk = VolMgr.get_pkg_mount_for_user(email) vols = { home_disk.disk_path: { 'bind': SessContainer.VOLUMES[0], 'ro': False }, pkgs_disk.disk_path: { 'bind': SessContainer.VOLUMES[1], 'ro': True } } port_bindings = {p: ('127.0.0.1',) for p in SessContainer.PORTS} hostcfg = docker.utils.create_host_config(binds=vols, port_bindings=port_bindings, mem_limit=SessContainer.MEM_LIMIT) jsonobj = BaseContainer.DCKR.create_container(SessContainer.DCKR_IMAGE, detach=True, host_config=hostcfg, cpu_shares=SessContainer.CPU_LIMIT, ports=SessContainer.PORTS, volumes=SessContainer.VOLUMES, hostname='juliabox', name=name) dockid = jsonobj["Id"] cont = SessContainer(dockid) SessContainer.log_info("Created %s with hostcfg %r, cpu_limit: %r, volumes: %r", cont.debug_str(), hostcfg, SessContainer.CPU_LIMIT, vols) return cont
def maintain(max_timeout=0, inactive_timeout=0): SessContainer.log_info("Starting container maintenance...") tnow = datetime.datetime.now(pytz.utc) tmin = datetime.datetime(datetime.MINYEAR, 1, 1, tzinfo=pytz.utc) stop_before = (tnow - datetime.timedelta(seconds=max_timeout)) if (max_timeout > 0) else tmin stop_inacive_before = (tnow - datetime.timedelta(seconds=inactive_timeout)) if (inactive_timeout > 0) else tmin all_containers = BaseContainer.session_containers(allcontainers=True) all_cnames = {} container_id_list = [] for cdesc in all_containers: cid = cdesc['Id'] cont = SessContainer(cid) container_id_list.append(cid) cname = cont.get_name() if cname is None: SessContainer.log_debug("Ignoring %s", cont.debug_str()) continue all_cnames[cname] = cid c_is_active = cont.is_running() or cont.is_restarting() last_ping = SessContainer._get_last_ping(cname) # if we don't have a ping record, create one (we must have restarted) if (last_ping is None) and c_is_active: SessContainer.log_info("Discovered new container %s", cont.debug_str()) SessContainer.record_ping(cname) start_time = cont.time_started() # check that start time is not absurdly small (indicates a continer that's starting up) start_time_not_zero = (tnow-start_time).total_seconds() < (365*24*60*60) if (start_time < stop_before) and start_time_not_zero: # don't allow running beyond the limit for long running sessions # SessContainer.log_info("time_started " + str(cont.time_started()) + # " delete_before: " + str(delete_before) + # " cond: " + str(cont.time_started() < delete_before)) SessContainer.log_warn("Running beyond allowed time %s. Scheduling cleanup.", cont.debug_str()) SessContainer.invalidate_container(cont.get_name()) JBoxAsyncJob.async_backup_and_cleanup(cont.dockid) elif (last_ping is not None) and c_is_active and (last_ping < stop_inacive_before): # if inactive for too long, stop it # SessContainer.log_info("last_ping " + str(last_ping) + " stop_before: " + str(stop_before) + # " cond: " + str(last_ping < stop_before)) SessContainer.log_warn("Inactive beyond allowed time %s. Scheduling cleanup.", cont.debug_str()) SessContainer.invalidate_container(cont.get_name()) JBoxAsyncJob.async_backup_and_cleanup(cont.dockid) # delete ping entries for non exixtent containers for cname in SessContainer.PINGS.keys(): if cname not in all_cnames: del SessContainer.PINGS[cname] SessContainer.VALID_CONTAINERS = all_cnames VolMgr.refresh_disk_use_status(container_id_list=container_id_list) SessContainer.log_info("Finished container maintenance.")
def copy_for_upload(tstamp): img_dir, img_file = os.path.split(JBoxVol.USER_HOME_IMG) new_img_file_name = 'user_home_' + tstamp + '.tar.gz' new_img_file = os.path.join(img_dir, new_img_file_name) shutil.copyfile(JBoxVol.USER_HOME_IMG, new_img_file) new_pkg_file_name = 'julia_packages_' + tstamp + '.tar.gz' new_pkg_file = os.path.join(img_dir, new_pkg_file_name) shutil.copyfile(JBoxVol.PKG_IMG, new_pkg_file) VolMgr.log_debug("new image files : %s, %s", new_img_file, new_pkg_file) return new_img_file, new_pkg_file
def before_delete(self, cname, backup): for disktype in (JBoxVol.JBP_USERHOME, JBoxVol.JBP_PKGBUNDLE, JBoxVol.JBP_DATA, JBoxVol.JBP_CONFIG): disk = VolMgr.get_disk_from_container(self.dockid, disktype) if disk is not None: disk.release(backup=backup) if cname is not None: SessContainer.PINGS.pop(cname, None)
def create_user_script(cont): vol = VolMgr.get_disk_from_container(cont.dockid, JBoxVol.JBP_USERHOME) pub_key_file = os.path.join(vol.disk_path, ".ssh", "id_rsa.pub") with open(pub_key_file, 'r') as f: pub_key = f.read() auth_key_file = "/home/juser/.ssh/authorized_keys" template = '#! /usr/bin/env bash\n\njulia -e 0\nsudo -u juser sh -c "echo \\\"%s\\\" >> %s && chmod 600 %s"' return template % (pub_key, auth_key_file, auth_key_file)
def _write_machinefile(self, cont, filename, machines): cluster_hosts = set(machines) if len(cluster_hosts) == 0: return # write out the machinefile on the docker's filesystem vol = VolMgr.get_disk_from_container(cont.dockid, JBoxVol.JBP_USERHOME) machinefile = os.path.join(vol.disk_path, ".juliabox", filename) existing_hosts = set() try: with open(machinefile, 'r') as f: existing_hosts = set([x.rstrip('\n') for x in f.readlines()]) except: pass if cluster_hosts == existing_hosts: return self.log_debug("writing machinefile for %s to path: %s", cont.debug_str(), machinefile) with open(machinefile, 'w') as f: for host in cluster_hosts: f.write(host+'\n')
copyname = expanduser(os.path.join("~", fname)) shutil.copyfile(f, copyname) if __name__ == "__main__": conf_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), '../../engine/conf')) conf_file = os.path.join(conf_dir, 'tornado.conf') user_conf_file = os.path.join('/jboxengine/conf', 'jbox.user') JBoxCfg.read(conf_file, user_conf_file) JBoxCfg.dckr = docker.Client() LoggerMixin.configure() db.configure() SessContainer.configure() VolMgr.configure() plugin = JBPluginCloud.jbox_get_plugin(JBPluginCloud.JBP_BUCKETSTORE) if plugin is None: VolMgr.log_error("No plugin found for bucketstore") exit(1) ts = JBoxVol._get_user_home_timestamp() tsstr = ts.strftime("%Y%m%d_%H%M") VolMgr.log_debug("user_home_timestamp: %s", tsstr) imgf, pkgf = copy_for_upload(tsstr) copy_for_boot() bucket = 'juliabox-user-home-templates'
def init(): LoggerMixin.configure() db.configure() VolMgr.configure()
def get_disk_allocated(self): disk = VolMgr.get_disk_from_container(self.dockid, JBoxVol.JBP_USERHOME) if disk is not None: return disk.get_disk_allocated_size() return 0
def get_disk_space_used(self): disk = VolMgr.get_disk_from_container(self.dockid, JBoxVol.JBP_USERHOME) if disk is not None: return disk.get_disk_space_used() return 0
def maintain(max_timeout=0, inactive_timeout=0): SessContainer.log_info("Starting container maintenance...") tnow = datetime.datetime.now(pytz.utc) tmin = datetime.datetime(datetime.MINYEAR, 1, 1, tzinfo=pytz.utc) stop_before = (tnow - datetime.timedelta(seconds=max_timeout)) if ( max_timeout > 0) else tmin stop_inacive_before = (tnow - datetime.timedelta( seconds=inactive_timeout)) if (inactive_timeout > 0) else tmin all_containers = BaseContainer.session_containers(allcontainers=True) all_cnames = {} container_id_list = [] for cdesc in all_containers: cid = cdesc['Id'] cont = SessContainer(cid) container_id_list.append(cid) cname = cont.get_name() if cname is None: SessContainer.log_debug("Ignoring %s", cont.debug_str()) continue all_cnames[cname] = cid c_is_active = cont.is_running() or cont.is_restarting() last_ping = SessContainer._get_last_ping(cname) # if we don't have a ping record, create one (we must have restarted) if (last_ping is None) and c_is_active: SessContainer.log_info("Discovered new container %s", cont.debug_str()) SessContainer.record_ping(cname) start_time = cont.time_started() # check that start time is not absurdly small (indicates a continer that's starting up) start_time_not_zero = (tnow - start_time).total_seconds() < ( 365 * 24 * 60 * 60) if (start_time < stop_before) and start_time_not_zero: # don't allow running beyond the limit for long running sessions # SessContainer.log_info("time_started " + str(cont.time_started()) + # " delete_before: " + str(delete_before) + # " cond: " + str(cont.time_started() < delete_before)) SessContainer.log_warn( "Running beyond allowed time %s. Scheduling cleanup.", cont.debug_str()) SessContainer.invalidate_container(cont.get_name()) JBoxAsyncJob.async_backup_and_cleanup(cont.dockid) elif (last_ping is not None) and c_is_active and ( last_ping < stop_inacive_before): # if inactive for too long, stop it # SessContainer.log_info("last_ping " + str(last_ping) + " stop_before: " + str(stop_before) + # " cond: " + str(last_ping < stop_before)) SessContainer.log_warn( "Inactive beyond allowed time %s. Scheduling cleanup.", cont.debug_str()) SessContainer.invalidate_container(cont.get_name()) JBoxAsyncJob.async_backup_and_cleanup(cont.dockid) elif not c_is_active and ( (tnow - cont.time_finished()).total_seconds() > (10 * 60)): SessContainer.log_warn("Dead container %s. Deleting.", cont.debug_str()) cont.delete(backup=False) del all_cnames[cname] container_id_list.remove(cid) # delete ping entries for non exixtent containers for cname in SessContainer.PINGS.keys(): if cname not in all_cnames: del SessContainer.PINGS[cname] SessContainer.VALID_CONTAINERS = all_cnames VolMgr.refresh_disk_use_status(container_id_list=container_id_list) SessContainer.log_info("Finished container maintenance.")
def init(): LoggerMixin.configure() db.configure() VolMgr.configure() JBoxDisk.PLUGIN = JBPluginCloud.jbox_get_plugin( JBPluginCloud.JBP_BUCKETSTORE)
def init(): LoggerMixin.configure() db.configure() VolMgr.configure() JBoxDisk.PLUGIN = JBPluginCloud.jbox_get_plugin(JBPluginCloud.JBP_BUCKETSTORE)