def _addroute_report(report): proxyconf = KOOPLEX.get('proxy', {}) reportconf = KOOPLEX.get('reportserver', {}) target_url = reportconf.get('base_url', 'localhost') route_prefix = 'report' if report.reporttype != report.TP_STATIC: route_prefix = 'notebook' target_url = report.url_external kw = { 'url': os.path.join(proxyconf.get('base_url', 'localhost'), 'api', 'routes', route_prefix, report.proxy_path), 'headers': { 'Authorization': 'token %s' % proxyconf.get('auth_token', '') }, 'data': json.dumps({'target': target_url}), } logging.debug("+ %s ---> %s" % (kw['url'], target_url)) keeptrying(requests.post, 50, **kw) kw = { 'url': os.path.join(proxyconf.get('base_url', 'localhost'), 'api', 'routes', route_prefix, report.proxy_path_latest), 'headers': { 'Authorization': 'token %s' % proxyconf.get('auth_token', '') }, 'data': json.dumps({'target': target_url}), } logging.debug("Report proxy + %s ---> %s" % (kw['url'], target_url)) return keeptrying(requests.post, 50, **kw)
def openreport(request, report_id): """Renders new report list.""" user = request.user logger.debug("user %s, method: %s" % (user, request.method)) try: report = Report.objects.get(id = report_id) except Exception as e: logger.warning('Cannot resolve report id: %s -- %s' % (report_id, e)) return redirect('indexpage') if report.reporttype == report.TP_STATIC: url_external = report.url_external logger.debug('redirect: %s' % url_external) return redirect(url_external) elif report.reporttype == report.TP_DYNAMIC: container = Container.get_reportcontainer(report, create = True) container.docker_start() url_external = "%s/notebook/%s/report" % (KOOPLEX.get('base_url', 'localhost'), container.name) logger.debug('redirect: %s' % url_external) return redirect(url_external) elif report.reporttype == report.TP_BOKEH: container = Container.get_reportcontainer(report, create = True) container.docker_start() url_external = "%s/notebook/%s/report" % (KOOPLEX.get('base_url', 'localhost'), container.name) logger.debug('redirect: %s' % url_external) return redirect(url_external) elif report.reporttype == report.TP_SERVICE: container = Container.get_reportcontainer(report, create = True) container.docker_start() url_external = "%s/notebook/%s/report" % (KOOPLEX.get('base_url', 'localhost'), container.name) msg = "Report %s is started. API is at %s" % (report.name, url_external) logger.info(msg) messages.info(request, msg) return redirect('report:list') messages.error(request, "Rendering report type %s is not implemeted yet" % report.reporttype) return redirect('report:list')
def _removeroute_report(report): proxyconf = KOOPLEX.get('proxy', {}) reportconf = KOOPLEX.get('reportserver', {}) target_url = reportconf.get('base_url', 'localhost') kw = { 'url': os.path.join(proxyconf.get('base_url','localhost'), 'api', 'routes', report.proxy_path), 'headers': {'Authorization': 'token %s' % proxyconf.get('auth_token', '') }, } logging.debug("- %s -/-> %s" % (kw['url'], target_url)) return keeptrying(requests.delete, 5, **kw)
def openreport(request, report_id): """Renders new report list.""" user = request.user logger.debug("user %s, method: %s" % (user, request.method)) try: report = Report.objects.get(id=report_id) except Exception as e: logger.warning('Cannot resolve report id: %s -- %s' % (report_id, e)) return redirect('indexpage') if report.reporttype == report.TP_STATIC: url_external = report.url_external logger.debug('redirect: %s' % url_external) return redirect(url_external) elif report.reporttype == report.TP_SHINY: url_external = report.url_external logger.debug('redirect: %s' % url_external) return redirect(url_external) elif report.reporttype == report.TP_DYNAMIC: container = Container.get_reportcontainer(report, create=True) container.docker_start() url_external = "%s/notebook/%s/notebooks/%s?token=%s" % (KOOPLEX.get( 'base_url', 'localhost'), container.name, report.index, user.profile.token) logger.debug('redirect: %s ' % url_external) return redirect(url_external) elif report.reporttype == report.TP_BOKEH: container = Container.get_reportcontainer(report, create=True) container.docker_start() url_external = "%s/notebook/%s/report" % (KOOPLEX.get( 'base_url', 'localhost'), container.name) logger.debug('redirect: %s' % url_external) return redirect(url_external) elif report.reporttype == report.TP_DASH: container = Container.get_reportcontainer(report, create=True) container.docker_start() url_external = "%s/notebook/%s/report" % (KOOPLEX.get( 'base_url', 'localhost'), container.name) logger.debug('redirect: %s' % url_external) return redirect(url_external) elif report.reporttype == report.TP_SERVICE: container = Container.get_reportcontainer(report, create=True) container.docker_start() url_external = "%s/notebook/%s/report/help" % (KOOPLEX.get( 'base_url', 'localhost'), container.name) msg = "Report %s is started. API is at %s" % (report.name, url_external) logger.info(msg) messages.info(request, msg) return redirect('report:list') messages.error( request, "Rendering report type %s is not implemeted yet" % report.reporttype) return redirect('report:list')
def _removeroute_report(report): proxyconf = KOOPLEX.get('proxy', {}) reportconf = KOOPLEX.get('reportserver', {}) target_url = reportconf.get('base_url', 'localhost') kw = { 'url': os.path.join(proxyconf.get('base_url', 'localhost'), 'api', 'routes', report.proxy_path), 'headers': { 'Authorization': 'token %s' % proxyconf.get('auth_token', '') }, } logging.debug("- %s -/-> %s" % (kw['url'], target_url)) return keeptrying(requests.delete, 5, **kw)
def getroutes(): proxyconf = KOOPLEX.get('proxy', {}) kw = { 'url': os.path.join(proxyconf.get('base_url','localhost'), 'api', 'routes'), 'headers': {'Authorization': 'token %s' % proxyconf.get('auth_token', '') }, } return keeptrying(requests.get, 50, **kw)
def create_user_profile(sender, instance, created, **kwargs): if created: logger.info("New user %s" % instance) last_uid = Profile.objects.all().aggregate(models.Max('userid'))['userid__max'] uid = KOOPLEX.get('min_userid', 1000) if last_uid is None else last_uid + 1 token = pwgen.pwgen(64) Profile.objects.create(user = instance, userid = uid, token = token)
def _removeroute_container(container): proxyconf = KOOPLEX.get('proxy', {}) kw = { 'url': os.path.join(proxyconf.get('base_url','localhost'), 'api', 'routes', container.proxy_path), 'headers': {'Authorization': 'token %s' % proxyconf.get('auth_token', '') }, } logging.debug("- %s -/-> %s" % (kw['url'], container.url)) return keeptrying(requests.delete, 5, **kw)
def jupyter_session(container): """ """ info = { 'containername': container.name } kw = { 'url': os.path.join(KOOPLEX.get('spawner', {}).get('pattern_jupyterapi') % info, 'sessions'), 'headers': {'Authorization': 'token %s' % container.report.password, }, } return keeptrying(requests.get, 50, **kw)
def get_redirect_uri(self, state = None): """Build redirect with redirect_state parameter.""" base_url = KOOPLEX.get('base_url', 'http://localhost') url = os.path.join(base_url, "hub/oauth/complete/google-oauth2/") if self.REDIRECT_STATE and state: uri = url_add_parameters(url, {'redirect_state': state}) else: uri = url return uri
def create_user_profile(sender, instance, created, **kwargs): if created: logger.info("New user %s" % instance) last_uid = Profile.objects.all().aggregate( models.Max('userid'))['userid__max'] uid = KOOPLEX.get('min_userid', 1000) if last_uid is None else last_uid + 1 token = pwgen.pwgen(64) Profile.objects.create(user=instance, userid=uid, token=token)
class HydraOpenID(OpenIdConnectAuth): name = 'hydraoidc' OIDC_ENDPOINT = KOOPLEX.get('hydra_oidc_endpoint', 'https://localhost:4444') def get_redirect_uri(self, state=None): """Build redirect with redirect_state parameter.""" base_url = KOOPLEX.get('base_url', 'http://localhost') url = os.path.join(base_url, "hub/oauth/complete/hydraoidc/") logger.debug("IDP resp") if self.REDIRECT_STATE and state: uri = url_add_parameters(url, {'redirect_state': state}) else: uri = url return uri def get_user_details(self, response): try: return {#FIXME: a hydra most furan tolja az attributumokat! 'username': response['idp_user'], 'email': response['mail'][0], 'fullname': response['displayName'][0], 'first_name': response['givenName'][0], 'last_name': response['sn'][0], } except: 1 == 1 return { #FIXME: a hydra most furan tolja az attributumokat! 'username': response['idp_user'], 'email': response['mail'], #[0], 'fullname': response['displayName'], #[0], 'first_name': response['displayName'], #[0], 'last_name': response['displayName'], #[0], } def authenticate(self, request, **credentials): user = super(HydraOpenID, self).authenticate(request, **credentials) response = credentials.get('response', {}) logger.debug("IDP resp %s" % response) try: logger.debug("Authenticated (username) %s" % user.username) except: pass # currently held courses a = response.get('niifEduPersonHeldCourse', {}) coursecodes = CourseCode.parse(a) UserCourseCodeBinding.userattributes(user, coursecodes, is_teacher=True) # currently attended courses a = response.get('niifEduPersonAttendedCourse', {}) coursecodes = CourseCode.parse(a) UserCourseCodeBinding.userattributes(user, coursecodes, is_teacher=False) return user
def _addroute_report(report): proxyconf = KOOPLEX.get('proxy', {}) reportconf = KOOPLEX.get('reportserver', {}) target_url = reportconf.get('base_url', 'localhost') kw = { 'url': os.path.join(proxyconf.get('base_url','localhost'), 'api', 'routes', 'report', report.proxy_path), 'headers': {'Authorization': 'token %s' % proxyconf.get('auth_token', '') }, 'data': json.dumps({ 'target': target_url }), } logging.debug("+ %s ---> %s" % (kw['url'], target_url)) keeptrying(requests.post, 50, **kw) kw = { 'url': os.path.join(proxyconf.get('base_url','localhost'), 'api', 'routes', 'report', report.proxy_path_latest), 'headers': {'Authorization': 'token %s' % proxyconf.get('auth_token', '') }, 'data': json.dumps({ 'target': target_url }), } logging.debug("+ %s ---> %s" % (kw['url'], target_url)) return keeptrying(requests.post, 50, **kw)
def getroutes(): proxyconf = KOOPLEX.get('proxy', {}) kw = { 'url': os.path.join(proxyconf.get('base_url', 'localhost'), 'api', 'routes'), 'headers': { 'Authorization': 'token %s' % proxyconf.get('auth_token', '') }, } return keeptrying(requests.get, 50, **kw)
def list_imagenames(self): logger.debug("Listing image names") pattern_imagenamefilter = KOOPLEX.get('docker', {}).get('pattern_imagename_filter', r'^image-%(\w+):\w$') for image in self.client.images(all = True): if image['RepoTags'] is None: continue for tag in image['RepoTags']: if re.match(pattern_imagenamefilter, tag): _, imagename, _ = re.split(pattern_imagenamefilter, tag) logger.debug("Found image: %s" % imagename) yield imagename
def list_imagenames(self): logger.debug("Listing image names") pattern_imagenamefilter = KOOPLEX.get('docker', {}).get( 'pattern_imagename_filter', r'^image-%(\w+):\w$') for image in self.client.images(all=True): if image['RepoTags'] is None: continue for tag in image['RepoTags']: if re.match(pattern_imagenamefilter, tag): _, imagename, _ = re.split(pattern_imagenamefilter, tag) logger.debug("Found image: %s" % imagename) yield imagename
def _removeroute_container(container): proxyconf = KOOPLEX.get('proxy', {}) kw = { 'url': os.path.join(proxyconf.get('base_url', 'localhost'), 'api', 'routes', container.proxy_path), 'headers': { 'Authorization': 'token %s' % proxyconf.get('auth_token', '') }, } logging.debug("- %s -/-> %s" % (kw['url'], container.url)) return keeptrying(requests.delete, 5, **kw)
def droproutes(): proxyconf = KOOPLEX.get('proxy', {}) resp = getroutes() routes = json.loads(resp.content.decode()) for r, v in routes.items(): kw = { 'url': os.path.join(proxyconf.get('base_url','localhost'), 'api', 'routes', r[1:]), 'headers': {'Authorization': 'token %s' % proxyconf.get('auth_token', '') }, } logging.debug("- %s -/-> %s" % (kw['url'], v['target'])) resp_latest = keeptrying(requests.delete, 5, **kw) return resp_latest
def environment(self): envs = { 'NB_USER': self.user.username, 'NB_UID': self.user.profile.userid, 'NB_GID': self.user.profile.groupid, 'NB_URL': self.proxy_path, 'NB_PORT': KOOPLEX.get('spawner', {}).get('port', 8000), 'NB_HOST': KOOPLEX.get('base_url', 'localhost'), 'NB_TOKEN': self.user.profile.token, 'CONTAINER_NAME': self.name, } for ce in ContainerEnvironment.objects.filter(container = self): envs[ce.name] = ce.value logger.debug("Adding extra envs: %s"%ce.value) report = self.report if report: envs['REPORT_TYPE'] = report.reporttype envs['REPORT_DIR'] = os.path.join('/home/', 'report', report.cleanname, report.tag_name) envs['REPORT_ABS'] = os.path.join('notebook/', 'report-%s-%s'%(self.user.username, report.cleanname)) envs['REPORT_PORT'] = 9000 envs['REPORT_INDEX'] = report.index return envs
def _addroute_container(container, test=False): proxyconf = KOOPLEX.get('proxy', {}) if test: kw = { 'url': os.path.join(proxyconf.get('base_url', 'localhost'), 'api', 'routes', container.proxy_path_test), 'headers': { 'Authorization': 'token %s' % proxyconf.get('auth_token', '') }, 'data': json.dumps({'target': container.url_test}), } logging.debug("+ %s ---> %s test port" % (kw['url'], container.url_test)) else: kw = { 'url': os.path.join(proxyconf.get('base_url', 'localhost'), 'api', 'routes', container.proxy_path), 'headers': { 'Authorization': 'token %s' % proxyconf.get('auth_token', '') }, 'data': json.dumps({'target': container.url}), } logging.debug("+ %s ---> %s proxy path" % (kw['url'], container.url)) try: rc = ReportContainerBinding.objects.get(container=container) report = rc.report kw = { 'url': os.path.join(proxyconf.get('base_url', 'localhost'), 'api', 'routes', 'notebook', report.proxy_path_latest), 'headers': { 'Authorization': 'token %s' % proxyconf.get('auth_token', '') }, 'data': json.dumps({'target': container.url_test}), } logging.debug("+ %s ---> %s report proxy path latest" % (kw['url'], container.url)) keeptrying(requests.post, 50, **kw) except: logging.debug("Container is not for report") logging.debug("+ %s ---> %s" % (kw['url'], container.url)) return keeptrying(requests.post, 50, **kw)
def droproutes(): proxyconf = KOOPLEX.get('proxy', {}) resp = getroutes() routes = json.loads(resp.content.decode()) for r, v in routes.items(): kw = { 'url': os.path.join(proxyconf.get('base_url', 'localhost'), 'api', 'routes', r[1:]), 'headers': { 'Authorization': 'token %s' % proxyconf.get('auth_token', '') }, } logging.debug("- %s -/-> %s" % (kw['url'], v['target'])) resp_latest = keeptrying(requests.delete, 5, **kw)
def _addroute_container(container, test=False): proxyconf = KOOPLEX.get('proxy', {}) if test: kw = { 'url': os.path.join(proxyconf.get('base_url','localhost'), 'api', 'routes', container.proxy_path_test), 'headers': {'Authorization': 'token %s' % proxyconf.get('auth_token', '') }, 'data': json.dumps({ 'target': container.url_test }), } else: kw = { 'url': os.path.join(proxyconf.get('base_url','localhost'), 'api', 'routes', container.proxy_path), 'headers': {'Authorization': 'token %s' % proxyconf.get('auth_token', '') }, 'data': json.dumps({ 'target': container.url }), } logging.debug("+ %s ---> %s" % (kw['url'], container.url)) return keeptrying(requests.post, 50, **kw)
def __init__(self): logger.debug("init") ldapconf = KOOPLEX.get('ldap', {}) self.host = ldapconf.get('host', 'localhost') self.port = ldapconf.get('port', 389) try: self.base_dn = ldapconf['base_dn'] self.bind_dn = ldapconf['bind_dn'] self.bind_pw = ldapconf['bind_password'] except KeyError as e: logger.error("Cannot initialize ldap, KOOPLEX['ldap'][key] key is missing -- %s" % e) raise server = ldap3.Server(host = self.host, port = self.port) self.connection = ldap3.Connection(server, self.bind_dn, self.bind_pw) success = self.connection.bind() if not success: logger.error("Cannot bind to ldap server") raise LdapException("Cannot bind to ldap server")
def get_reportcontainer(report, create): logger.debug("report %s" % (report)) try: return ReportContainerBinding.objects.get(report = report).container except ReportContainerBinding.DoesNotExist: logger.debug("ReportContainer for report %s does not exist" % report) if create: container_name_dict = { 'un': report.creator.username, 'rn': report.cleanname, 'tn': report.tag_name, # 'ts': report.ts_human.replace(':', '').replace('_', '') } containername = KOOPLEX.get('pattern_report_containername','report-%(un)s-%(rn)s-%(tn)s') % container_name_dict container = Container.objects.create(name = containername, user = report.creator, image=report.image) ReportContainerBinding.objects.create(report = report, container = container) logger.debug("new container in db %s" % container) return container raise Container.DoesNotExist
def __init__(self): logger.debug("init") ldapconf = KOOPLEX.get('ldap', {}) self.host = ldapconf.get('host', 'localhost') self.port = ldapconf.get('port', 389) try: self.base_dn = ldapconf['base_dn'] self.bind_dn = ldapconf['bind_dn'] self.bind_pw = ldapconf['bind_password'] except KeyError as e: logger.error( "Cannot initialize ldap, KOOPLEX['ldap'][key] key is missing -- %s" % e) raise server = ldap3.Server(host=self.host, port=self.port) self.connection = ldap3.Connection(server, self.bind_dn, self.bind_pw) success = self.connection.bind() if not success: logger.error("Cannot bind to ldap server") raise LdapException("Cannot bind to ldap server")
def groupid(self): return KOOPLEX.get('ldap', {}).get('gid_users', 1000)
class Volume(models.Model): pattern = KOOPLEX.get('volumepattern', {}) HOME = 'home' GARBAGE = 'garbage' GIT = 'git' FILESYNC = 'filesync' SHARE = 'share' WORKDIR = 'workdir' FUNCTIONAL = 'functional' STORAGE = 'storage' PRIVATE = 'private' COURSE_SHARE = 'course' COURSE_WORKDIR = 'usercourse' COURSE_ASSIGNMENTDIR = 'assignment' REPORT = 'report' VOLUME_TYPE_LIST = [ HOME, GARBAGE, GIT, FILESYNC, SHARE, WORKDIR, FUNCTIONAL, STORAGE, COURSE_SHARE, COURSE_WORKDIR, COURSE_ASSIGNMENTDIR, REPORT, PRIVATE ] VOLUME_TYPE_LIST_USER = [FUNCTIONAL] #, STORAGE, PRIVATE ] name = models.CharField(max_length=64, unique=True) displayname = models.CharField(max_length=64) description = models.TextField(null=True) volumetype = models.CharField(max_length=16, choices=[(x, TYPE_LOOKUP[x]) for x in VOLUME_TYPE_LIST]) is_present = models.BooleanField(default=True) def __str__(self): return self.displayname @staticmethod def try_create(volumename): for x in Volume.VOLUME_TYPE_LIST: pattern = Volume.pattern.get(x, r'^(%s)$' % x) if re.match(pattern, volumename): _, dirname, _ = re.split(pattern, volumename) return Volume.objects.create(name=volumename, displayname=dirname, description='%s (%s)' % (TYPE_LOOKUP[x], dirname), volumetype=x) # @staticmethod # def lookup(volumetype, **kw): #FIXME: check if still used somewhere # if not volumetype in Volume.VOLUME_TYPE_LIST: # raise Volume.DoesNotExist # return Volume.objects.get(volumetype = volumetype['tag'], **kw) @staticmethod def filter(volumetype, **kw): if not volumetype in Volume.VOLUME_TYPE_LIST: raise Volume.DoesNotExist user = kw.pop('user', None) if user: logger.error("NotImplementedError") #FIXME pass for volume in Volume.objects.filter(volumetype=volumetype, **kw): yield volume # def is_volumetype(self, volumetype): # try: # return self.volumetype == volumetype['tag'] # except KeyError: # return False def mode(self, user): if self.volumetype == Volume.FUNCTIONAL: for binding in VolumeOwnerBinding.objects.filter(volume=self): if binding.owner == user: return 'rw' return 'ro' if self.volumetype == Volume.STORAGE: if hasattr(self, 'extrafields'): return 'rw' if self.extrafields.readwrite else 'ro' else: return 'ro' return 'rw' @property def mountpoint(self): pattern = self.pattern.get(self.volumetype, r'^(%s)$' % self.volumetype) _, dirname, _ = re.split(pattern, self.name) if self.volumetype == Volume.FUNCTIONAL: return os.path.join('/vol', dirname) if self.volumetype == Volume.STORAGE: return os.path.join('/data', dirname) return os.path.join('/mnt/.volumes', dirname)
class Dirname: mountpoint = KOOPLEX.get('mountpoint', {}) @staticmethod def userhome(user): return os.path.join(Dirname.mountpoint['home'], user.username) @staticmethod def usergarbage(user): return os.path.join(Dirname.mountpoint['garbage'], user.username) @staticmethod def reportroot(user): return os.path.join(Dirname.mountpoint['report'], user.username) @staticmethod def reportprepare(user): return os.path.join(Dirname.reportroot(user), '_prepare') @staticmethod def report(report): return os.path.join(Dirname.reportroot(report.creator), standardize_str(report.name)) @staticmethod def report_with_tag(report): return os.path.join(Dirname.reportroot(report.creator), standardize_str(report.name), report.tag_name) @staticmethod def share(userprojectbinding): return os.path.join(Dirname.mountpoint['share'], userprojectbinding.project.uniquename) @staticmethod def workdir(userprojectbinding): return os.path.join(Dirname.mountpoint['workdir'], userprojectbinding.uniquename) @staticmethod def vcpcache(vcproject): return os.path.join(Dirname.mountpoint['git'], vcproject.clone_folder) @staticmethod def fscache(fslibrary): return os.path.join(Dirname.mountpoint['filesync'], fslibrary.sync_folder) @staticmethod def course(course): return os.path.join(Dirname.mountpoint['course'], course.folder) @staticmethod def courseprivate(course): return os.path.join(Dirname.course(course), 'private') @staticmethod def coursepublic(course): return os.path.join(Dirname.course(course), 'public') @staticmethod def courseworkdir(usercoursebinding): return os.path.join(Dirname.mountpoint['usercourse'], usercoursebinding.course.folder) @staticmethod def usercourseworkdir(usercoursebinding): return os.path.join(Dirname.courseworkdir(usercoursebinding), usercoursebinding.user.username) @staticmethod def assignmentsource(assignment): return os.path.join( Dirname.courseprivate(assignment.coursecode.course), assignment.folder) @staticmethod def assignmentworkdir(userassignmentbinding): from hub.models import UserCourseBinding usercoursebinding = UserCourseBinding.objects.get( user=userassignmentbinding.user, course=userassignmentbinding.assignment.coursecode.course) wd = Dirname.usercourseworkdir(usercoursebinding) return os.path.join(wd, userassignmentbinding.assignment.safename) @staticmethod def assignmentcorrectdir(userassignmentbinding): assignment = userassignmentbinding.assignment user = userassignmentbinding.user namefield = "%s%s_%s" % (deaccent_str(user.first_name).replace( ' ', ''), deaccent_str(user.last_name).replace(' ', ''), user.username) datefield = userassignmentbinding.submitted_at.strftime('%Y_%m_%d') return os.path.join( Dirname.mountpoint['assignment'], assignment.coursecode.course.folder, 'feedback-%s-%s-%s' % (assignment.safename, namefield, datefield)) @staticmethod def containervolume_listfolders(container, volume): from hub.models import UserCourseBinding, UserAssignmentBinding def get_usercoursebinding_userstate(): try: usercoursebinding = UserCourseBinding.objects.get( user=container.user, course=container.course) except UserCourseBinding.DoesNotExist: logger.error( "Silly situation, cannot map %s %s COZ user course binding instance is missing" % (volume, container)) return None, None if container.course in container.user.profile.courses_taught(): return usercoursebinding, 'teacher' elif container.course in container.user.profile.courses_attend(): return usercoursebinding, 'student' else: logger.error("Silly situation, cannot map %s %s" % (volume, container)) return None, None if volume.volumetype == volume.HOME: yield Dirname.userhome(container.user) elif volume.volumetype == volume.GARBAGE: yield Dirname.usergarbage(container.user) elif volume.volumetype == volume.SHARE: for upb in container.userprojectbindings: yield Dirname.share(upb) elif volume.volumetype == volume.WORKDIR: for upb in container.userprojectbindings: yield Dirname.workdir(upb) elif volume.volumetype == volume.GIT: for vcppb in container.vcprojectprojectbindings: yield Dirname.vcpcache(vcppb.vcproject) elif volume.volumetype == volume.FILESYNC: for fslpb in container.fslibraryprojectbindings: yield Dirname.fscache(fslpb.fslibrary) elif volume.volumetype == volume.COURSE_SHARE: if container.course in container.user.profile.courses_taught(): yield Dirname.course(container.course) elif container.course in container.user.profile.courses_attend(): yield Dirname.coursepublic(container.course) else: logger.error("Silly situation, cannot map %s %s" % (volume, container)) elif volume.volumetype == volume.COURSE_WORKDIR and container.course: usercoursebinding, userstatus = get_usercoursebinding_userstate() if userstatus == 'teacher': yield Dirname.courseworkdir(usercoursebinding) elif userstatus == 'student': yield Dirname.usercourseworkdir(usercoursebinding) else: yield "OOPS_%s" % volume.volumetype elif volume.volumetype == volume.COURSE_ASSIGNMENTDIR and container.course: _, userstatus = get_usercoursebinding_userstate() if userstatus == 'teacher': for binding in UserAssignmentBinding.objects.all(): if binding.state == UserAssignmentBinding.ST_QUEUED or binding.corrector is None or binding.assignment.coursecode.course != container.course: continue if binding.corrector != container.user: continue yield Dirname.assignmentcorrectdir(binding) elif userstatus == 'student': for binding in UserAssignmentBinding.objects.filter( user=container.user): if binding.state == UserAssignmentBinding.ST_QUEUED or binding.corrector is None or binding.assignment.coursecode.course != container.course: continue yield Dirname.assignmentcorrectdir(binding) else: yield "OOPS_%s" % volume.volumetype elif volume.volumetype == volume.REPORT: yield Dirname.reportroot(container.user) else: yield "MISSING_DIRNAME_%s" % volume.volumetype
class Docker: dockerconf = KOOPLEX.get('docker', {}) def __init__(self): base_url = self.dockerconf.get('base_url', '') self.client = Client(base_url=base_url) logger.debug("Client init") self.check = None def list_imagenames(self): logger.debug("Listing image names") pattern_imagenamefilter = KOOPLEX.get('docker', {}).get( 'pattern_imagename_filter', r'^image-%(\w+):\w$') for image in self.client.images(all=True): if image['RepoTags'] is None: continue for tag in image['RepoTags']: if re.match(pattern_imagenamefilter, tag): _, imagename, _ = re.split(pattern_imagenamefilter, tag) logger.debug("Found image: %s" % imagename) yield imagename def list_volumenames(self): logger.debug("Listing volume names") volumes = self.client.volumes() for volume in volumes['Volumes']: yield volume['Name'] def create_volume(self, volume): volume_dir = None #self.dockerconf.get('volume_dir', '') if volume_dir: self.client.create_volume(name=volume.name, driver='local', driver_opts={ 'device': '%s/%s/' % (volume_dir, volume.name), 'o': 'bind', 'type': 'none' }) else: self.client.create_volume(name=volume.name, ) logger.debug("Volume %s created" % volume.name) return True #self.get_container(container) def delete_volume(self, volume): self.client.remove_volume(name=volume.name) logger.debug("Volume %s deleted" % volume.name) def get_container(self, container): for item in self.client.containers(all=True): # docker API prepends '/' in front of container names if '/' + container.name in item['Names']: logger.debug("Get container %s" % container.name) return item return None def create_container(self, container): volumes = [] # the list of mount points in the container binds = {} # a mapping dictionary of the container mounts for volume in container.volumes: logger.debug("container %s, volume %s" % (container, volume)) mp = volume.mountpoint volumes.append(mp) binds[volume.name] = { 'bind': mp, 'mode': volume.mode(container.user) } logger.debug("container %s binds %s" % (container, binds)) host_config = self.client.create_host_config( binds=binds, privileged=True, mem_limit='2g', memswap_limit='170m', mem_swappiness=0, # oom_kill_disable = True, cpu_shares=2, ) network = self.dockerconf.get('network', 'host') networking_config = {'EndpointsConfig': {network: {}}} ports = self.dockerconf.get('container_ports', [8000, 9000]) imagename = container.image.imagename if container.image else self.dockerconf.get( 'default_image', 'basic') args = { 'name': container.name, 'image': imagename, 'detach': True, 'hostname': container.name, 'host_config': host_config, 'networking_config': networking_config, 'environment': container.environment, 'volumes': volumes, 'ports': ports, } self.client.create_container(**args) logger.debug("Container created") self.managemount(container) #FIXME: check if not called twice return self.get_container(container) def _writefile(self, container_name, path, filename, content): import tarfile import time from io import BytesIO tarstream = BytesIO() tar = tarfile.TarFile(fileobj=tarstream, mode='w') tarinfo = tarfile.TarInfo(name=filename) tarinfo.size = len(content) tarinfo.mtime = time.time() tar.addfile(tarinfo, BytesIO(content)) tar.close() tarstream.seek(0) try: status = self.client.put_archive(container=container_name, path=path, data=tarstream) logger.info("container %s put_archive %s/%s returns %s" % (container_name, path, filename, status)) except Exception as e: logger.error("container %s put_archive %s/%s fails -- %s" % (container_name, path, filename, e)) def managemount(self, container): from kooplex.lib.fs_dirname import Dirname path, filename = os.path.split( self.dockerconf.get('mountconf', '/tmp/mount.conf')) mapper = [] for v in container.volumes: mapper.extend([ "%s:%s" % (v.volumetype, d) for d in Dirname.containervolume_listfolders(container, v) ]) #NOTE: mounter uses read to process the mapper configuration, thus we need to make sure '\n' terminates the config mapper file mapper.append('') logger.debug("container %s map %s" % (container, mapper)) file_data = "\n".join(mapper).encode('utf8') self._writefile(container.name, path, filename, file_data) def trigger_impersonator(self, vcproject): #FIXME: dont call it 1-by-1 from kooplex.lib.fs_dirname import Dirname container_name = self.dockerconf.get('impersonator', 'impersonator') path, filename = os.path.split( self.dockerconf.get('gitcommandconf', '/tmp/gitcommand.conf')) cmdmaps = [] token = vcproject.token fn_clonesh = os.path.join(Dirname.vcpcache(vcproject), "clone.sh") fn_key = os.path.join(Dirname.userhome(vcproject.token.user), '.ssh', token.fn_rsa) cmdmaps.append( "%s:%s:%s:%s" % (token.user.username, fn_key, token.repository.domain, fn_clonesh)) cmdmaps.append('') file_data = "\n".join(cmdmaps).encode('utf8') self._writefile(container_name, path, filename, file_data) def run_container(self, container): docker_container_info = self.get_container(container) if docker_container_info is None: logger.debug("Container did not exist, Creating new one") docker_container_info = self.create_container(container) container_state = docker_container_info['Status'] if container_state == 'Created' or container_state.startswith( 'Exited'): logger.debug("Starting container") self.start_container(container) def refresh_container_state(self, container): docker_container_info = self.get_container(container) container_state = docker_container_info['State'] logger.debug("Container state %s" % container_state) container.last_message = str(container_state) container.last_message_at = now() container.save() def start_container(self, container): self.client.start(container.name) # we need to retrieve the container state after starting it docker_container_info = self.get_container(container) container_state = docker_container_info['State'] logger.debug("Container state %s" % container_state) container.last_message = str(container_state) container.last_message_at = now() assert container_state == 'running', "Container failed to start: %s" % docker_container_info def stop_container(self, container): try: self.client.stop(container.name) container.last_message = 'Container stopped' except Exception as e: logger.warn("docker container not found by API -- %s" % e) container.last_message = str(e) def remove_container(self, container): try: self.client.remove_container(container.name) container.last_message = 'Container removed' container.last_message_at = now() except Exception as e: logger.warn("docker container not found by API -- %s" % e) container.last_message = str(e) container.last_message_at = now() logger.debug("Container removed %s" % container.name) #FIXME: az execute2 lesz az igazi... def execute(self, container, command): logger.info("execution: %s in %s" % (command, container)) execution = self.client.exec_create(container=container.name, cmd=shlex.split(command)) return self.client.exec_start(execution, detach=False) def execute2(self, container, command): logger.info("execution: %s in %s" % (command, container)) execution = self.client.exec_create(container=container.name, cmd=shlex.split(command)) response = self.client.exec_start(exec_id=execution['Id'], stream=False) check = self.client.exec_inspect(exec_id=execution['Id']) self.check = check if check['ExitCode'] != 0: logger.error('Execution %s in %s failed -- %s' % (command, container, check)) return response.decode()
""" @author: David Visontai, Jozsef Steger """ import os import json import requests import logging from kooplex.settings import KOOPLEX from kooplex.lib import keeptrying, standardize_str logger = logging.getLogger(__name__) reportconf = KOOPLEX.get('reportserver', {}) def removeroute(instance): if isinstance(instance, Container): return _removeroute_container(instance) elif isinstance(instance, Report): return _removeroute_report(instance) logger.error('Not implemented') def add_report_nginx_api(report): # import htpasswd str_name = standardize_str(report.proxy_path) conf_text = """ location /report/%s { auth_basic "username is 'report' "; auth_basic_user_file /etc/passwords/%s;
class Filename: mountpoint = KOOPLEX.get('mountpoint', {}) @staticmethod def userhome_garbage(user): return os.path.join(Dirname.mountpoint['garbage'], "user-%s.%f.tar.gz" % (user.username, time.time())) @staticmethod def share_garbage(userprojectbinding): return os.path.join( Dirname.mountpoint['garbage'], "projectshare-%s.%f.tar.gz" % (userprojectbinding.project.uniquename, time.time())) @staticmethod def workdir_archive(userprojectbinding): return os.path.join( Dirname.mountpoint['home'], userprojectbinding.user.username, "garbage", "workdir-%s.%f.tar.gz" % (userprojectbinding.uniquename, time.time())) @staticmethod def vcpcache_archive(vcproject): return os.path.join( Dirname.mountpoint['home'], vcproject.token.user.username, "garbage", "git-%s.%f.tar.gz" % (vcproject.uniquename, time.time())) @staticmethod def course_garbage(course): return os.path.join( Dirname.mountpoint['garbage'], "course-%s.%f.tar.gz" % (course.folder, time.time())) @staticmethod def courseworkdir_archive(usercoursebinding): return os.path.join( Dirname.mountpoint['home'], usercoursebinding.user.username, "garbage", "%s.%f.tar.gz" % (usercoursebinding.course.folder, time.time())) @staticmethod def assignmentsnapshot(assignment): return os.path.join( Dirname.mountpoint['assignment'], assignment.coursecode.course.folder, 'assignmentsnapshot-%s.%d.tar.gz' % (assignment.safename, assignment.created_at.timestamp())) @staticmethod def assignmentsnapshot_garbage(assignment): return os.path.join( Dirname.mountpoint['garbage'], 'assignmentsnapshot-%s-%s-%s-%f.tar.gz' % (assignment.coursecode.course.folder, assignment.safename, assignment.created_at.timestamp(), time.time())) @staticmethod def assignmentcollection(userassignmentbinding): assignment = userassignmentbinding.assignment return os.path.join( Dirname.mountpoint['assignment'], assignment.coursecode.course.folder, 'submitted-%s-%s.%d.tar.gz' % (assignment.safename, userassignmentbinding.user.username, userassignmentbinding.submitted_at.timestamp())) @staticmethod def report_garbage(report): return os.path.join( Dirname.mountpoint['garbage'], report.creator.username, "report-%s-%s.%f.tar.gz" % (report.name, report.ts_human, time.time()))
def url_test(self): return "http://%s:%d" % (self.name, KOOPLEX.get('spawner', {}).get('port_test', 9000))
def url(self): return "http://%s:%d" % (self.name, KOOPLEX.get('spawner', {}).get('port', 8000))
def imagename(self): return KOOPLEX.get('docker', {}).get('pattern_imagename', 'image-%(imagename)s') % { 'imagename': self.name }