def _addroute_report(report):
proxyconf = KOOPLEX.get('proxy', {})
reportconf = KOOPLEX.get('reportserver', {})
target_url = reportconf.get('base_url', 'localhost')
route_prefix = 'report'
if report.reporttype != report.TP_STATIC:
route_prefix = 'notebook'
target_url = report.url_external
kw = {
'url':
os.path.join(proxyconf.get('base_url', 'localhost'), 'api', 'routes',
route_prefix, report.proxy_path),
'headers': {
'Authorization': 'token %s' % proxyconf.get('auth_token', '')
},
'data':
json.dumps({'target': target_url}),
}
logging.debug("+ %s ---> %s" % (kw['url'], target_url))
keeptrying(requests.post, 50, **kw)
kw = {
'url':
os.path.join(proxyconf.get('base_url', 'localhost'), 'api', 'routes',
route_prefix, report.proxy_path_latest),
'headers': {
'Authorization': 'token %s' % proxyconf.get('auth_token', '')
},
'data':
json.dumps({'target': target_url}),
}
logging.debug("Report proxy + %s ---> %s" % (kw['url'], target_url))
return keeptrying(requests.post, 50, **kw)
def openreport(request, report_id):
"""Renders new report list."""
user = request.user
logger.debug("user %s, method: %s" % (user, request.method))
try:
report = Report.objects.get(id = report_id)
except Exception as e:
logger.warning('Cannot resolve report id: %s -- %s' % (report_id, e))
return redirect('indexpage')
if report.reporttype == report.TP_STATIC:
url_external = report.url_external
logger.debug('redirect: %s' % url_external)
return redirect(url_external)
elif report.reporttype == report.TP_DYNAMIC:
container = Container.get_reportcontainer(report, create = True)
container.docker_start()
url_external = "%s/notebook/%s/report" % (KOOPLEX.get('base_url', 'localhost'), container.name)
logger.debug('redirect: %s' % url_external)
return redirect(url_external)
elif report.reporttype == report.TP_BOKEH:
container = Container.get_reportcontainer(report, create = True)
container.docker_start()
url_external = "%s/notebook/%s/report" % (KOOPLEX.get('base_url', 'localhost'), container.name)
logger.debug('redirect: %s' % url_external)
return redirect(url_external)
elif report.reporttype == report.TP_SERVICE:
container = Container.get_reportcontainer(report, create = True)
container.docker_start()
url_external = "%s/notebook/%s/report" % (KOOPLEX.get('base_url', 'localhost'), container.name)
msg = "Report %s is started. API is at %s" % (report.name, url_external)
logger.info(msg)
messages.info(request, msg)
return redirect('report:list')
messages.error(request, "Rendering report type %s is not implemeted yet" % report.reporttype)
return redirect('report:list')
def _removeroute_report(report):
proxyconf = KOOPLEX.get('proxy', {})
reportconf = KOOPLEX.get('reportserver', {})
target_url = reportconf.get('base_url', 'localhost')
kw = {
'url': os.path.join(proxyconf.get('base_url','localhost'), 'api', 'routes', report.proxy_path),
'headers': {'Authorization': 'token %s' % proxyconf.get('auth_token', '') },
}
logging.debug("- %s -/-> %s" % (kw['url'], target_url))
return keeptrying(requests.delete, 5, **kw)
def openreport(request, report_id):
"""Renders new report list."""
user = request.user
logger.debug("user %s, method: %s" % (user, request.method))
try:
report = Report.objects.get(id=report_id)
except Exception as e:
logger.warning('Cannot resolve report id: %s -- %s' % (report_id, e))
return redirect('indexpage')
if report.reporttype == report.TP_STATIC:
url_external = report.url_external
logger.debug('redirect: %s' % url_external)
return redirect(url_external)
elif report.reporttype == report.TP_SHINY:
url_external = report.url_external
logger.debug('redirect: %s' % url_external)
return redirect(url_external)
elif report.reporttype == report.TP_DYNAMIC:
container = Container.get_reportcontainer(report, create=True)
container.docker_start()
url_external = "%s/notebook/%s/notebooks/%s?token=%s" % (KOOPLEX.get(
'base_url',
'localhost'), container.name, report.index, user.profile.token)
logger.debug('redirect: %s ' % url_external)
return redirect(url_external)
elif report.reporttype == report.TP_BOKEH:
container = Container.get_reportcontainer(report, create=True)
container.docker_start()
url_external = "%s/notebook/%s/report" % (KOOPLEX.get(
'base_url', 'localhost'), container.name)
logger.debug('redirect: %s' % url_external)
return redirect(url_external)
elif report.reporttype == report.TP_DASH:
container = Container.get_reportcontainer(report, create=True)
container.docker_start()
url_external = "%s/notebook/%s/report" % (KOOPLEX.get(
'base_url', 'localhost'), container.name)
logger.debug('redirect: %s' % url_external)
return redirect(url_external)
elif report.reporttype == report.TP_SERVICE:
container = Container.get_reportcontainer(report, create=True)
container.docker_start()
url_external = "%s/notebook/%s/report/help" % (KOOPLEX.get(
'base_url', 'localhost'), container.name)
msg = "Report %s is started. API is at %s" % (report.name,
url_external)
logger.info(msg)
messages.info(request, msg)
return redirect('report:list')
messages.error(
request,
"Rendering report type %s is not implemeted yet" % report.reporttype)
return redirect('report:list')
def _removeroute_report(report):
proxyconf = KOOPLEX.get('proxy', {})
reportconf = KOOPLEX.get('reportserver', {})
target_url = reportconf.get('base_url', 'localhost')
kw = {
'url':
os.path.join(proxyconf.get('base_url', 'localhost'), 'api', 'routes',
report.proxy_path),
'headers': {
'Authorization': 'token %s' % proxyconf.get('auth_token', '')
},
}
logging.debug("- %s -/-> %s" % (kw['url'], target_url))
return keeptrying(requests.delete, 5, **kw)
def getroutes():
proxyconf = KOOPLEX.get('proxy', {})
kw = {
'url': os.path.join(proxyconf.get('base_url','localhost'), 'api', 'routes'),
'headers': {'Authorization': 'token %s' % proxyconf.get('auth_token', '') },
}
return keeptrying(requests.get, 50, **kw)
def create_user_profile(sender, instance, created, **kwargs):
if created:
logger.info("New user %s" % instance)
last_uid = Profile.objects.all().aggregate(models.Max('userid'))['userid__max']
uid = KOOPLEX.get('min_userid', 1000) if last_uid is None else last_uid + 1
token = pwgen.pwgen(64)
Profile.objects.create(user = instance, userid = uid, token = token)
def _removeroute_container(container):
proxyconf = KOOPLEX.get('proxy', {})
kw = {
'url': os.path.join(proxyconf.get('base_url','localhost'), 'api', 'routes', container.proxy_path),
'headers': {'Authorization': 'token %s' % proxyconf.get('auth_token', '') },
}
logging.debug("- %s -/-> %s" % (kw['url'], container.url))
return keeptrying(requests.delete, 5, **kw)
def jupyter_session(container):
"""
"""
info = { 'containername': container.name }
kw = {
'url': os.path.join(KOOPLEX.get('spawner', {}).get('pattern_jupyterapi') % info, 'sessions'),
'headers': {'Authorization': 'token %s' % container.report.password, },
}
return keeptrying(requests.get, 50, **kw)
def get_redirect_uri(self, state = None):
"""Build redirect with redirect_state parameter."""
base_url = KOOPLEX.get('base_url', 'http://localhost')
url = os.path.join(base_url, "hub/oauth/complete/google-oauth2/")
if self.REDIRECT_STATE and state:
uri = url_add_parameters(url, {'redirect_state': state})
else:
uri = url
return uri
def create_user_profile(sender, instance, created, **kwargs):
if created:
logger.info("New user %s" % instance)
last_uid = Profile.objects.all().aggregate(
models.Max('userid'))['userid__max']
uid = KOOPLEX.get('min_userid',
1000) if last_uid is None else last_uid + 1
token = pwgen.pwgen(64)
Profile.objects.create(user=instance, userid=uid, token=token)
def jupyter_session(container):
"""
"""
info = { 'containername': container.name }
kw = {
'url': os.path.join(KOOPLEX.get('spawner', {}).get('pattern_jupyterapi') % info, 'sessions'),
'headers': {'Authorization': 'token %s' % container.report.password, },
}
return keeptrying(requests.get, 50, **kw)
class HydraOpenID(OpenIdConnectAuth):
name = 'hydraoidc'
OIDC_ENDPOINT = KOOPLEX.get('hydra_oidc_endpoint',
'https://localhost:4444')
def get_redirect_uri(self, state=None):
"""Build redirect with redirect_state parameter."""
base_url = KOOPLEX.get('base_url', 'http://localhost')
url = os.path.join(base_url, "hub/oauth/complete/hydraoidc/")
logger.debug("IDP resp")
if self.REDIRECT_STATE and state:
uri = url_add_parameters(url, {'redirect_state': state})
else:
uri = url
return uri
def get_user_details(self, response):
try:
return {#FIXME: a hydra most furan tolja az attributumokat!
'username': response['idp_user'],
'email': response['mail'][0],
'fullname': response['displayName'][0],
'first_name': response['givenName'][0],
'last_name': response['sn'][0],
}
except:
1 == 1
return { #FIXME: a hydra most furan tolja az attributumokat!
'username': response['idp_user'],
'email': response['mail'], #[0],
'fullname': response['displayName'], #[0],
'first_name': response['displayName'], #[0],
'last_name': response['displayName'], #[0],
}
def authenticate(self, request, **credentials):
user = super(HydraOpenID, self).authenticate(request, **credentials)
response = credentials.get('response', {})
logger.debug("IDP resp %s" % response)
try:
logger.debug("Authenticated (username) %s" % user.username)
except:
pass
# currently held courses
a = response.get('niifEduPersonHeldCourse', {})
coursecodes = CourseCode.parse(a)
UserCourseCodeBinding.userattributes(user,
coursecodes,
is_teacher=True)
# currently attended courses
a = response.get('niifEduPersonAttendedCourse', {})
coursecodes = CourseCode.parse(a)
UserCourseCodeBinding.userattributes(user,
coursecodes,
is_teacher=False)
return user
def _addroute_report(report):
proxyconf = KOOPLEX.get('proxy', {})
reportconf = KOOPLEX.get('reportserver', {})
target_url = reportconf.get('base_url', 'localhost')
kw = {
'url': os.path.join(proxyconf.get('base_url','localhost'), 'api', 'routes', 'report', report.proxy_path),
'headers': {'Authorization': 'token %s' % proxyconf.get('auth_token', '') },
'data': json.dumps({ 'target': target_url }),
}
logging.debug("+ %s ---> %s" % (kw['url'], target_url))
keeptrying(requests.post, 50, **kw)
kw = {
'url': os.path.join(proxyconf.get('base_url','localhost'), 'api', 'routes', 'report', report.proxy_path_latest),
'headers': {'Authorization': 'token %s' % proxyconf.get('auth_token', '') },
'data': json.dumps({ 'target': target_url }),
}
logging.debug("+ %s ---> %s" % (kw['url'], target_url))
return keeptrying(requests.post, 50, **kw)
def getroutes():
proxyconf = KOOPLEX.get('proxy', {})
kw = {
'url':
os.path.join(proxyconf.get('base_url', 'localhost'), 'api', 'routes'),
'headers': {
'Authorization': 'token %s' % proxyconf.get('auth_token', '')
},
}
return keeptrying(requests.get, 50, **kw)
def list_imagenames(self):
logger.debug("Listing image names")
pattern_imagenamefilter = KOOPLEX.get('docker', {}).get('pattern_imagename_filter', r'^image-%(\w+):\w$')
for image in self.client.images(all = True):
if image['RepoTags'] is None:
continue
for tag in image['RepoTags']:
if re.match(pattern_imagenamefilter, tag):
_, imagename, _ = re.split(pattern_imagenamefilter, tag)
logger.debug("Found image: %s" % imagename)
yield imagename
def list_imagenames(self):
logger.debug("Listing image names")
pattern_imagenamefilter = KOOPLEX.get('docker', {}).get(
'pattern_imagename_filter', r'^image-%(\w+):\w$')
for image in self.client.images(all=True):
if image['RepoTags'] is None:
continue
for tag in image['RepoTags']:
if re.match(pattern_imagenamefilter, tag):
_, imagename, _ = re.split(pattern_imagenamefilter, tag)
logger.debug("Found image: %s" % imagename)
yield imagename
def _removeroute_container(container):
proxyconf = KOOPLEX.get('proxy', {})
kw = {
'url':
os.path.join(proxyconf.get('base_url', 'localhost'), 'api', 'routes',
container.proxy_path),
'headers': {
'Authorization': 'token %s' % proxyconf.get('auth_token', '')
},
}
logging.debug("- %s -/-> %s" % (kw['url'], container.url))
return keeptrying(requests.delete, 5, **kw)
def droproutes():
proxyconf = KOOPLEX.get('proxy', {})
resp = getroutes()
routes = json.loads(resp.content.decode())
for r, v in routes.items():
kw = {
'url': os.path.join(proxyconf.get('base_url','localhost'), 'api', 'routes', r[1:]),
'headers': {'Authorization': 'token %s' % proxyconf.get('auth_token', '') },
}
logging.debug("- %s -/-> %s" % (kw['url'], v['target']))
resp_latest = keeptrying(requests.delete, 5, **kw)
return resp_latest
def environment(self):
envs = {
'NB_USER': self.user.username,
'NB_UID': self.user.profile.userid,
'NB_GID': self.user.profile.groupid,
'NB_URL': self.proxy_path,
'NB_PORT': KOOPLEX.get('spawner', {}).get('port', 8000),
'NB_HOST': KOOPLEX.get('base_url', 'localhost'),
'NB_TOKEN': self.user.profile.token,
'CONTAINER_NAME': self.name,
}
for ce in ContainerEnvironment.objects.filter(container = self):
envs[ce.name] = ce.value
logger.debug("Adding extra envs: %s"%ce.value)
report = self.report
if report:
envs['REPORT_TYPE'] = report.reporttype
envs['REPORT_DIR'] = os.path.join('/home/', 'report', report.cleanname, report.tag_name)
envs['REPORT_ABS'] = os.path.join('notebook/', 'report-%s-%s'%(self.user.username, report.cleanname))
envs['REPORT_PORT'] = 9000
envs['REPORT_INDEX'] = report.index
return envs
def _addroute_container(container, test=False):
proxyconf = KOOPLEX.get('proxy', {})
if test:
kw = {
'url':
os.path.join(proxyconf.get('base_url', 'localhost'), 'api',
'routes', container.proxy_path_test),
'headers': {
'Authorization': 'token %s' % proxyconf.get('auth_token', '')
},
'data':
json.dumps({'target': container.url_test}),
}
logging.debug("+ %s ---> %s test port" %
(kw['url'], container.url_test))
else:
kw = {
'url':
os.path.join(proxyconf.get('base_url', 'localhost'), 'api',
'routes', container.proxy_path),
'headers': {
'Authorization': 'token %s' % proxyconf.get('auth_token', '')
},
'data':
json.dumps({'target': container.url}),
}
logging.debug("+ %s ---> %s proxy path" % (kw['url'], container.url))
try:
rc = ReportContainerBinding.objects.get(container=container)
report = rc.report
kw = {
'url':
os.path.join(proxyconf.get('base_url', 'localhost'), 'api',
'routes', 'notebook', report.proxy_path_latest),
'headers': {
'Authorization':
'token %s' % proxyconf.get('auth_token', '')
},
'data':
json.dumps({'target': container.url_test}),
}
logging.debug("+ %s ---> %s report proxy path latest" %
(kw['url'], container.url))
keeptrying(requests.post, 50, **kw)
except:
logging.debug("Container is not for report")
logging.debug("+ %s ---> %s" % (kw['url'], container.url))
return keeptrying(requests.post, 50, **kw)
def droproutes():
proxyconf = KOOPLEX.get('proxy', {})
resp = getroutes()
routes = json.loads(resp.content.decode())
for r, v in routes.items():
kw = {
'url':
os.path.join(proxyconf.get('base_url', 'localhost'), 'api',
'routes', r[1:]),
'headers': {
'Authorization': 'token %s' % proxyconf.get('auth_token', '')
},
}
logging.debug("- %s -/-> %s" % (kw['url'], v['target']))
resp_latest = keeptrying(requests.delete, 5, **kw)
def _addroute_container(container, test=False):
proxyconf = KOOPLEX.get('proxy', {})
if test:
kw = {
'url': os.path.join(proxyconf.get('base_url','localhost'), 'api', 'routes', container.proxy_path_test),
'headers': {'Authorization': 'token %s' % proxyconf.get('auth_token', '') },
'data': json.dumps({ 'target': container.url_test }),
}
else:
kw = {
'url': os.path.join(proxyconf.get('base_url','localhost'), 'api', 'routes', container.proxy_path),
'headers': {'Authorization': 'token %s' % proxyconf.get('auth_token', '') },
'data': json.dumps({ 'target': container.url }),
}
logging.debug("+ %s ---> %s" % (kw['url'], container.url))
return keeptrying(requests.post, 50, **kw)
def __init__(self):
logger.debug("init")
ldapconf = KOOPLEX.get('ldap', {})
self.host = ldapconf.get('host', 'localhost')
self.port = ldapconf.get('port', 389)
try:
self.base_dn = ldapconf['base_dn']
self.bind_dn = ldapconf['bind_dn']
self.bind_pw = ldapconf['bind_password']
except KeyError as e:
logger.error("Cannot initialize ldap, KOOPLEX['ldap'][key] key is missing -- %s" % e)
raise
server = ldap3.Server(host = self.host, port = self.port)
self.connection = ldap3.Connection(server, self.bind_dn, self.bind_pw)
success = self.connection.bind()
if not success:
logger.error("Cannot bind to ldap server")
raise LdapException("Cannot bind to ldap server")
def get_reportcontainer(report, create):
logger.debug("report %s" % (report))
try:
return ReportContainerBinding.objects.get(report = report).container
except ReportContainerBinding.DoesNotExist:
logger.debug("ReportContainer for report %s does not exist" % report)
if create:
container_name_dict = {
'un': report.creator.username,
'rn': report.cleanname,
'tn': report.tag_name,
# 'ts': report.ts_human.replace(':', '').replace('_', '')
}
containername = KOOPLEX.get('pattern_report_containername','report-%(un)s-%(rn)s-%(tn)s') % container_name_dict
container = Container.objects.create(name = containername, user = report.creator, image=report.image)
ReportContainerBinding.objects.create(report = report, container = container)
logger.debug("new container in db %s" % container)
return container
raise Container.DoesNotExist
def __init__(self):
logger.debug("init")
ldapconf = KOOPLEX.get('ldap', {})
self.host = ldapconf.get('host', 'localhost')
self.port = ldapconf.get('port', 389)
try:
self.base_dn = ldapconf['base_dn']
self.bind_dn = ldapconf['bind_dn']
self.bind_pw = ldapconf['bind_password']
except KeyError as e:
logger.error(
"Cannot initialize ldap, KOOPLEX['ldap'][key] key is missing -- %s"
% e)
raise
server = ldap3.Server(host=self.host, port=self.port)
self.connection = ldap3.Connection(server, self.bind_dn, self.bind_pw)
success = self.connection.bind()
if not success:
logger.error("Cannot bind to ldap server")
raise LdapException("Cannot bind to ldap server")
def groupid(self):
return KOOPLEX.get('ldap', {}).get('gid_users', 1000)
class Volume(models.Model):
pattern = KOOPLEX.get('volumepattern', {})
HOME = 'home'
GARBAGE = 'garbage'
GIT = 'git'
FILESYNC = 'filesync'
SHARE = 'share'
WORKDIR = 'workdir'
FUNCTIONAL = 'functional'
STORAGE = 'storage'
PRIVATE = 'private'
COURSE_SHARE = 'course'
COURSE_WORKDIR = 'usercourse'
COURSE_ASSIGNMENTDIR = 'assignment'
REPORT = 'report'
VOLUME_TYPE_LIST = [
HOME, GARBAGE, GIT, FILESYNC, SHARE, WORKDIR, FUNCTIONAL, STORAGE,
COURSE_SHARE, COURSE_WORKDIR, COURSE_ASSIGNMENTDIR, REPORT, PRIVATE
]
VOLUME_TYPE_LIST_USER = [FUNCTIONAL] #, STORAGE, PRIVATE ]
name = models.CharField(max_length=64, unique=True)
displayname = models.CharField(max_length=64)
description = models.TextField(null=True)
volumetype = models.CharField(max_length=16,
choices=[(x, TYPE_LOOKUP[x])
for x in VOLUME_TYPE_LIST])
is_present = models.BooleanField(default=True)
def __str__(self):
return self.displayname
@staticmethod
def try_create(volumename):
for x in Volume.VOLUME_TYPE_LIST:
pattern = Volume.pattern.get(x, r'^(%s)$' % x)
if re.match(pattern, volumename):
_, dirname, _ = re.split(pattern, volumename)
return Volume.objects.create(name=volumename,
displayname=dirname,
description='%s (%s)' %
(TYPE_LOOKUP[x], dirname),
volumetype=x)
# @staticmethod
# def lookup(volumetype, **kw): #FIXME: check if still used somewhere
# if not volumetype in Volume.VOLUME_TYPE_LIST:
# raise Volume.DoesNotExist
# return Volume.objects.get(volumetype = volumetype['tag'], **kw)
@staticmethod
def filter(volumetype, **kw):
if not volumetype in Volume.VOLUME_TYPE_LIST:
raise Volume.DoesNotExist
user = kw.pop('user', None)
if user:
logger.error("NotImplementedError") #FIXME
pass
for volume in Volume.objects.filter(volumetype=volumetype, **kw):
yield volume
# def is_volumetype(self, volumetype):
# try:
# return self.volumetype == volumetype['tag']
# except KeyError:
# return False
def mode(self, user):
if self.volumetype == Volume.FUNCTIONAL:
for binding in VolumeOwnerBinding.objects.filter(volume=self):
if binding.owner == user:
return 'rw'
return 'ro'
if self.volumetype == Volume.STORAGE:
if hasattr(self, 'extrafields'):
return 'rw' if self.extrafields.readwrite else 'ro'
else:
return 'ro'
return 'rw'
@property
def mountpoint(self):
pattern = self.pattern.get(self.volumetype,
r'^(%s)$' % self.volumetype)
_, dirname, _ = re.split(pattern, self.name)
if self.volumetype == Volume.FUNCTIONAL:
return os.path.join('/vol', dirname)
if self.volumetype == Volume.STORAGE:
return os.path.join('/data', dirname)
return os.path.join('/mnt/.volumes', dirname)
class Dirname:
mountpoint = KOOPLEX.get('mountpoint', {})
@staticmethod
def userhome(user):
return os.path.join(Dirname.mountpoint['home'], user.username)
@staticmethod
def usergarbage(user):
return os.path.join(Dirname.mountpoint['garbage'], user.username)
@staticmethod
def reportroot(user):
return os.path.join(Dirname.mountpoint['report'], user.username)
@staticmethod
def reportprepare(user):
return os.path.join(Dirname.reportroot(user), '_prepare')
@staticmethod
def report(report):
return os.path.join(Dirname.reportroot(report.creator),
standardize_str(report.name))
@staticmethod
def report_with_tag(report):
return os.path.join(Dirname.reportroot(report.creator),
standardize_str(report.name), report.tag_name)
@staticmethod
def share(userprojectbinding):
return os.path.join(Dirname.mountpoint['share'],
userprojectbinding.project.uniquename)
@staticmethod
def workdir(userprojectbinding):
return os.path.join(Dirname.mountpoint['workdir'],
userprojectbinding.uniquename)
@staticmethod
def vcpcache(vcproject):
return os.path.join(Dirname.mountpoint['git'], vcproject.clone_folder)
@staticmethod
def fscache(fslibrary):
return os.path.join(Dirname.mountpoint['filesync'],
fslibrary.sync_folder)
@staticmethod
def course(course):
return os.path.join(Dirname.mountpoint['course'], course.folder)
@staticmethod
def courseprivate(course):
return os.path.join(Dirname.course(course), 'private')
@staticmethod
def coursepublic(course):
return os.path.join(Dirname.course(course), 'public')
@staticmethod
def courseworkdir(usercoursebinding):
return os.path.join(Dirname.mountpoint['usercourse'],
usercoursebinding.course.folder)
@staticmethod
def usercourseworkdir(usercoursebinding):
return os.path.join(Dirname.courseworkdir(usercoursebinding),
usercoursebinding.user.username)
@staticmethod
def assignmentsource(assignment):
return os.path.join(
Dirname.courseprivate(assignment.coursecode.course),
assignment.folder)
@staticmethod
def assignmentworkdir(userassignmentbinding):
from hub.models import UserCourseBinding
usercoursebinding = UserCourseBinding.objects.get(
user=userassignmentbinding.user,
course=userassignmentbinding.assignment.coursecode.course)
wd = Dirname.usercourseworkdir(usercoursebinding)
return os.path.join(wd, userassignmentbinding.assignment.safename)
@staticmethod
def assignmentcorrectdir(userassignmentbinding):
assignment = userassignmentbinding.assignment
user = userassignmentbinding.user
namefield = "%s%s_%s" % (deaccent_str(user.first_name).replace(
' ', ''), deaccent_str(user.last_name).replace(' ',
''), user.username)
datefield = userassignmentbinding.submitted_at.strftime('%Y_%m_%d')
return os.path.join(
Dirname.mountpoint['assignment'],
assignment.coursecode.course.folder,
'feedback-%s-%s-%s' % (assignment.safename, namefield, datefield))
@staticmethod
def containervolume_listfolders(container, volume):
from hub.models import UserCourseBinding, UserAssignmentBinding
def get_usercoursebinding_userstate():
try:
usercoursebinding = UserCourseBinding.objects.get(
user=container.user, course=container.course)
except UserCourseBinding.DoesNotExist:
logger.error(
"Silly situation, cannot map %s %s COZ user course binding instance is missing"
% (volume, container))
return None, None
if container.course in container.user.profile.courses_taught():
return usercoursebinding, 'teacher'
elif container.course in container.user.profile.courses_attend():
return usercoursebinding, 'student'
else:
logger.error("Silly situation, cannot map %s %s" %
(volume, container))
return None, None
if volume.volumetype == volume.HOME:
yield Dirname.userhome(container.user)
elif volume.volumetype == volume.GARBAGE:
yield Dirname.usergarbage(container.user)
elif volume.volumetype == volume.SHARE:
for upb in container.userprojectbindings:
yield Dirname.share(upb)
elif volume.volumetype == volume.WORKDIR:
for upb in container.userprojectbindings:
yield Dirname.workdir(upb)
elif volume.volumetype == volume.GIT:
for vcppb in container.vcprojectprojectbindings:
yield Dirname.vcpcache(vcppb.vcproject)
elif volume.volumetype == volume.FILESYNC:
for fslpb in container.fslibraryprojectbindings:
yield Dirname.fscache(fslpb.fslibrary)
elif volume.volumetype == volume.COURSE_SHARE:
if container.course in container.user.profile.courses_taught():
yield Dirname.course(container.course)
elif container.course in container.user.profile.courses_attend():
yield Dirname.coursepublic(container.course)
else:
logger.error("Silly situation, cannot map %s %s" %
(volume, container))
elif volume.volumetype == volume.COURSE_WORKDIR and container.course:
usercoursebinding, userstatus = get_usercoursebinding_userstate()
if userstatus == 'teacher':
yield Dirname.courseworkdir(usercoursebinding)
elif userstatus == 'student':
yield Dirname.usercourseworkdir(usercoursebinding)
else:
yield "OOPS_%s" % volume.volumetype
elif volume.volumetype == volume.COURSE_ASSIGNMENTDIR and container.course:
_, userstatus = get_usercoursebinding_userstate()
if userstatus == 'teacher':
for binding in UserAssignmentBinding.objects.all():
if binding.state == UserAssignmentBinding.ST_QUEUED or binding.corrector is None or binding.assignment.coursecode.course != container.course:
continue
if binding.corrector != container.user:
continue
yield Dirname.assignmentcorrectdir(binding)
elif userstatus == 'student':
for binding in UserAssignmentBinding.objects.filter(
user=container.user):
if binding.state == UserAssignmentBinding.ST_QUEUED or binding.corrector is None or binding.assignment.coursecode.course != container.course:
continue
yield Dirname.assignmentcorrectdir(binding)
else:
yield "OOPS_%s" % volume.volumetype
elif volume.volumetype == volume.REPORT:
yield Dirname.reportroot(container.user)
else:
yield "MISSING_DIRNAME_%s" % volume.volumetype
class Docker:
dockerconf = KOOPLEX.get('docker', {})
def __init__(self):
base_url = self.dockerconf.get('base_url', '')
self.client = Client(base_url=base_url)
logger.debug("Client init")
self.check = None
def list_imagenames(self):
logger.debug("Listing image names")
pattern_imagenamefilter = KOOPLEX.get('docker', {}).get(
'pattern_imagename_filter', r'^image-%(\w+):\w$')
for image in self.client.images(all=True):
if image['RepoTags'] is None:
continue
for tag in image['RepoTags']:
if re.match(pattern_imagenamefilter, tag):
_, imagename, _ = re.split(pattern_imagenamefilter, tag)
logger.debug("Found image: %s" % imagename)
yield imagename
def list_volumenames(self):
logger.debug("Listing volume names")
volumes = self.client.volumes()
for volume in volumes['Volumes']:
yield volume['Name']
def create_volume(self, volume):
volume_dir = None #self.dockerconf.get('volume_dir', '')
if volume_dir:
self.client.create_volume(name=volume.name,
driver='local',
driver_opts={
'device':
'%s/%s/' % (volume_dir, volume.name),
'o':
'bind',
'type':
'none'
})
else:
self.client.create_volume(name=volume.name, )
logger.debug("Volume %s created" % volume.name)
return True #self.get_container(container)
def delete_volume(self, volume):
self.client.remove_volume(name=volume.name)
logger.debug("Volume %s deleted" % volume.name)
def get_container(self, container):
for item in self.client.containers(all=True):
# docker API prepends '/' in front of container names
if '/' + container.name in item['Names']:
logger.debug("Get container %s" % container.name)
return item
return None
def create_container(self, container):
volumes = [] # the list of mount points in the container
binds = {} # a mapping dictionary of the container mounts
for volume in container.volumes:
logger.debug("container %s, volume %s" % (container, volume))
mp = volume.mountpoint
volumes.append(mp)
binds[volume.name] = {
'bind': mp,
'mode': volume.mode(container.user)
}
logger.debug("container %s binds %s" % (container, binds))
host_config = self.client.create_host_config(
binds=binds,
privileged=True,
mem_limit='2g',
memswap_limit='170m',
mem_swappiness=0,
# oom_kill_disable = True,
cpu_shares=2,
)
network = self.dockerconf.get('network', 'host')
networking_config = {'EndpointsConfig': {network: {}}}
ports = self.dockerconf.get('container_ports', [8000, 9000])
imagename = container.image.imagename if container.image else self.dockerconf.get(
'default_image', 'basic')
args = {
'name': container.name,
'image': imagename,
'detach': True,
'hostname': container.name,
'host_config': host_config,
'networking_config': networking_config,
'environment': container.environment,
'volumes': volumes,
'ports': ports,
}
self.client.create_container(**args)
logger.debug("Container created")
self.managemount(container) #FIXME: check if not called twice
return self.get_container(container)
def _writefile(self, container_name, path, filename, content):
import tarfile
import time
from io import BytesIO
tarstream = BytesIO()
tar = tarfile.TarFile(fileobj=tarstream, mode='w')
tarinfo = tarfile.TarInfo(name=filename)
tarinfo.size = len(content)
tarinfo.mtime = time.time()
tar.addfile(tarinfo, BytesIO(content))
tar.close()
tarstream.seek(0)
try:
status = self.client.put_archive(container=container_name,
path=path,
data=tarstream)
logger.info("container %s put_archive %s/%s returns %s" %
(container_name, path, filename, status))
except Exception as e:
logger.error("container %s put_archive %s/%s fails -- %s" %
(container_name, path, filename, e))
def managemount(self, container):
from kooplex.lib.fs_dirname import Dirname
path, filename = os.path.split(
self.dockerconf.get('mountconf', '/tmp/mount.conf'))
mapper = []
for v in container.volumes:
mapper.extend([
"%s:%s" % (v.volumetype, d)
for d in Dirname.containervolume_listfolders(container, v)
])
#NOTE: mounter uses read to process the mapper configuration, thus we need to make sure '\n' terminates the config mapper file
mapper.append('')
logger.debug("container %s map %s" % (container, mapper))
file_data = "\n".join(mapper).encode('utf8')
self._writefile(container.name, path, filename, file_data)
def trigger_impersonator(self, vcproject): #FIXME: dont call it 1-by-1
from kooplex.lib.fs_dirname import Dirname
container_name = self.dockerconf.get('impersonator', 'impersonator')
path, filename = os.path.split(
self.dockerconf.get('gitcommandconf', '/tmp/gitcommand.conf'))
cmdmaps = []
token = vcproject.token
fn_clonesh = os.path.join(Dirname.vcpcache(vcproject), "clone.sh")
fn_key = os.path.join(Dirname.userhome(vcproject.token.user), '.ssh',
token.fn_rsa)
cmdmaps.append(
"%s:%s:%s:%s" %
(token.user.username, fn_key, token.repository.domain, fn_clonesh))
cmdmaps.append('')
file_data = "\n".join(cmdmaps).encode('utf8')
self._writefile(container_name, path, filename, file_data)
def run_container(self, container):
docker_container_info = self.get_container(container)
if docker_container_info is None:
logger.debug("Container did not exist, Creating new one")
docker_container_info = self.create_container(container)
container_state = docker_container_info['Status']
if container_state == 'Created' or container_state.startswith(
'Exited'):
logger.debug("Starting container")
self.start_container(container)
def refresh_container_state(self, container):
docker_container_info = self.get_container(container)
container_state = docker_container_info['State']
logger.debug("Container state %s" % container_state)
container.last_message = str(container_state)
container.last_message_at = now()
container.save()
def start_container(self, container):
self.client.start(container.name)
# we need to retrieve the container state after starting it
docker_container_info = self.get_container(container)
container_state = docker_container_info['State']
logger.debug("Container state %s" % container_state)
container.last_message = str(container_state)
container.last_message_at = now()
assert container_state == 'running', "Container failed to start: %s" % docker_container_info
def stop_container(self, container):
try:
self.client.stop(container.name)
container.last_message = 'Container stopped'
except Exception as e:
logger.warn("docker container not found by API -- %s" % e)
container.last_message = str(e)
def remove_container(self, container):
try:
self.client.remove_container(container.name)
container.last_message = 'Container removed'
container.last_message_at = now()
except Exception as e:
logger.warn("docker container not found by API -- %s" % e)
container.last_message = str(e)
container.last_message_at = now()
logger.debug("Container removed %s" % container.name)
#FIXME: az execute2 lesz az igazi...
def execute(self, container, command):
logger.info("execution: %s in %s" % (command, container))
execution = self.client.exec_create(container=container.name,
cmd=shlex.split(command))
return self.client.exec_start(execution, detach=False)
def execute2(self, container, command):
logger.info("execution: %s in %s" % (command, container))
execution = self.client.exec_create(container=container.name,
cmd=shlex.split(command))
response = self.client.exec_start(exec_id=execution['Id'],
stream=False)
check = self.client.exec_inspect(exec_id=execution['Id'])
self.check = check
if check['ExitCode'] != 0:
logger.error('Execution %s in %s failed -- %s' %
(command, container, check))
return response.decode()
"""
@author: David Visontai, Jozsef Steger
"""
import os
import json
import requests
import logging
from kooplex.settings import KOOPLEX
from kooplex.lib import keeptrying, standardize_str
logger = logging.getLogger(__name__)
reportconf = KOOPLEX.get('reportserver', {})
def removeroute(instance):
if isinstance(instance, Container):
return _removeroute_container(instance)
elif isinstance(instance, Report):
return _removeroute_report(instance)
logger.error('Not implemented')
def add_report_nginx_api(report):
# import htpasswd
str_name = standardize_str(report.proxy_path)
conf_text = """
location /report/%s {
auth_basic "username is 'report' ";
auth_basic_user_file /etc/passwords/%s;
class Filename:
mountpoint = KOOPLEX.get('mountpoint', {})
@staticmethod
def userhome_garbage(user):
return os.path.join(Dirname.mountpoint['garbage'],
"user-%s.%f.tar.gz" % (user.username, time.time()))
@staticmethod
def share_garbage(userprojectbinding):
return os.path.join(
Dirname.mountpoint['garbage'], "projectshare-%s.%f.tar.gz" %
(userprojectbinding.project.uniquename, time.time()))
@staticmethod
def workdir_archive(userprojectbinding):
return os.path.join(
Dirname.mountpoint['home'], userprojectbinding.user.username,
"garbage", "workdir-%s.%f.tar.gz" %
(userprojectbinding.uniquename, time.time()))
@staticmethod
def vcpcache_archive(vcproject):
return os.path.join(
Dirname.mountpoint['home'], vcproject.token.user.username,
"garbage",
"git-%s.%f.tar.gz" % (vcproject.uniquename, time.time()))
@staticmethod
def course_garbage(course):
return os.path.join(
Dirname.mountpoint['garbage'],
"course-%s.%f.tar.gz" % (course.folder, time.time()))
@staticmethod
def courseworkdir_archive(usercoursebinding):
return os.path.join(
Dirname.mountpoint['home'], usercoursebinding.user.username,
"garbage",
"%s.%f.tar.gz" % (usercoursebinding.course.folder, time.time()))
@staticmethod
def assignmentsnapshot(assignment):
return os.path.join(
Dirname.mountpoint['assignment'],
assignment.coursecode.course.folder,
'assignmentsnapshot-%s.%d.tar.gz' %
(assignment.safename, assignment.created_at.timestamp()))
@staticmethod
def assignmentsnapshot_garbage(assignment):
return os.path.join(
Dirname.mountpoint['garbage'],
'assignmentsnapshot-%s-%s-%s-%f.tar.gz' %
(assignment.coursecode.course.folder, assignment.safename,
assignment.created_at.timestamp(), time.time()))
@staticmethod
def assignmentcollection(userassignmentbinding):
assignment = userassignmentbinding.assignment
return os.path.join(
Dirname.mountpoint['assignment'],
assignment.coursecode.course.folder, 'submitted-%s-%s.%d.tar.gz' %
(assignment.safename, userassignmentbinding.user.username,
userassignmentbinding.submitted_at.timestamp()))
@staticmethod
def report_garbage(report):
return os.path.join(
Dirname.mountpoint['garbage'], report.creator.username,
"report-%s-%s.%f.tar.gz" %
(report.name, report.ts_human, time.time()))
def url_test(self):
return "http://%s:%d" % (self.name, KOOPLEX.get('spawner', {}).get('port_test', 9000))
def url(self):
return "http://%s:%d" % (self.name, KOOPLEX.get('spawner', {}).get('port', 8000))
def groupid(self):
return KOOPLEX.get('ldap', {}).get('gid_users', 1000)
def imagename(self):
return KOOPLEX.get('docker', {}).get('pattern_imagename',
'image-%(imagename)s') % {
'imagename': self.name
}
