def __check_expr_status(self, user, hackathon, template):
"""
check experiment status, if there are pre-allocate experiments, the experiment will be assigned directly
:param user:
:param hackathon:
:param template:
:return:
"""
criterion = Q(status__in=[ExprStatus.RUNNING, ExprStatus.STARTING], hackathon=hackathon, user=user)
is_admin = self.admin_manager.is_hackathon_admin(hackathon.id, user.id)
if is_admin:
criterion &= Q(template=template)
expr = Experiment.objects(criterion).first()
if expr:
# user has a running/starting experiment
return expr
# try to assign pre-configured expr to user
expr = Experiment.objects(status=ExprStatus.RUNNING, hackathon=hackathon, template=template, user=None).first()
if expr:
expr.user = user
expr.save()
return expr
def stop_expr(self, expr_id):
"""
:param expr_id: experiment id
:return:
"""
self.log.debug("begin to stop %s" % str(expr_id))
expr = Experiment.objects(id=expr_id).first()
if expr is not None:
starter = self.get_starter(expr.hackathon, expr.template)
if starter:
starter.stop_expr(Context(experiment_id=expr.id, experiment=expr))
self.log.debug("experiment %s ended success" % expr_id)
return ok('OK')
else:
return ok()
def _internal_stop_expr(self, context):
expr = Experiment.objects(id=context.experiment_id).first()
if not expr:
return
if len(expr.virtual_environments) == 0:
expr.status = ExprStatus.ROLL_BACKED
expr.save()
return
# delete containers and change expr status
for ve in expr.virtual_environments:
context = context.copy() # create new context for every virtual_environment
context.virtual_environment_name = ve.name
self._stop_virtual_environment(ve, expr, context)
def roll_back(self, expr_id):
"""
roll back when exception occurred
:param expr_id: experiment id
"""
self.log.debug("Starting rollback experiment %s..." % expr_id)
expr = Experiment.objects(id=expr_id)
if not expr:
self.log.warn("rollback failed due to experiment not found")
return
starter = self.get_starter(expr.hackathon, expr.template)
if not starter:
self.log.warn("rollback failed due to no starter found")
return
return starter.rollback(Context(experiment=expr))
def delete_template(self, template_id):
self.log.debug("delete template [%s]" % template_id)
try:
template = self.get_template_info_by_id(template_id)
if template is None:
return ok("already removed")
# user can only delete the template which created by himself except super admin
if g.user.id != template.creator.id and not g.user.is_super:
return forbidden()
if Experiment.objects(template=template).count() > 0:
return forbidden("template already in use")
# remove record in DB
# the Hackathon used this template will imply the mongoengine's PULL reverse_delete_rule
self.log.debug("delete template {}".format(template.name))
template.delete()
return ok("delete template success")
except Exception as ex:
self.log.error(ex)
return internal_server_error("delete template failed")
def scheduler_recycle_expr(self):
"""recycle experiment according to hackathon basic info on recycle configuration
According to the hackathon's basic info on 'recycle_enabled', find out time out experiments
Then call function to recycle them
:return:
"""
self.log.debug("start checking recyclable experiment ... ")
for hackathon in self.hackathon_manager.get_recyclable_hackathon_list():
try:
# check recycle enabled
mins = self.hackathon_manager.get_recycle_minutes(hackathon)
# filter out the experiments that need to be recycled
exprs = Experiment.objects(create_time__lt=self.util.get_now() - timedelta(minutes=mins),
status=ExprStatus.RUNNING,
hackathon=hackathon)
for expr in exprs:
self.__recycle_expr(expr)
except Exception as e:
self.log.error(e)
def _enable_guacd_file_transfer(self, context):
"""
This function should be invoked after container is started in hosted_docker.py
:param ve: virtual environment
"""
expr = Experiment.objects(id=context.experiment_id).no_dereference().first()
virtual_env = expr.virtual_environments.get(name=context.virtual_environment_name)
remote = virtual_env.remote_paras
p = pexpect.spawn("scp -P %s %s %s@%s:/usr/local/sbin/guacctl" %
(remote["port"],
abspath("%s/../expr/guacctl" % dirname(realpath(__file__))),
remote["username"],
remote["hostname"]))
i = p.expect([pexpect.TIMEOUT, 'yes/no', 'password: '******'password:'])
if i != 0:
p.sendline(remote["password"])
p.expect(pexpect.EOF)
p.close()
def __start_virtual_environment(self, context, docker_template_unit):
origin_name = docker_template_unit.get_name()
prefix = str(context.experiment_id)[0:9]
suffix = "".join(random.sample(string.ascii_letters + string.digits, 8))
new_name = '%s-%s-%s' % (prefix, origin_name, suffix.lower())
docker_template_unit.set_name(new_name)
self.log.debug("starting to start container: %s" % new_name)
# db document for VirtualEnvironment
ve = VirtualEnvironment(provider=VirtualEnvProvider.DOCKER,
name=new_name,
image=docker_template_unit.get_image_with_tag(),
status=ExprEnvStatus.INIT,
remote_provider=VERemoteProvider.Guacamole)
# create a new context for current ve only
context = context.copy()
experiment = Experiment.objects(id=context.experiment_id).no_dereference().only("virtual_environments").first()
experiment.virtual_environments.append(ve)
experiment.save()
# start container remotely , use hosted docker
context.virtual_environment_name = ve.name
context.unit = docker_template_unit
self._internal_start_virtual_environment(context)
def get_expr_status_and_confirm_starting(self, expr_id):
expr = Experiment.objects(id=expr_id).first()
if expr:
return self.__report_expr_status(expr, isToConfirmExprStarting=True)
else:
return not_found('Experiment Not found')