class DockerHelper(Component):
hosted_docker = RequiredFeature("hosted_docker")
alauda_docker = RequiredFeature("alauda_docker")
def get_docker(self, hackathon):
if hackathon.is_alauda_enabled():
return self.alauda_docker
else:
return self.hosted_docker
def get_local_pem_url(self, pem_url):
local_pem_url = self.CERT_BASE + "/" + pem_url.split("/")[-1]
if not isfile(local_pem_url):
self.log.debug("Recover local %s.pem file from azure storage %s" % (local_pem_url, pem_url))
cryptor = RequiredFeature("cryptor")
cryptor.recover_local_file(pem_url, local_pem_url)
return local_pem_url
def run_job(mdl_cls_func, cls_args, func_args, second=DEFAULT_TICK):
exec_time = get_now() + timedelta(seconds=second)
scheduler = RequiredFeature("scheduler")
scheduler.get_scheduler().add_job(call,
'date',
run_date=exec_time,
args=[mdl_cls_func, cls_args, func_args])
def upload_files(self, user_id, file_type):
"""Handle uploaded files from http request"""
try:
self.__validate_upload_files()
except Exception as e:
self.log.error(e)
return bad_request("file size or file type unsupport")
file_list = []
storage = RequiredFeature("storage")
for file in request.files:
file_content = request.files[file]
pre_file_name = file_content.filename
file_suffix = pre_file_name[pre_file_name.rfind('.'):]
new_file_name = self.__generate_file_name(user_id, file_type,
file_suffix)
self.log.debug("upload file: " + new_file_name)
context = Context(file_name=new_file_name,
file_type=file_type,
content=file_content)
context = storage.save(context)
# file_name is a random name created by server, pre_file_name is the original name
file_info = {
"file_name": new_file_name,
"pre_file_name": pre_file_name,
"url": context.url
}
file_list.append(file_info)
return {"files": file_list}
def get_starter(self, hackathon, template):
# load expr starter
starter = None
if not hackathon or not template:
return starter
# TODO Interim workaround for kubernetes, need real implementation
if hackathon.config.get('cloud_provider') == CloudProvider.KUBERNETES:
return RequiredFeature("k8s_service")
if template.provider == VirtualEnvProvider.DOCKER:
raise NotImplementedError()
elif template.provider == VirtualEnvProvider.AZURE:
raise NotImplementedError()
elif template.provider == VirtualEnvProvider.K8S:
starter = RequiredFeature("k8s_service")
return starter
def get_starter(self, hackathon, template):
# load expr starter
starter = None
if not hackathon or not template:
return starter
if template.provider == VE_PROVIDER.DOCKER:
if HACKATHON_CONFIG.CLOUD_PROVIDER in hackathon.config:
if hackathon.config[HACKATHON_CONFIG.
CLOUD_PROVIDER] == CLOUD_PROVIDER.AZURE:
starter = RequiredFeature("azure_docker")
elif hackathon.config[HACKATHON_CONFIG.
CLOUD_PROVIDER] == CLOUD_PROVIDER.ALAUDA:
starter = RequiredFeature("alauda_docker")
elif template.provider == VE_PROVIDER.AZURE:
starter = RequiredFeature("azure_vm")
return starter
def __init__(self):
self.__containers = {
FILE_TYPE.TEMPLATE: self.util.safe_get_config("storage.azure.template_container", "templates"),
FILE_TYPE.HACK_IMAGE: self.util.safe_get_config("storage.azure.image_container", "images"),
FILE_TYPE.AZURE_CERT: self.util.safe_get_config("storage.azure.certificate_container", "certificate"),
FILE_TYPE.USER_FILE: self.util.safe_get_config("storage.azure.user_file_container", "userfile"),
FILE_TYPE.TEAM_FILE: self.util.safe_get_config("storage.azure.team_file_container", "teamfile"),
FILE_TYPE.HACK_FILE: self.util.safe_get_config("storage.azure.hack_file_container", "hackfile"),
}
self.azure_blob_service = RequiredFeature("azure_blob_service")
def stop_expr(self, expr_id, force=0):
"""
:param expr_id: experiment id
:param force: 0: only stop container and release ports, 1: force stop and delete container and release ports.
:return:
"""
self.log.debug("begin to stop %d" % expr_id)
expr = self.db.find_first_object_by(Experiment,
id=expr_id,
status=EStatus.RUNNING)
if expr is not None:
# Docker
if expr.template.provider == VE_PROVIDER.DOCKER:
# stop containers
for c in expr.virtual_environments.all():
try:
self.log.debug("begin to stop %s" % c.name)
docker = self.__get_docker(expr.hackathon, c)
if force:
docker.delete(c.name,
virtual_environment=c,
container=c.container,
expr_id=expr_id)
c.status = VEStatus.DELETED
else:
docker.stop(c.name,
virtual_environment=c,
container=c.container,
expr_id=expr_id)
c.status = VEStatus.STOPPED
except Exception as e:
self.log.error(e)
self.__roll_back(expr_id)
return internal_server_error(
'Failed stop/delete container')
if force:
expr.status = EStatus.DELETED
else:
expr.status = EStatus.STOPPED
self.db.commit()
else:
try:
# todo support delete azure vm
hosted_docker = RequiredFeature("hosted_docker")
af = AzureFormation(
hosted_docker.load_azure_key_id(expr_id))
af.stop(expr_id, AVMStatus.STOPPED_DEALLOCATED)
except Exception as e:
self.log.error(e)
return internal_server_error('Failed stopping azure')
self.log.debug("experiment %d ended success" % expr_id)
return ok('OK')
else:
return ok()
def is_pre_allocate_enabled(hackathon):
if hackathon.status != HACK_STATUS.ONLINE:
return False
if hackathon.event_end_time < util.get_now():
return False
hack_manager = RequiredFeature("hackathon_manager")
value = hack_manager.get_basic_property(
hackathon, HACKATHON_BASIC_INFO.PRE_ALLOCATE_ENABLED, "1")
return util.str2bool(value)
def get_starter(self, hackathon, template):
# load expr starter
starter = None
if not hackathon or not template:
return starter
# TODO Interim workaround for kubernetes, need real implementation
if hackathon.config.get('cloud_provider') == CLOUD_PROVIDER.KUBERNETES:
return RequiredFeature("k8s_service")
if template.provider == VE_PROVIDER.DOCKER:
if HACKATHON_CONFIG.CLOUD_PROVIDER in hackathon.config:
if hackathon.config[HACKATHON_CONFIG.CLOUD_PROVIDER] == CLOUD_PROVIDER.AZURE:
starter = RequiredFeature("azure_docker")
elif hackathon.config[HACKATHON_CONFIG.CLOUD_PROVIDER] == CLOUD_PROVIDER.ALAUDA:
starter = RequiredFeature("alauda_docker")
elif template.provider == VE_PROVIDER.AZURE:
starter = RequiredFeature("azure_vm")
elif template.provider == VE_PROVIDER.K8S:
starter = RequiredFeature("k8s_service")
return starter
class SubscriptionService(Component):
azure_adapter = RequiredFeature("azure_adapter")
"""
Subscription of azure resources according to given subscription id
"""
ERROR_RESULT = -1
def get_available_storage_account_count(self, azure_key_id):
"""
Get available count of storage account
Return -1 if failed
:return:
"""
try:
result = self.azure_adapter.get_subscription(azure_key_id)
except Exception as e:
self.log.error(e)
return self.ERROR_RESULT
return result.max_storage_accounts - result.current_storage_accounts
def get_available_cloud_service_count(self, azure_key_id):
"""
Get available count of cloud service
Return -1 if failed
:return:
"""
try:
result = self.azure_adapter.get_subscription(azure_key_id)
except Exception as e:
self.log.error(e)
return self.ERROR_RESULT
return result.max_hosted_services - result.current_hosted_services
def get_available_core_count(self, azure_key_id):
"""
Get available count of core
Return -1 if failed
:return:
"""
try:
result = self.azure_adapter.get_subscription(azure_key_id)
except Exception as e:
self.log.error(e)
return self.ERROR_RESULT
return result.max_core_count - result.current_core_count
def upload_files(self):
status, return_info = self.validate_args()
if not status:
return return_info
image_container_name = self.util.safe_get_config("storage.image_container", "images")
images = []
storage = RequiredFeature("storage")
for file_name in request.files:
file = request.files[file_name]
self.log.debug("upload image file : " + file_name)
context = Context(
hackathon_name=g.hackathon.name,
file_name=file.filename,
file_type=FILE_TYPE.HACK_IMAGE,
content=file
)
context = storage.save(context)
image = {}
image['name'] = file.filename
image['url'] = context.url
image['thumbnailUrl'] = context.url
# context.file_name is a random name created by server, file.filename is the original name
image['deleteUrl'] = '/api/admin/file?key=' + context.file_name
images.append(image)
# for file_name in request.files:
# file = request.files[file_name]
# real_name = self.generate_file_name(file)
# self.log.debug("upload image file : " + real_name)
#
# url = self.file_service.upload_file_to_azure(file, image_container_name, real_name)
# if url is not None:
# image = {}
# image['name'] = file.filename
# image['url'] = url
# # frontUI components needed return values
# image['thumbnailUrl'] = url
# image['deleteUrl'] = '/api/file?key=' + real_name
# images.append(image)
# else:
# return internal_server_error("upload file failed")
return {"files": images}
class DockerHostManager(Component):
"""Component to manage docker host server"""
docker = RequiredFeature("docker")
def get_available_docker_host(self, req_count, hackathon):
vms = self.db.find_all_objects(
DockerHostServer, DockerHostServer.container_count + req_count <=
DockerHostServer.container_max_count,
DockerHostServer.hackathon_id == hackathon.id)
# todo connect to azure to launch new VM if no existed VM meet the requirement
# since it takes some time to launch VM,
# it's more reasonable to launch VM when the existed ones are almost used up.
# The new-created VM must run 'cloudvm service by default(either cloud-init or python remote ssh)
# todo the VM public/private IP will change after reboot, need sync the IP in db with azure in this case
for docker_host in vms:
if self.docker.hosted_docker.ping(docker_host):
return docker_host
raise Exception("No available VM.")
def get_host_server_by_id(self, id):
return self.db.find_first_object_by(DockerHostServer, id=id)
def upload_files(self):
"""Handle uploaded files from http request"""
self.__validate_upload_files()
images = []
storage = RequiredFeature("storage")
for file_name in request.files:
file_storage = request.files[file_name]
self.log.debug("upload image file : " + file_name)
context = Context(hackathon_name=g.hackathon.name,
file_name=file_storage.filename,
file_type=FILE_TYPE.HACK_IMAGE,
content=file_storage)
context = storage.save(context)
image = {
"name": file_storage.filename,
"url": context.url,
"thumbnailUrl": context.url,
"deleteUrl": '/api/admin/file?key=' + context.file_name
}
# context.file_name is a random name created by server, file.filename is the original name
images.append(image)
return {"files": images}
def __init__(self):
self.alauda_docker = RequiredFeature("alauda_docker")
def get_basic_property(hackathon, property_name, default_value=None):
hack_manager = RequiredFeature("hackathon_manager")
return hack_manager.get_basic_property(hackathon, property_name,
default_value)
def __init__(self):
self.storage = RequiredFeature("storage")
def get_pre_allocate_number(hackathon):
hack_manager = RequiredFeature("hackathon_manager")
value = hack_manager.get_basic_property(
hackathon, HACKATHON_BASIC_INFO.PRE_ALLOCATE_NUMBER, 1)
return int(value)
def is_alauda_enabled(hackathon):
hack_manager = RequiredFeature("hackathon_manager")
value = hack_manager.get_basic_property(
hackathon, HACKATHON_BASIC_INFO.ALAUDA_ENABLED, "0")
return util.str2bool(value)
def is_auto_approve(hackathon):
hack_manager = RequiredFeature("hackathon_manager")
value = hack_manager.get_basic_property(hackathon,
HACKATHON_BASIC_INFO.AUTO_APPROVE,
"1")
return util.str2bool(value)
class ExprManager(Component):
user_manager = RequiredFeature("user_manager")
hackathon_manager = RequiredFeature("hackathon_manager")
admin_manager = RequiredFeature("admin_manager")
template_library = RequiredFeature("template_library")
hosted_docker_proxy = RequiredFeature("hosted_docker_proxy")
def start_expr(self, user, template_name, hackathon_name=None):
"""
A user uses a template to start a experiment under a hackathon
:param hackathon_name:
:param template_name:
:param user_id:
:return:
"""
self.log.debug(
"try to start experiment for hackathon %s using template %s" %
(hackathon_name, template_name))
hackathon = self.__verify_hackathon(hackathon_name)
template = self.__verify_template(hackathon, template_name)
if user:
expr = self.__check_expr_status(user, hackathon, template)
if expr:
return self.__report_expr_status(expr)
# new expr
return self.__start_new_expr(hackathon, template, user)
def restart_stopped_expr(self, experiment_id):
# todo: now just support hosted_docker, not support for alauda and windows
experiment = Experiment.objects(id=experiment_id).first()
for ve in experiment.virtual_environments:
if ve.provider == VE_PROVIDER.DOCKER:
if not self.hosted_docker_proxy.is_container_running(
ve.docker_container):
self.hosted_docker_proxy.start_container(
ve.docker_container.host_server,
ve.docker_container.container_id)
elif ve.provider == VE_PROVIDER.ALAUDA:
pass
elif ve.provider == VE_PROVIDER.AZURE:
pass
self.__check_expr_real_status(experiment)
return experiment.dic()
def heart_beat(self, expr_id):
expr = Experiment.objects(id=expr_id, status=EStatus.RUNNING).first()
if expr is None:
return not_found('Experiment is not running')
expr.last_heart_beat_time = self.util.get_now()
expr.save()
return ok()
def stop_expr(self, expr_id):
"""
:param expr_id: experiment id
:return:
"""
self.log.debug("begin to stop %s" % str(expr_id))
expr = Experiment.objects(id=expr_id, status=EStatus.RUNNING).first()
if expr is not None:
starter = self.get_starter(expr.hackathon, expr.template)
if starter:
starter.stop_expr(
Context(experiment_id=expr.id, experiment=expr))
self.log.debug("experiment %s ended success" % expr_id)
return ok('OK')
else:
return ok()
def get_expr_status_and_confirm_starting(self, expr_id):
expr = Experiment.objects(id=expr_id).first()
if expr:
return self.__report_expr_status(expr,
isToConfirmExprStarting=True)
else:
return not_found('Experiment Not found')
def check_expr_status(self, experiment):
# update experiment status
virtual_environment_list = experiment.virtual_environments
if all(x.status == VEStatus.RUNNING for x in virtual_environment_list) \
and len(virtual_environment_list) == experiment.template.virtual_environment_count:
experiment.status = EStatus.RUNNING
experiment.save()
try:
self.template_library.template_verified(experiment.template.id)
except:
pass
def get_expr_list_by_hackathon_id(self, hackathon, context):
# get a list of all experiments' detail
user_name = context.user_name if "user_name" in context else None
status = context.status if "status" in context else None
page = int(context.page) if "page" in context else 1
per_page = int(context.per_page) if "per_page" in context else 10
users = User.objects(name=user_name).all() if user_name else []
if user_name and status:
experiments_pagi = Experiment.objects(hackathon=hackathon,
status=status,
user__in=users).paginate(
page, per_page)
elif user_name and not status:
experiments_pagi = Experiment.objects(hackathon=hackathon,
user__in=users).paginate(
page, per_page)
elif not user_name and status:
experiments_pagi = Experiment.objects(hackathon=hackathon,
status=status).paginate(
page, per_page)
else:
experiments_pagi = Experiment.objects(
hackathon=hackathon).paginate(page, per_page)
return self.util.paginate(experiments_pagi,
self.__get_expr_with_detail)
def scheduler_recycle_expr(self):
"""recycle experiment according to hackathon basic info on recycle configuration
According to the hackathon's basic info on 'recycle_enabled', find out time out experiments
Then call function to recycle them
:return:
"""
self.log.debug("start checking recyclable experiment ... ")
for hackathon in self.hackathon_manager.get_recyclable_hackathon_list(
):
try:
# check recycle enabled
mins = self.hackathon_manager.get_recycle_minutes(hackathon)
# filter out the experiments that need to be recycled
exprs = Experiment.objects(
create_time__lt=self.util.get_now() -
timedelta(minutes=mins),
status=EStatus.RUNNING,
hackathon=hackathon)
for expr in exprs:
self.__recycle_expr(expr)
except Exception as e:
self.log.error(e)
def pre_allocate_expr(self, context):
# TODO: too complex, not check
hackathon_id = context.hackathon_id
self.log.debug("executing pre_allocate_expr for hackathon %s " %
hackathon_id)
hackathon = Hackathon.objects(id=hackathon_id).first()
hackathon_templates = hackathon.templates
for template in hackathon_templates:
try:
template = template
pre_num = int(hackathon.config.get("pre_allocate_number", 1))
query = Q(status=EStatus.STARTING) | Q(status=EStatus.RUNNING)
curr_num = Experiment.objects(
user=None, hackathon=hackathon,
template=template).filter(query).count()
if template.provider == VE_PROVIDER.AZURE:
if curr_num < pre_num:
remain_num = pre_num - curr_num
start_num = Experiment.objects(
user=None,
template=template,
status=EStatus.STARTING).count()
if start_num > 0:
self.log.debug(
"there is an azure env starting, will check later ... "
)
return
else:
self.log.debug(
"no starting template: %s , remain num is %d ... "
% (template.name, remain_num))
self.start_expr(None, template.name,
hackathon.name)
break
elif template.provider == VE_PROVIDER.DOCKER:
if hackathon.config.get(
'cloud_provider') == CLOUD_PROVIDER.ALAUDA:
# don't create pre-env if alauda used
continue
self.log.debug(
"template name is %s, hackathon name is %s" %
(template.name, hackathon.name))
if curr_num < pre_num:
remain_num = pre_num - curr_num
start_num = Experiment.objects(
user=None,
template=template,
status=EStatus.STARTING).count()
if start_num > 0:
self.log.debug(
"there is an docker container starting, will check later ... "
)
return
self.log.debug(
"no idle template: %s, remain num is %d ... " %
(template.name, remain_num))
self.start_expr(None, template.name, hackathon.name)
break
except Exception as e:
self.log.error(e)
self.log.error("check default experiment failed")
def assign_expr_to_admin(self, expr):
"""assign expr to admin to trun expr into pre_allocate_expr
:type expr: Experiment
:param expr: which expr you want to assign
:return:
"""
expr.user = None
expr.save()
# --------------------------------------------- helper function ---------------------------------------------#
def __verify_hackathon(self, hackathon_name):
"""validate the event_start_time and event_end_time of a hackathon
Will return None if hackathon not found or current time is not between its start time and end time
"""
hackathon = self.hackathon_manager.get_hackathon_by_name(
hackathon_name)
if hackathon:
if HACKATHON_CONFIG.CLOUD_PROVIDER not in hackathon.config:
raise PreconditionFailed(
"No cloud resource is configured for this hackathon.")
if self.util.get_now() < hackathon.event_end_time:
return hackathon
else:
raise PreconditionFailed("Hackathon was already ended")
else:
raise NotFound("Hackathon with name %s not found" % hackathon_name)
def get_starter(self, hackathon, template):
# load expr starter
starter = None
if not hackathon or not template:
return starter
if template.provider == VE_PROVIDER.DOCKER:
if HACKATHON_CONFIG.CLOUD_PROVIDER in hackathon.config:
if hackathon.config[HACKATHON_CONFIG.
CLOUD_PROVIDER] == CLOUD_PROVIDER.AZURE:
starter = RequiredFeature("azure_docker")
elif hackathon.config[HACKATHON_CONFIG.
CLOUD_PROVIDER] == CLOUD_PROVIDER.ALAUDA:
starter = RequiredFeature("alauda_docker")
elif template.provider == VE_PROVIDER.AZURE:
starter = RequiredFeature("azure_vm")
elif template.provider == VE_PROVIDER.K8S:
starter = RequiredFeature("k8s_service")
return starter
def __start_new_expr(self, hackathon, template, user):
starter = self.get_starter(hackathon, template)
if not starter:
raise PreconditionFailed(
"either template not supported or hackathon resource not configured"
)
context = starter.start_expr(
Context(template=template, user=user, hackathon=hackathon))
return self.__report_expr_status(context.experiment)
def on_expr_started(self, experiment):
hackathon = experiment.hackathon
user = experiment.user
def __report_expr_status(self, expr, isToConfirmExprStarting=False):
# todo check whether need to restart Window-expr and Alauda-expr if it shutdown
ret = {
"expr_id": str(expr.id),
"status": expr.status,
"hackathon_name": expr.hackathon.name if expr.hackathon else "",
"hackathon": str(expr.hackathon.id) if expr.hackathon else "",
"create_time": str(expr.create_time),
"last_heart_beat_time": str(expr.last_heart_beat_time)
}
if expr.status != EStatus.RUNNING:
return ret
# return remote clients include guacamole
remote_servers = []
for ve in expr.virtual_environments:
if ve.remote_provider == VERemoteProvider.Guacamole:
try:
guacamole_config = ve.remote_paras
guacamole_host = self.util.safe_get_config(
"guacamole.host", "localhost:8080")
# target url format:
# http://localhost:8080/guacamole/#/client/c/{name}?name={name}&oh={token}
name = guacamole_config["name"]
url = guacamole_host + '/guacamole/#/client/c/%s?name=%s' % (
name, name)
remote_servers.append({
"name": guacamole_config["name"],
"guacamole_host": guacamole_host,
"url": url
})
except Exception as e:
self.log.error(e)
# so that the frontend can query again?
ret["status"] = EStatus.STARTING
return ret
ret["remote_servers"] = remote_servers
# return public accessible web url
public_urls = []
if expr.template.provider == VE_PROVIDER.DOCKER:
for ve in expr.virtual_environments:
container = ve.docker_container
# to restart hosted_docker expr if it stopped.
if isToConfirmExprStarting:
if not self.hosted_docker_proxy.is_container_running(
container):
self.hosted_docker_proxy.start_container(
container.host_server, container.container_id)
for p in container.port_bindings.filter(is_public=True):
if p.url:
public_urls.append({
"name":
p.name,
"url":
p.url.format(container.host_server.public_dns,
p.public_port)
})
else:
for ve in expr.virtual_environments:
vm = ve.azure_resource
if not vm or not vm.end_points:
continue
for endpoint in vm.end_points:
if endpoint.url:
public_urls.append({
"name":
endpoint.name,
"url":
endpoint.url.format(vm.dns, endpoint.public_port)
})
ret["public_urls"] = public_urls
return ret
def __verify_template(self, hackathon, template_name):
template = self.template_library.get_template_info_by_name(
template_name)
if not template:
raise NotFound("template cannot be found by name '%s'" %
template_name)
if not hackathon:
# hackathon is None means it's starting expr for template testing
return template
hackathon_templates = hackathon.templates
template_ids = [t.id for t in hackathon_templates]
if template.id not in template_ids:
raise PreconditionFailed(
"template '%s' not allowed for hackathon '%s'" %
(template_name, hackathon.name))
return template
def __check_expr_status(self, user, hackathon, template):
"""
check experiment status, if there are pre-allocate experiments, the experiment will be assigned directly
:param user:
:param hackathon:
:param template:
:return:
"""
criterion = Q(status__in=[EStatus.RUNNING, EStatus.STARTING],
hackathon=hackathon,
user=user)
is_admin = self.admin_manager.is_hackathon_admin(hackathon.id, user.id)
if is_admin:
criterion &= Q(template=template)
expr = Experiment.objects(criterion).first()
if expr:
# user has a running/starting experiment
return expr
# try to assign pre-configured expr to user
expr = Experiment.objects(status=EStatus.RUNNING,
hackathon=hackathon,
template=template,
user=None).first()
if expr:
expr.user = user
expr.save()
return expr
def roll_back(self, expr_id):
"""
roll back when exception occurred
:param expr_id: experiment id
"""
self.log.debug("Starting rollback experiment %s..." % expr_id)
expr = Experiment.objects(id=expr_id)
if not expr:
self.log.warn("rollback failed due to experiment not found")
return
starter = self.get_starter(expr.hackathon, expr.template)
if not starter:
self.log.warn("rollback failed due to no starter found")
return
return starter.rollback(Context(experiment=expr))
def __get_expr_with_detail(self, experiment):
self.__check_expr_real_status(experiment)
info = experiment.dic()
# replace OjbectId with user info
info['user'] = self.user_manager.user_display_info(experiment.user)
return info
def __check_expr_real_status(self, experiment):
# todo: it is only support for hosted_docker right now. Please support Window-expr and Alauda-expr in future
for ve in experiment.virtual_environments:
if ve.provider == VE_PROVIDER.DOCKER:
if not self.hosted_docker_proxy.is_container_running(
ve.docker_container):
if ve.status == VEStatus.RUNNING:
ve.status = VEStatus.STOPPED
else:
if ve.status == VEStatus.STOPPED:
ve.status = VEStatus.RUNNING
elif ve.provider == VE_PROVIDER.ALAUDA:
pass
elif ve.provider == VE_PROVIDER.AZURE:
pass
if all(ve.status == VEStatus.STOPPED
for ve in experiment.virtual_environments):
experiment.status = EStatus.STOPPED
if all(ve.status == VEStatus.RUNNING
for ve in experiment.virtual_environments):
experiment.status = EStatus.RUNNING
experiment.update_time = self.util.get_now()
experiment.save()
def __recycle_expr(self, expr):
"""recycle expr
If it is a docker experiment , stop it ; else assign it to default user
:type expr: Experiment
:param expr: the exper which you want to recycle
:return:
"""
providers = map(lambda x: x.provider, expr.virtual_environments)
if VE_PROVIDER.DOCKER in providers:
self.stop_expr(expr.id)
self.log.debug("it's stopping " + str(expr.id) +
" inactive experiment now")
else:
self.assign_expr_to_admin(expr)
self.log.debug("assign " + str(expr.id) + " to default admin")
class HackathonManager(Component):
"""Component to manage hackathon
Note that it only handle operations directly related to Hackathon table. Things like registerd users, templates are
in separated components
"""
admin_manager = RequiredFeature("admin_manager")
user_manager = RequiredFeature("user_manager")
register_manager = RequiredFeature("register_manager")
# basic xss prevention
cleaner = Cleaner(safe_attrs=lxml.html.defs.safe_attrs
| set(['style'])) # preserve style
def is_hackathon_name_existed(self, name):
"""Check whether hackathon with specific name exists or not
:type name: str|unicode
:param name: name of hackathon
:rtype: bool
:return True if hackathon with specific name exists otherwise False
"""
hackathon = self.get_hackathon_by_name(name)
return hackathon is not None
def is_recycle_enabled(self, hackathon):
key = HACKATHON_BASIC_INFO.RECYCLE_ENABLED
return self.util.str2bool(
self.get_basic_property(hackathon, key, False))
def get_hackathon_by_name(self, name):
"""Get hackathon accoring the unique name
:type name: str|unicode
:param name: name of hackathon
:rtype: Hackathon
:return hackathon instance if found else None
"""
if not name:
return None
return self.db.find_first_object_by(Hackathon, name=name)
def get_hackathon_by_id(self, hackathon_id):
"""Query hackathon by id
:return hackathon instance or None
"""
return self.db.find_first_object_by(Hackathon, id=hackathon_id)
def get_hackathon_detail(self, hackathon):
user = None
if self.user_manager.validate_login():
user = g.user
return self.__get_hackathon_detail(hackathon, user)
def get_hackathon_stat(self, hackathon):
def internal_get_stat():
return self.__get_hackathon_stat(hackathon)
cache_key = "hackathon_stat_%s" % hackathon.id
return self.cache.get_cache(key=cache_key,
createfunc=internal_get_stat)
def get_hackathon_list(self, args):
# get values from request's QueryString
page = int(args.get("page", 1))
per_page = int(args.get("per_page", 20))
order_by = args.get("order_by", "create_time")
status = args.get("status")
name = args.get("name")
# build query by search conditions and order_by
query = Hackathon.query
if status:
query = query.filter(Hackathon.status == status)
if name:
query = query.filter(Hackathon.name.like("%" + name + "%"))
if order_by == "create_time":
query = query.order_by(Hackathon.create_time.desc())
elif order_by == "event_start_time":
# all started and coming hackathon-activities would be shown based on event_start_time.
query = query.order_by(Hackathon.event_start_time.desc())
# just coming hackathon-activities would be shown based on event_start_time.
# query = query.order_by(Hackathon.event_start_time.asc()).filter(Hackathon.event_start_time > self.util.get_now())
elif order_by == "registered_users_num":
# hackathons with zero registered users would not be shown.
query = query.join(HackathonStat).order_by(
HackathonStat.count.desc())
else:
query = query.order_by(Hackathon.id.desc())
# perform db query with pagination
pagination = self.db.paginate(query, page, per_page)
# check whether it's anonymous user or not
user = None
if self.user_manager.validate_login():
user = g.user
def func(hackathon):
return self.__get_hackathon_detail(hackathon, user)
# return serializable items as well as total count
return self.util.paginate(pagination, func)
def get_online_hackathons(self):
return self.db.find_all_objects(Hackathon,
Hackathon.status == HACK_STATUS.ONLINE)
def get_user_hackathon_list_with_detail(self, user_id):
user = self.user_manager.get_user_by_id(user_id)
user_hack_list = self.db.session().query(Hackathon, UserHackathonRel) \
.outerjoin(UserHackathonRel, UserHackathonRel.user_id == user_id) \
.filter(UserHackathonRel.deleted != 1, UserHackathonRel.user_id == user_id).all()
return map(lambda h: self.__get_hackathon_detail(h, user),
user_hack_list)
def get_recyclable_hackathon_list(self):
all_hackathon = self.db.find_all_objects(Hackathon)
return filter(lambda h: self.is_recycle_enabled(h), all_hackathon)
def get_entitled_hackathon_list_with_detail(self, user):
hackathon_ids = self.admin_manager.get_entitled_hackathon_ids(user.id)
if -1 in hackathon_ids:
hackathon_list = self.db.find_all_objects(Hackathon)
else:
hackathon_list = self.db.find_all_objects(
Hackathon, Hackathon.id.in_(hackathon_ids))
return map(lambda h: self.__get_hackathon_detail(h, user),
hackathon_list)
def get_basic_property(self, hackathon, key, default=None):
"""Get basic property of hackathon from HackathonConfig"""
config = self.db.find_first_object_by(HackathonConfig,
key=key,
hackathon_id=hackathon.id)
if config:
return config.value
return default
def get_all_properties(self, hackathon):
configs = self.db.find_all_objects_by(HackathonConfig,
hackathon_id=hackathon.id)
return [c.dic() for c in configs]
def set_basic_property(self, hackathon, properties):
"""Set basic property in table HackathonConfig"""
if isinstance(properties, list):
map(lambda p: self.__set_basic_property(hackathon, p), properties)
else:
self.__set_basic_property(hackathon, properties)
self.cache.invalidate(self.__get_config_cache_key(hackathon))
return ok()
def delete_property(self, hackathon, key):
self.db.delete_all_objects_by(HackathonConfig,
hackathon_id=hackathon.id,
key=key)
return ok()
def get_recycle_minutes(self, hackathon):
key = HACKATHON_BASIC_INFO.RECYCLE_MINUTES
minutes = self.get_basic_property(hackathon, key, 60)
return int(minutes)
def validate_hackathon_name(self):
if HTTP_HEADER.HACKATHON_NAME in request.headers:
try:
hackathon_name = request.headers[HTTP_HEADER.HACKATHON_NAME]
hackathon = self.get_hackathon_by_name(hackathon_name)
if hackathon is None:
self.log.debug("cannot find hackathon by name %s" %
hackathon_name)
return False
else:
g.hackathon = hackathon
return True
except Exception as ex:
self.log.error(ex)
self.log.debug("hackathon_name invalid")
return False
else:
self.log.debug("hackathon_name not found in headers")
return False
def create_new_hackathon(self, context):
"""Create new hackathon based on the http body
Hackathon name is unique so duplicated names are not allowd.
:type context: Context
:param context: the body of http request that contains fields to create a new hackathon
:rtype: dict
"""
hackathon = self.get_hackathon_by_name(context.name)
if hackathon is not None:
raise PreconditionFailed("hackathon name already exists")
self.log.debug("add a new hackathon:" + context.name)
new_hack = self.__create_hackathon(context)
# init data is for local only
if self.util.is_local():
self.__create_default_data_for_local(new_hack)
return new_hack.dic()
def update_hackathon(self, args):
"""Update hackathon properties
:type args: dict
:param args: arguments from http request body that contains properties with new values
:rtype dict
:return hackathon in dict if updated successfully.
"""
hackathon = g.hackathon
try:
update_items = self.__parse_update_items(args, hackathon)
self.log.debug("update hackathon items :" + str(args.keys()))
# basic xss prevention
if 'description' in update_items and update_items['description']:
update_items['description'] = self.cleaner.clean_html(
update_items['description'])
self.log.debug("hackathon description :" +
update_items['description'])
self.db.update_object(hackathon, **update_items)
return hackathon.dic()
except Exception as e:
self.log.error(e)
return internal_server_error("fail to update hackathon")
def upload_files(self):
"""Handle uploaded files from http request"""
self.__validate_upload_files()
images = []
storage = RequiredFeature("storage")
for file_name in request.files:
file_storage = request.files[file_name]
self.log.debug("upload image file : " + file_name)
context = Context(hackathon_name=g.hackathon.name,
file_name=file_storage.filename,
file_type=FILE_TYPE.HACK_IMAGE,
content=file_storage)
context = storage.save(context)
image = {
"name": file_storage.filename,
"url": context.url,
"thumbnailUrl": context.url,
"deleteUrl": '/api/admin/file?key=' + context.file_name
}
# context.file_name is a random name created by server, file.filename is the original name
images.append(image)
return {"files": images}
def like_hackathon(self, user, hackathon):
like = self.db.find_first_object_by(HackathonLike,
user_id=user.id,
hackathon_id=hackathon.id)
if not like:
like = HackathonLike(user_id=user.id, hackathon_id=hackathon.id)
self.db.add_object(like)
self.db.commit()
# increase the count of users that like this hackathon
self.increase_hackathon_stat(hackathon, HACKATHON_STAT.LIKE, 1)
return ok()
def unlike_hackathon(self, user, hackathon):
self.db.delete_all_objects_by(HackathonLike,
user_id=user.id,
hackathon_id=hackathon.id)
self.db.commit()
# sync the like count
like_count = self.db.count_by(HackathonLike, hackathon_id=hackathon.id)
self.update_hackathon_stat(hackathon, HACKATHON_STAT.LIKE, like_count)
return ok()
def update_hackathon_stat(self, hackathon, stat_type, count):
"""Increase or descrease the count for certain hackathon stat
:type hackathon: Hackathon
:param hackathon: instance of Hackathon to be counted
:type stat_type: str|unicode
:param stat_type: type of stat that defined in constants.py#HACKATHON_STAT
:type count: int
:param count: the new count for this stat item
"""
stat = self.db.find_first_object_by(HackathonStat,
hackathon_id=hackathon.id,
type=stat_type)
if stat:
stat.count = count
stat.update_time = self.util.get_now()
else:
stat = HackathonStat(hackathon_id=hackathon.id,
type=stat_type,
count=count)
self.db.add_object(stat)
if stat.count < 0:
stat.count = 0
self.db.commit()
def increase_hackathon_stat(self, hackathon, stat_type, increase):
"""Increase or descrease the count for certain hackathon stat
:type hackathon: Hackathon
:param hackathon: instance of Hackathon to be counted
:type stat_type: str|unicode
:param stat_type: type of stat that defined in constants.py#HACKATHON_STAT
:type increase: int
:param increase: increase of the count. Can be positive or negative
"""
stat = self.db.find_first_object_by(HackathonStat,
hackathon_id=hackathon.id,
type=stat_type)
if stat:
stat.count += increase
else:
stat = HackathonStat(hackathon_id=hackathon.id,
type=stat_type,
count=increase)
self.db.add_object(stat)
if stat.count < 0:
stat.count = 0
self.db.commit()
def get_hackathon_tags(self, hackathon):
tags = self.db.find_all_objects_by(HackathonTag,
hackathon_id=hackathon.id)
return ",".join([t.tag for t in tags])
def set_hackathon_tags(self, hackathon, tags):
"""Set hackathon tags
:type tags: list
:param tags: a list of str, every str is a tag
"""
self.db.delete_all_objects_by(HackathonTag, hackathon_id=hackathon.id)
for tag in tags:
t = tag.strip('"').strip("'")
self.db.add_object(HackathonTag(tag=t, hackathon_id=hackathon.id))
self.db.commit()
return ok()
def get_distinct_tags(self):
"""Return all distinct hackathon tags for auto-complete usage"""
return self.db.session().query(HackathonTag.tag).distinct().all()
def qet_organizer_by_id(self, organizer_id):
organizer = self.db.get_object(HackathonOrganizer, organizer_id)
if organizer:
return organizer.dic()
return not_found()
def create_hackathon_organizer(self, hackathon, body):
organizer = HackathonOrganizer(
hackathon_id=hackathon.id,
name=body["name"],
organization_type=body.get("organization_type"),
description=body.get("description"),
homepage=body.get("homepage"),
logo=body.get("logo"),
create_time=self.util.get_now())
self.db.add_object(organizer)
return organizer.dic()
def update_hackathon_organizer(self, hackathon, body):
organizer = self.db.get_object(HackathonOrganizer, body["id"])
if not organizer:
return not_found()
if organizer.hackathon_id != hackathon.id:
return forbidden()
organizer.name = body.get("name", organizer.name)
organizer.organization_type = body.get("organization_type",
organizer.organization_type)
organizer.description = body.get("description", organizer.description)
organizer.homepage = body.get("homepage", organizer.homepage)
organizer.logo = body.get("logo", organizer.logo)
organizer.update_time = self.util.get_now()
self.db.commit()
return organizer.dic()
def delete_hackathon_organizer(self, hackathon, organizer_id):
self.db.delete_all_objects_by(HackathonOrganizer,
id=organizer_id,
hackathon_id=hackathon.id)
return ok()
def create_hackathon_award(self, hackathon, body):
level = int(body.level)
if level > 10:
level = 10
award = Award(hackathon_id=hackathon.id,
name=body.name,
level=level,
quota=body.quota,
award_url=body.get("award_url"),
description=body.get("description"))
self.db.add_object(award)
return award.dic()
def update_hackathon_award(self, hackathon, body):
award = self.db.get_object(Award, body.id)
if not award:
return not_found("award not found")
if award.hackathon.name != hackathon.name:
return forbidden()
level = award.level
if body.get("level"):
level = int(body.level)
if level > 10:
level = 10
award.name = body.get("name", award.name)
award.level = body.get("level", level)
award.quota = body.get("quota", award.quota)
award.award_url = body.get("award_url", award.award_url)
award.description = body.get("description", award.description)
award.update_time = self.util.get_now()
self.db.commit()
return award.dic()
def delete_hackathon_award(self, hackathon, award_id):
self.db.delete_all_objects_by(Award,
hackathon_id=hackathon.id,
id=award_id)
return ok()
def list_hackathon_awards(self, hackathon):
awards = hackathon.award_contents.order_by(Award.level.desc()).all()
return [a.dic() for a in awards]
def schedule_pre_allocate_expr_job(self):
"""Add an interval schedule job to check all hackathons"""
next_run_time = self.util.get_now() + timedelta(seconds=3)
self.scheduler.add_interval(
feature="hackathon_manager",
method="check_hackathon_for_pre_allocate_expr",
id="check_hackathon_for_pre_allocate_expr",
next_run_time=next_run_time,
minutes=10)
def check_hackathon_for_pre_allocate_expr(self):
"""Check all hackathon for pre-allocate
Add an interval job for hackathon if it's pre-allocate is enabled.
Otherwise try to remove the schedule job
"""
hackathon_list = self.db.find_all_objects(Hackathon)
for hack in hackathon_list:
job_id = "pre_allocate_expr_" + str(hack.id)
is_job_exists = self.scheduler.has_job(job_id)
if hack.is_pre_allocate_enabled():
if is_job_exists:
self.log.debug(
"pre_allocate job already exists for hackathon %s" %
str(hack.id))
continue
self.log.debug("add pre_allocate job for hackathon %s" %
str(hack.id))
next_run_time = self.util.get_now() + timedelta(
seconds=hack.id * 10)
pre_allocate_interval = self.__get_pre_allocate_interval(hack)
self.scheduler.add_interval(
feature="expr_manager",
method="pre_allocate_expr",
id=job_id,
context=Context(hackathon_id=hack.id),
next_run_time=next_run_time,
seconds=pre_allocate_interval)
elif is_job_exists:
self.log.debug(
"remove job for hackathon %s since pre_allocate is disabled"
% str(hack.id))
self.scheduler.remove_job(job_id)
return True
def check_hackathon_online(self, hackathon):
alauda_enabled = is_alauda_enabled(hackathon)
can_online = True
if alauda_enabled == "0":
if self.util.is_local():
can_online = True
else:
can_online = docker_host_manager.check_subscription_id(
hackathon.id)
return ok(can_online)
def __get_hackathon_detail(self, hackathon, user=None):
"""Return hackathon info as well as its details including configs, stat, organizers, like if user logon"""
detail = hackathon.dic()
detail["config"] = self.__get_hackathon_configs(hackathon)
detail["stat"] = self.get_hackathon_stat(hackathon)
detail["tag"] = self.get_hackathon_tags(hackathon)
detail["organizer"] = self.__get_hackathon_organizers(hackathon)
if user:
detail["user"] = self.user_manager.user_display_info(user)
detail["user"]["is_admin"] = self.admin_manager.is_hackathon_admin(
hackathon.id, user.id)
asset = self.db.find_all_objects_by(UserHackathonAsset,
user_id=user.id,
hackathon_id=hackathon.id)
if asset:
detail["asset"] = [o.dic() for o in asset]
like = self.db.find_first_object_by(HackathonLike,
user_id=user.id,
hackathon_id=hackathon.id)
if like:
detail["like"] = like.dic()
register = self.register_manager.get_registration_by_user_and_hackathon(
user.id, hackathon.id)
if register:
detail["registration"] = register.dic()
team_rel = self.db.find_first_object_by(UserTeamRel,
user_id=user.id,
hackathon_id=hackathon.id)
if team_rel:
detail["team"] = team_rel.team.dic()
return detail
def __create_hackathon(self, context):
"""Insert hackathon and admin_hackathon_rel to database
We enforce that default config are used during the creation
:type context: Context
:param context: context of the args to create a new hackathon
:rtype: Hackathon
:return hackathon instance
"""
new_hack = Hackathon(
name=context.name,
display_name=context.display_name,
ribbon=context.get("ribbon"),
description=context.get("description"),
short_description=context.get("short_description"),
banners=context.get("banners"),
status=HACK_STATUS.INIT,
creator_id=g.user.id,
event_start_time=context.get("event_start_time"),
event_end_time=context.get("event_end_time"),
registration_start_time=context.get("registration_start_time"),
registration_end_time=context.get("registration_end_time"),
judge_start_time=context.get("judge_start_time"),
judge_end_time=context.get("judge_end_time"),
type=context.get("type", HACK_TYPE.HACKATHON))
# basic xss prevention
if new_hack.description: # case None type
new_hack.description = self.cleaner.clean_html(
new_hack.description)
# insert into table hackathon
self.db.add_object(new_hack)
# add the current login user as admin and creator
try:
ahl = AdminHackathonRel(user_id=g.user.id,
role_type=ADMIN_ROLE_TYPE.ADMIN,
hackathon_id=new_hack.id,
status=HACK_STATUS.INIT,
remarks='creator',
create_time=self.util.get_now())
self.db.add_object(ahl)
except Exception as ex:
# TODO: send out a email to remind administrator to deal with this problems
self.log.error(ex)
raise InternalServerError(
"fail to create the default administrator")
return new_hack
def __get_pre_allocate_interval(self, hackathon):
interval = self.get_basic_property(
hackathon, HACKATHON_BASIC_INFO.PRE_ALLOCATE_INTERVAL_SECONDS)
if interval:
return int(interval)
else:
return 300 + hackathon.id * 10
def __get_hackathon_configs(self, hackathon):
def __internal_get_config():
configs = {}
for c in hackathon.configs.all():
configs[c.key] = c.value
return configs
cache_key = self.__get_config_cache_key(hackathon)
return self.cache.get_cache(key=cache_key,
createfunc=__internal_get_config)
def __get_hackathon_organizers(self, hackathon):
organizers = self.db.find_all_objects_by(HackathonOrganizer,
hackathon_id=hackathon.id)
return [o.dic() for o in organizers]
def __parse_update_items(self, args, hackathon):
"""Parse properties that need to update
Only those whose value changed items will be returned. Also some static property like id, create_time should
NOT be updated.
:type args: dict
:param args: arguments from http body which contains new values
:type hackathon: Hackathon
:param hackathon: the existing Hackathon object which contains old values
:rtype: dict
:return a dict that contains all properties that are updated.
"""
result = {}
for key in dict(args):
if dict(args)[key] != hackathon.dic()[key]:
result[key] = dict(args)[key]
result.pop('id', None)
result.pop('create_time', None)
result.pop('creator_id', None)
result['update_time'] = self.util.get_now()
return result
def __get_hackathon_stat(self, hackathon):
stats = self.db.find_all_objects_by(HackathonStat,
hackathon_id=hackathon.id)
result = {"hackathon_id": hackathon.id, "online": 0, "offline": 0}
for item in stats:
result[item.type] = item.count
reg_list = hackathon.registers.filter(
UserHackathonRel.deleted != 1,
UserHackathonRel.status.in_(
[RGStatus.AUTO_PASSED, RGStatus.AUDIT_PASSED])).all()
reg_count = len(reg_list)
if reg_count > 0:
user_id_list = [r.user_id for r in reg_list]
user_id_online = self.db.count(User, (User.id.in_(user_id_list) &
(User.online == 1)))
result["online"] = user_id_online
result["offline"] = reg_count - user_id_online
return result
def __get_config_cache_key(self, hackathon):
return "hackathon_config_%s" % hackathon.id
def __create_default_data_for_local(self, hackathon):
"""
create test data for new hackathon. It's for local development only
:param hackathon:
:return:
"""
try:
# test docker host server
docker_host = DockerHostServer(
vm_name="localhost",
public_dns="localhost",
public_ip="127.0.0.1",
public_docker_api_port=4243,
private_ip="127.0.0.1",
private_docker_api_port=4243,
container_count=0,
container_max_count=100,
disabled=0,
state=DockerHostServerStatus.DOCKER_READY,
hackathon=hackathon)
if self.db.find_first_object_by(DockerHostServer,
vm_name=docker_host.vm_name,
hackathon_id=hackathon.id) is None:
self.db.add_object(docker_host)
except Exception as e:
self.log.error(e)
self.log.warn("fail to create test data")
return
def __validate_upload_files(self):
# check file size
if request.content_length > len(request.files) * self.util.get_config(
"storage.size_limit_kilo_bytes") * 1024:
raise BadRequest("more than the file size limited")
# check each file type
for file_name in request.files:
if request.files.get(file_name).filename.endswith('jpg'):
continue # jpg is not considered in imghdr
if imghdr.what(request.files.get(file_name)) is None:
raise BadRequest("only images can be uploaded")
def __set_basic_property(self, hackathon, prop):
"""Set basic property in table HackathonConfig"""
config = self.db.find_first_object_by(HackathonConfig,
hackathon_id=hackathon.id,
key=prop.key)
if config:
config.value = prop.value
else:
config = HackathonConfig(key=prop.key,
value=prop.value,
hackathon_id=hackathon.id)
self.db.add_object(config)
self.db.commit()
class UserManager(Component):
"""Component for user management"""
admin_manager = RequiredFeature("admin_manager")
oauth_login_manager = RequiredFeature("oauth_login_manager")
def validate_login(self):
"""Make sure user token is included in http request headers and it must NOT be expired
If valid token is found , the related user will be set int flask global g. So you can access g.user to get the
current login user throughout the request. There is no need to query user from DB again.
:rtype: bool
:return True if valid token found in DB otherwise False
"""
if HTTP_HEADER.TOKEN not in request.headers:
return False
user = self.__validate_token(request.headers[HTTP_HEADER.TOKEN])
if user is None:
return False
g.user = user
return True
def logout(self, user_id):
try:
user = self.get_user_by_id(user_id)
if user:
user.online = False
user.save()
g.user = None
g.token.delete()
return ok()
except Exception as e:
self.log.error(e)
return internal_server_error(e.message)
def login(self, provider, context):
if provider == "db":
return self.__db_login(context)
else:
return self.__oauth_login(provider, context)
def update_user_operation_time(self):
"""Update the user's last operation time.
:rtype:bool
:return True if success in updating, return False if token not found or token is overtime.
"""
if HTTP_HEADER.TOKEN not in request.headers:
return False
user = self.__validate_token(request.headers[HTTP_HEADER.TOKEN])
if user is None:
return False
else:
time_interval = timedelta(hours=self.util.safe_get_config(
"login.token_valid_time_minutes", 60))
new_toke_time = self.util.get_now() + time_interval
UserToken.objects(token=request.headers[HTTP_HEADER.TOKEN]).update(
expire_date=new_toke_time)
users_operation_time[user.id] = self.util.get_now()
return True
def check_user_online_status(self):
"""Check whether the user is offline. If the answer is yes, update its status in DB."""
overtime_user_ids = [
user_id for user_id in users_operation_time
if (self.util.get_now() -
users_operation_time[user_id]).seconds > 3600
]
# 3600s- expire as token expire
User.objects(id__in=overtime_user_ids).update(online=False)
for user_id in overtime_user_ids:
users_operation_time.pop(user_id, "")
def get_user_by_id(self, user_id):
"""Query user by unique id
:type user_id: str
:param user_id: _id of the user to query
:rtype: User
:return: instance of User or None if user not found
"""
return User.objects(id=user_id).first()
def load_user(self, user_id):
'''get user for flask_login user_loader'''
user = self.get_user_by_id(user_id)
dic = user.dic() if user else not_found()
return dic
def get_user_by_email(self, email):
"""Query user by email
:type email: str|unicode
:param email: email address
:rtype: User
:return instance of User or None if user cannot be found
"""
if email is None:
return None
return User.objects(emails__email=email).first()
def get_user_fezzy_search(self, hackathon, args):
"""fezzy search user by name, nickname and email
:type **kwargs: dict
:param **kwargs: dict should has key['condition']
:rtype: list
:return a list of users or empty list if not user match conditions
"""
keyword = args.get("keyword", "")
page = int(args.get("page", 1))
per_page = int(args.get("per_page", 20))
pagination = User.objects(
Q(name__icontains=keyword) | Q(nickname__icontains=keyword)
| Q(emails__email__icontains=keyword)).paginate(page, per_page)
def get_user_details(user):
user_info = self.user_display_info(user)
user_hackathon = UserHackathon.objects(hackathon=hackathon,
user=user).first()
user_info[
"role"] = user_hackathon.role if user_hackathon else HACK_USER_TYPE.VISITOR
user_info[
"remark"] = user_hackathon.remark if user_hackathon else ""
return user_info
# return serializable items as well as total count
return self.util.paginate(pagination, get_user_details)
def cleaned_user_dic(self, user):
"""trim the harmful and security info from the user object
this function return the cleaned info that can return to low-security client
such as web browser
:type user: User
:param user: User instance to be cleaned
:rtype: dict
:return: cleaned user dict
"""
ret = user.dic()
# pop high-security-risk data
if "password" in ret:
ret.pop("password")
if "access_token" in ret:
ret.pop("access_token")
return ret
def user_display_info(self, user):
"""Return user detail information
Sensitive information like password is filtered
Other info such as email, profile will be returned too to decrease http request times.
:type user: User
:param user: User instance to be returned which shouldn't be None
:rtype dict
:return user detail info from collection User
"""
if user is None:
return None
ret = self.cleaned_user_dic(user)
# set avatar_url to display
if "profile" in ret and "avatar_url" in ret["profile"]:
ret["avatar_url"] = ret["profile"]["avatar_url"]
return ret
def get_talents(self):
# todo real talents list
users = User.objects(name__ne="admin").order_by("-login_times")[:10]
# fixme why delete this log will panic
self.log.debug("get talents {}".format(users))
return [self.user_display_info(u) for u in users]
@staticmethod
def update_user_avatar_url(user, url):
if not user.profile:
user.profile = UserProfile()
user.profile.avatar_url = url
user.save()
return True
def upload_files(self, user_id, file_type):
"""Handle uploaded files from http request"""
try:
self.__validate_upload_files()
except Exception as e:
self.log.error(e)
return bad_request("file size or file type unsupport")
file_list = []
storage = RequiredFeature("storage")
for file in request.files:
file_content = request.files[file]
pre_file_name = file_content.filename
file_suffix = pre_file_name[pre_file_name.rfind('.'):]
new_file_name = self.__generate_file_name(user_id, file_type,
file_suffix)
self.log.debug("upload file: " + new_file_name)
context = Context(file_name=new_file_name,
file_type=file_type,
content=file_content)
context = storage.save(context)
# file_name is a random name created by server, pre_file_name is the original name
file_info = {
"file_name": new_file_name,
"pre_file_name": pre_file_name,
"url": context.url
}
file_list.append(file_info)
return {"files": file_list}
# ----------------------------private methods-------------------------------------
def __validate_token(self, token):
"""Validate token to make sure it exists and not expired
:type token: str|unicode
:param token: token strin
:rtype: User
:return user related to the token or None if token is invalid
"""
if "authenticated" in g and g.authenticated:
return g.user
else:
# todo eliminate the warning related to 'objects'
t = UserToken.objects(token=token).first()
if t and t.expire_date >= self.util.get_now():
g.authenticated = True
g.user = t.user
# save token to g, to determine which one to remove, when logout
g.token = t
return t.user
return None
def __generate_api_token(self, admin):
token_issue_date = self.util.get_now()
valid_period = timedelta(minutes=self.util.safe_get_config(
"login.token_valid_time_minutes", 60))
token_expire_date = token_issue_date + valid_period
user_token = UserToken(token=str(uuid.uuid1()),
user=admin,
expire_date=token_expire_date,
issue_date=token_issue_date)
user_token.save()
return user_token
def __db_login(self, context):
username = context.get("username")
enc_pwd = context.get("password")
user = User.objects(name=username, password=enc_pwd).first()
if user is None:
self.log.warn(
"invalid user/pwd login: username=%s, encoded pwd=%s" %
(username, enc_pwd))
return unauthorized("username or password error")
user.online = True
user.login_times = (user.login_times or 0) + 1
user.save()
token = self.__generate_api_token(user)
return {"token": token.dic(), "user": user.dic()}
def __create_or_update_email(self, user, email_info):
email = email_info['email']
primary_email = email_info['primary']
verified = email_info['verified']
new_mail = UserEmail(email=email,
primary_email=primary_email,
verified=verified)
existed = False
for i, e in enumerate(user.emails):
if e.email == email:
user.emails[i] = new_mail
existed = True
break
if not existed:
user.emails.append(new_mail)
user.save()
def __get_existing_user(self, openid, provider):
return User.objects(openid=openid, provider=provider).first()
def __oauth_login(self, provider, context):
self.log.info("Oauth login with %s and code: %s" %
(provider, context.code))
oauth_resp = self.oauth_login_manager.oauth_login(provider, context)
return self.__oauth_login_db(provider, Context.from_object(oauth_resp))
def __oauth_login_db(self, provider, context):
# update db
email_list = context.get('email_list', [])
openid = context.openid
user = self.__get_existing_user(openid, provider)
if user is not None:
user.update(provider=provider,
name=context.get("name", user.name),
nickname=context.get("nickname", user.nickname),
access_token=context.get("access_token",
user.access_token),
avatar_url=context.get("avatar_url", user.avatar_url),
last_login_time=self.util.get_now(),
login_times=user.login_times + 1,
online=True)
list(
map(lambda x: self.__create_or_update_email(user, x),
email_list))
else:
user = User(openid=openid,
name=context.name,
provider=provider,
nickname=context.nickname,
access_token=context.access_token,
avatar_url=context.get("avatar_url", ""),
login_times=1,
online=True)
try:
user.save()
except ValidationError as e:
self.log.error(e)
return internal_server_error("create user fail.")
list(
map(lambda x: self.__create_or_update_email(user, x),
email_list))
# generate API token
token = self.__generate_api_token(user)
resp = {"token": token.dic(), "user": user.dic()}
return resp
def __oxford(self, user, oxford_api):
if not oxford_api:
return
# TODO: not finish
# hackathon = Hackathon.objects(name="oxford").first()
# if hackathon:
# exist = self.db.find_first_object_by(UserHackathonAsset, asset_value=oxford_api)
# if exist:
# return
#
# asset = UserHackathonAsset(user_id=user.id,
# hackathon_id=hackathon.id,
# asset_name="Oxford Token",
# asset_value=oxford_api,
# description="Token for Oxford API")
# self.db.add_object(asset)
# self.db.commit()
def __generate_file_name(self, user_id, type, suffix):
# may generate differrnt file_names for different type. see FILE_TYPE.
file_name = "%s-%s%s" % (str(user_id), str(uuid.uuid1())[0:8], suffix)
return file_name
def __validate_upload_files(self):
# todo check file size and file type
# if request.content_length > len(request.files) * self.util.get_config("storage.size_limit_kilo_bytes") * 1024:
# raise BadRequest("more than the file size limited")
# check each file type and only jpg is allowed
# for file_name in request.files:
# if request.files.get(file_name).filename.endswith('jpg'):
# continue # jpg is not considered in imghdr
# if imghdr.what(request.files.get(file_name)) is None:
# raise BadRequest("only images can be uploaded")
pass
from datetime import timedelta
from werkzeug.exceptions import PreconditionFailed, InternalServerError, BadRequest
from flask import g, request
import lxml
from lxml.html.clean import Cleaner
from hackathon.database import Hackathon, User, AdminHackathonRel, DockerHostServer, HackathonLike, \
HackathonStat, HackathonConfig, HackathonTag, UserHackathonRel, HackathonOrganizer, Award, UserHackathonAsset, \
UserTeamRel
from hackathon.hackathon_response import internal_server_error, ok, not_found, forbidden
from hackathon.constants import HACKATHON_BASIC_INFO, ADMIN_ROLE_TYPE, HACK_STATUS, RGStatus, HTTP_HEADER, \
FILE_TYPE, HACK_TYPE, HACKATHON_STAT, DockerHostServerStatus
from hackathon import RequiredFeature, Component, Context
docker_host_manager = RequiredFeature("docker_host_manager")
__all__ = ["HackathonManager"]
util = RequiredFeature("util")
class HackathonManager(Component):
"""Component to manage hackathon
Note that it only handle operations directly related to Hackathon table. Things like registerd users, templates are
in separated components
"""
admin_manager = RequiredFeature("admin_manager")
user_manager = RequiredFeature("user_manager")
register_manager = RequiredFeature("register_manager")
class HostedDockerFormation(Component):
hackathon_template_manager = RequiredFeature("hackathon_template_manager")
hackathon_manager = RequiredFeature("hackathon_manager")
expr_manager = RequiredFeature("expr_manager")
"""
Docker resource management based on docker remote api v1.18
Host resource are required. Azure key required in case of azure.
"""
application_json = {'content-type': 'application/json'}
docker_host_manager = RequiredFeature("docker_host_manager")
def __init__(self):
self.lock = Lock()
def report_health(self):
"""Report health of DockerHostServers
:rtype: dict
:return health status item of docker. OK when all servers running, Warning if some of them working,
Error if no server running
"""
try:
# TODO skip hackathons that are offline or ended
hosts = self.db.find_all_objects(DockerHostServer)
alive = 0
for host in hosts:
if self.ping(host):
alive += 1
if alive == len(hosts):
return {HEALTH.STATUS: HEALTH_STATUS.OK}
elif alive > 0:
return {
HEALTH.STATUS: HEALTH_STATUS.WARNING,
HEALTH.DESCRIPTION:
'at least one docker host servers are down'
}
else:
return {
HEALTH.STATUS: HEALTH_STATUS.ERROR,
HEALTH.DESCRIPTION: 'all docker host servers are down'
}
except Exception as e:
return {
HEALTH.STATUS: HEALTH_STATUS.ERROR,
HEALTH.DESCRIPTION: e.message
}
def create_container(self, docker_host, container_config, container_name):
"""
only create a container, in this step, we cannot start a container.
:param docker_host:
:param container_config:
:param container_name:
:return:
"""
containers_url = '%s/containers/create?name=%s' % (
self.__get_vm_url(docker_host), container_name)
req = requests.post(containers_url,
data=json.dumps(container_config),
headers=self.application_json)
self.log.debug(req.content)
# todo check the http code first
container = json.loads(req.content)
if container is None:
raise AssertionError("container is none")
return container
def start_container(self, docker_host, container_id):
"""
start a container
:param docker_host:
:param container_id:
:return:
"""
url = '%s/containers/%s/start' % (self.__get_vm_url(docker_host),
container_id)
req = requests.post(url, headers=self.application_json)
self.log.debug(req.content)
def stop_container(self, host_server, container_name):
"""
delete a container
:param name:
:param docker_host:
:return:
"""
containers_url = '%s/containers/%s?force=1' % (
self.__get_vm_url(host_server), container_name)
req = requests.delete(containers_url)
return req
def pull_image(self, context):
# todo fix pull_image?
docker_host_id, image_name, tag = context.docker_host, context.image_name, context.tag
docker_host = self.db.find_first_object_by(DockerHostServer,
id=docker_host_id)
if not docker_host:
return
pull_image_url = self.__get_vm_url(
docker_host
) + "/images/create?fromImage=" + image_name + '&tag=' + tag
self.log.debug(" send request to pull image:" + pull_image_url)
return requests.post(pull_image_url)
def get_pulled_images(self, docker_host):
get_images_url = self.__get_vm_url(docker_host) + "/images/json?all=0"
current_images_info = json.loads(
requests.get(get_images_url).content) # [{},{},{}]
current_images_tags = map(lambda x: x['RepoTags'],
current_images_info) # [[],[],[]]
return flatten(current_images_tags) # [ imange:tag, image:tag ]
def ensure_images(self):
hackathons = self.hackathon_manager.get_online_hackathons()
map(lambda h: self.__ensure_images_for_hackathon(h), hackathons)
def is_container_running(self, docker_container):
"""check container's running status on docker host
if status is Running or Restarting returns True , else returns False
:type docker_container: DockerContainer
:param docker_container: the container that you want to check
:type: bool
:return True: the container running status is running or restarting , else returns False
"""
docker_host = docker_container.host_server
if docker_host:
container_info = self.__get_container_info_by_container_id(
docker_host, docker_container.container_id)
if container_info is None:
return False
return container_info['State']['Running'] or container_info[
'State']['Restarting']
else:
return False
def ping(self, docker_host, timeout=20):
"""Ping docker host to check running status
:type docker_host : DockerHostServer
:param docker_host: the hots that you want to check docker service running status
:type: bool
:return: True: running status is OK, else return False
"""
try:
ping_url = '%s/_ping' % self.__get_vm_url(docker_host)
req = requests.get(ping_url, timeout=timeout)
return req.status_code == 200 and req.content == 'OK'
except Exception as e:
self.log.error(e)
return False
def get_containers_detail_by_ve(self, virtual_environment):
"""Get all containers' detail from "Database" filtered by related virtual_environment
:rtype: dict
:return: get the info of all containers
"""
container = virtual_environment.docker_container
if container:
return self.util.make_serializable(container.to_mongo().to_dict())
return {}
def list_containers(self, docker_host, timeout=20):
"""
return: json(as list form) through "Docker restful API"
"""
containers_url = '%s/containers/json' % self.__get_vm_url(docker_host)
req = requests.get(containers_url, timeout=timeout)
self.log.debug(req.content)
return self.util.convert(json.loads(req.content))
def get_container_by_name(self, container_name, docker_host):
containers = self.list_containers(docker_host)
return next(
(c for c in containers if container_name in c["Names"] or '/' +
container_name in c["Names"]), None)
# --------------------------------------------- helper function ---------------------------------------------#
def __get_vm_url(self, docker_host):
return 'http://%s:%d' % (docker_host.public_dns,
docker_host.public_docker_api_port)
def __get_schedule_job_id(self, hackathon):
return "pull_images_for_hackathon_%s" % hackathon.id
def __ensure_images_for_hackathon(self, hackathon):
# only ensure those alauda is disabled
if hackathon.config.get(
HACKATHON_CONFIG.CLOUD_PROVIDER) == CLOUD_PROVIDER.ALAUDA:
self.log.debug(
"schedule job of hackathon '%s(%d)' removed for alauda enabled"
% (hackathon.name, hackathon.id))
self.scheduler.remove_job(self.__get_schedule_job_id(hackathon))
return
job_id = self.__get_schedule_job_id(hackathon)
job_exist = self.scheduler.has_job(job_id)
if hackathon.event_end_time < self.util.get_now():
if job_exist:
self.scheduler.remove_job(job_id)
return
else:
if job_exist:
self.log.debug("job %s existed" % job_id)
else:
self.log.debug(
"adding schedule job to ensure images for hackathon %s" %
hackathon.name)
next_run_time = self.util.get_now() + timedelta(seconds=3)
context = Context(hackathon_id=hackathon.id)
self.scheduler.add_interval(
feature="hackathon_template_manager",
method="pull_images_for_hackathon",
id=job_id,
context=context,
next_run_time=next_run_time,
minutes=60)
def __get_container_info_by_container_id(self, docker_host, container_id):
"""get a container info by container_id from a docker host
:param: the docker host which you want to search container from
:type container_id: str|unicode
:param as a parameter that you want to search container though docker remote API
:return dic object of the container info if not None
"""
try:
get_container_url = self.__get_vm_url(
docker_host) + "/containers/%s/json?all=0" % container_id
req = requests.get(get_container_url)
if 300 > req.status_code >= 200:
container_info = json.loads(req.content)
return container_info
return None
except Exception as ex:
self.log.error(ex)
return None
class TeamManager(Component):
"""Component to manage hackathon teams"""
user_manager = RequiredFeature("user_manager")
admin_manager = RequiredFeature("admin_manager")
register_manager = RequiredFeature("register_manager")
hackathon_template_manager = RequiredFeature("hackathon_template_manager")
def get_team_by_id(self, team_id):
team = self.__get_team_by_id(team_id)
# check whether it's anonymous user or not
user = None
if self.user_manager.validate_token():
user = g.user
if team:
# TODO: refine: dereference member users is not necessary
return self.__team_detail(team, user)
else:
return not_found()
def get_my_current_team(self, hackathon, user):
team = self.__get_valid_team_by_user(user.id, hackathon.id)
return self.__team_detail(team, user) if team else not_found(
"user has no team", friendly_message="组队异常，请联系管理员!")
def get_team_by_name(self, hackathon_id, team_name):
""" get user's team basic information stored on table 'team' based on team name
:type hackathon_id: int
:param hackathon_id: id of hackathon related to the team
:type team_name: str | unicode
:param team_name: name of the team
:rtype: dict
:return: team's information as a dict if team is found otherwise not_found()
"""
team = self.__get_team_by_name(hackathon_id, team_name)
# check whether it's anonymous user or not
user = None
if self.user_manager.validate_token():
user = g.user
if team:
return self.__team_detail(team, user)
else:
return not_found("no such team")
def get_team_members(self, team_id):
"""Get team member list of specific team
:rtype: dict
:return: team's information and team's members list if team is found otherwise not_found()
"""
try:
team = Team.objects(id=team_id).first()
except ValidationError:
return None
if not team:
return None
def sub(t):
m = to_dic(t)
m["user"] = self.user_manager.user_display_info(t.user)
return m
return [sub(t) for t in team.members]
def get_hackathon_team_list(self, hackathon_id, name=None, number=None):
"""Get the team list of selected hackathon
:type hackathon_id: string or object_id
:param hackathon_id: hackathon id
:type name: str|unicode
:param name: name of team. optional
:type number: int
:param number: querying condition, return number of teams
:rtype: list
:return: a list of team filter by name and number on selected hackathon
"""
query = Q(hackathon=hackathon_id)
if name is not None:
query &= Q(name__icontains=name)
try:
teams = Team.objects(query).order_by('name')[:number]
except ValidationError:
return []
# check whether it's anonymous user or not
user = None
if self.user_manager.validate_token():
user = g.user
def get_team(team):
teamDic = team.dic()
teamDic['leader'] = {
'id': str(team.leader.id),
'name': team.leader.name,
'nickname': team.leader.nickname,
'avatar_url': team.leader.avatar_url
}
teamDic['cover'] = teamDic.get('cover', '')
teamDic['project_name'] = teamDic.get('project_name', '')
teamDic['dev_plan'] = teamDic.get('dev_plan', '')
teamDic['works'] = teamDic.get('works', '')
[
teamDic.pop(key, None) for key in
['assets', 'azure_keys', 'scores', 'templates', 'hackathon']
]
teamDic["member_count"] = team.members.filter(
status=TEAM_MEMBER_STATUS.APPROVED).count()
def sub(t):
m = to_dic(t)
m["user"] = self.user_manager.user_display_info(t.user)
return m
teamDic["members"] = [sub(t) for t in team.members]
return teamDic
return [get_team(x) for x in teams]
def create_default_team(self, hackathon, user):
"""Create a default new team for user after registration.
Use user name as team name by default. Append user id in case user name is duplicate
"""
user_team = self.__get_valid_team_by_user(user.id, hackathon.id)
if user_team:
self.log.debug(
"fail to create team since user is already in some team.")
return precondition_failed("you must leave the current team first")
team_name = self.__generate_team_name(hackathon, user)
team_member = TeamMember(join_time=self.util.get_now(),
status=TEAM_MEMBER_STATUS.APPROVED,
user=user)
team = Team(name=team_name,
leader=user,
logo=user.avatar_url,
hackathon=hackathon,
members=[team_member])
team.save()
return team.dic()
def update_team(self, kwargs):
"""Update existing team information
:type kwargs: dict
:param kwargs: a dict to store update information for team
:rtype: dict
:return: updated team information in a dict
"""
team = self.__get_team_by_id(kwargs["id"])
if not team:
return not_found("team not exists")
# avoid duplicate team with same names
if "name" in kwargs and kwargs["name"] != team.name:
if self.__get_team_by_name(g.hackathon.id, kwargs["name"]):
return precondition_failed(
"team with the same name exists already")
self.__validate_team_permission(g.hackathon.id, team, g.user)
# hackathon.modify(**update_items)
# team.name = kwargs.get("name", team.name)
# team.description = kwargs.get("description", team.description)
# team.logo = kwargs.get("logo", team.logo)
kwargs.pop('id', None) # id should not be included
team.modify(**kwargs)
team.update_time = self.util.get_now()
team.save()
if "dev_plan" in kwargs and kwargs["dev_plan"] and not kwargs["dev_plan"] == "" \
and team.hackathon.config.get(HACKATHON_CONFIG.DEV_PLAN_REQUIRED, False):
t = threading.Thread(target=self.__email_notify_dev_plan_submitted,
args=(team, ))
t.setDaemon(True)
t.start()
return self.__team_detail(team)
def dismiss_team(self, operator, team_id):
"""Dismiss a team by team leader or hackathon admin
:rtype: bool
:return: if dismiss success, return ok. if not ,return bad request.
"""
team = self.__get_team_by_id(team_id)
if not team:
return ok()
hackathon = team.hackathon
self.__validate_team_permission(hackathon.id, team, operator)
members = team.members
member_users = [m.user for m in members]
# TODO: transcation?
team.delete()
for u in member_users:
self.create_default_team(hackathon, u)
return ok()
def quit_team_forcedly(self, team, user):
"""
The operator(admin or superadmin) forces a user(team leader or other members) to quit a team.
If the user is the only member of the team, the team will be deleted.
Else if the user is the leader of a team with several members, the team will be decomposed into several
new teams.
Else if the user is not the leader of a team with several members, just the user quits the team.
:rtype: bool
:return: if dismiss success, return ok. if not ,return bad request.
"""
# here we don't check whether the operator has the permission,
if not team.members or len(team.members) == 0:
self.log.warn("this team doesn't have any members")
return ok()
member_users = [
m.user for m in team.members
if m.status == TEAM_MEMBER_STATUS.APPROVED
]
num_team_members = len(member_users)
hackathon = team.hackathon
if num_team_members > 1:
if team.leader == user:
team.delete()
for u in member_users:
if u.id != user.id:
self.create_default_team(hackathon, u)
else:
Team.objects(id=team.id).update_one(pull__members__user=user)
else:
# num_team_members == 1
team.delete()
return ok()
def join_team(self, user, team_id):
"""Join a team will create a record on user_team_rel table which status will be 0.
:type user: User
:rtype: dict
:return: if user already joined team or team not exist, return bad request. Else, return a dict of joined
details.
"""
if Team.objects(id=team_id, members__user=user.id).count():
return ok("You already joined this team.")
team = self.__get_team_by_id(team_id)
if not team:
return not_found()
cur_team = self.__get_valid_team_by_user(user.id, team.hackathon.id)
if cur_team and cur_team.members.count() > 1:
return precondition_failed(
"Team leader cannot join another team for team member count greater than 1"
)
if not self.register_manager.is_user_registered(
user.id, team.hackathon):
return precondition_failed("user not registerd")
mem = TeamMember(join_time=self.util.get_now(),
status=TEAM_MEMBER_STATUS.INIT,
user=user)
team.members.append(mem)
team.save()
return to_dic(mem)
def update_team_member_status(self, operator, team_id, user_id, status):
""" update user's status on selected team. if current user doesn't have permission, return bad request.
Else, update user's status
:type status: int
:param status: the status of the team member, see TEAM_MEMBER_STATUS in constants.py
:rtype: bool
:return: if update success, return ok. if not , return bad request.
"""
team = self.__get_team_by_id(team_id)
if not team:
return not_found()
mem = [x for x in team.members if str(x.user.id) == user_id]
assert len(mem) < 2
if not mem:
return not_found()
mem = mem[0]
# #NOTE1# we have to re-check this here
# because of this situation:
# A is in a single-person team TeamA, and request join TeamB
# after that, C join TeamA and now TeamA has two members,
# this is not allowed when status == TEAM_MEMBER_STATUS.APPROVED
cur_team = self.__get_valid_team_by_user(mem.user.id,
team.hackathon.id)
if cur_team and cur_team.members.count() > 1:
return precondition_failed(
"Team leader cannot join another team for team member count greater than 1"
)
self.__validate_team_permission(team.hackathon.id, team, operator)
if mem.user.id == team.leader.id:
return precondition_failed("cannot update status of team leader")
if status == TEAM_MEMBER_STATUS.APPROVED:
# disable previous team first
# NOTE:
# Do we also have to delete status that is not TEAM_MEMBER_STATUS.APPROVED?
# i.e., if A request join both TeamB and TeamC, TeamC approve join first, then TeamB approved,
# this will cause A leave TeamB and join TeamC.
# is this the desired behaviour?
Team.objects(hackathon=team.hackathon.id).update(
__raw__={
"$pull": {
"members": {
"user": user_id,
"status": TEAM_MEMBER_STATUS.APPROVED
}
}
})
# because only team leader with single team can make join request
# so we don't have to make default team for other members in this team
# we make the check in #NOTE1# so this is always true
Team.objects(hackathon=team.hackathon.id,
leader=mem.user.id).delete()
mem.status = TEAM_MEMBER_STATUS.APPROVED
mem.update_time = self.util.get_now()
team.save()
return ok("approved")
if status == TEAM_MEMBER_STATUS.DENIED:
user = mem.user
hackathon = team.hackathon
team.members.remove(mem)
team.save()
self.create_default_team(hackathon, user)
return ok(
"Your request has been denied, please rejoin another team.")
def kick_or_leave(self, operator, team_id, user_id):
try:
team = Team.objects(id=team_id, members__user=user_id).first()
except ValidationError:
return not_found()
if not team:
return not_found()
mem = [x for x in team.members if str(x.user.id) == user_id]
assert len(mem) < 2
if not mem:
return not_found()
mem = mem[0]
hackathon = team.hackathon
user = mem.user
if str(
team.leader.id
) == user_id: # if the user to be leaved or kicked is team leader
return precondition_failed("leader cannot leave team")
if str(operator.id) == user_id: # leave team
team.members.remove(mem)
team.save()
self.create_default_team(hackathon, user)
else: # kick somebody else
self.__validate_team_permission(hackathon.id, team, operator)
team.members.remove(mem)
team.save()
self.create_default_team(hackathon, user)
return ok()
def add_template_for_team(self, args):
"""Add template to team of the current user by template name
template_id must be included in args. Current login user must have a team and HE must be its leader
"""
if "template_id" not in args:
return bad_request("template id invalid")
team = self.__get_valid_team_by_user(g.user.id, g.hackathon.id)
if not team:
return precondition_failed(
"you don't join any team so you cannot add teamplate")
if team.leader.id != g.user.id:
return forbidden("team leader required")
else:
return self.hackathon_template_manager.add_template_to_hackathon(
args["template_id"])
def delete_template_from_team(self, template_id):
"""Delete template from current user's team
Team should exist and current login user must be the leader
"""
team = self.__get_valid_team_by_user(g.user.id, g.hackathon.id)
if not team:
return precondition_failed(
"you don't join any team so you cannot add teamplate")
if team.leader.id != g.user.id:
return forbidden("team leader required")
else:
return self.hackathon_template_manager.delete_template_from_hackathon(
template_id)
def get_team_by_user_and_hackathon(self, user, hackathon):
team = Team.objects(hackathon=hackathon, members__user=user).first()
return team
def score_team(self, judge, ctx):
team = self.__get_team_by_id(ctx.team_id)
if not team:
return not_found("team not found")
if not self.admin_manager.is_hackathon_admin(team.hackathon.id,
judge.id):
return forbidden()
score = [x for x in team.scores if x.judge.id == judge.id]
assert len(score) < 2
if score:
score = score[0]
score.score = ctx.score
score.reason = ctx.get("reason")
score.update_time = self.util.get_now()
else:
score = TeamScore(score=ctx.score,
judge=judge,
reason=ctx.get("reason"))
team.scores.append(score)
team.save()
return self.__response_get_score(judge, team.scores)
def get_score(self, user, team_id):
team = self.__get_team_by_id(team_id)
if not team:
return not_found("team not found")
if not self.admin_manager.is_hackathon_admin(team.hackathon.id,
user.id):
return {}
return self.__response_get_score(user, team.scores)
def __response_get_score(self, user, scores):
resp = {"all": [to_dic(s) for s in scores]}
my = [sc for sc in scores if sc.judge.id == user.id]
assert len(my) < 2
if my:
resp["my"] = to_dic(my[0])
return resp
def add_team_show(self, user, context):
team = self.__get_team_by_id(context.team_id)
if not team:
return not_found()
self.__validate_team_permission(team.hackathon.id, team, user)
try:
work = TeamWork(id=uuid.uuid1(),
description=context.get("note"),
type=context.type,
uri=context.uri)
team.works.append(work)
team.save()
except ValidationError as e:
if "uri" in e.message:
return bad_request("`uri` field must be in uri format")
else:
raise e
return to_dic(work)
def delete_team_show(self, user, show_id):
try:
team = Team.objects(works__id=show_id).first()
except (ValidationError, ValueError):
return not_found("wrong id format")
if team:
self.__validate_team_permission(team.hackathon.id, team, user)
for i in range(len(team.works)):
if str(team.works[i].id) == show_id:
team.works.pop(i)
team.save()
break
return ok()
def get_team_show_list(self, team_id):
team = self.__get_team_by_id(team_id)
if not team:
return []
return [to_dic(s) for s in team.works]
def get_hackathon_show_list(self, hackathon_id, show_type=None, limit=6):
query = Q(hackathon=hackathon_id)
if show_type is not None:
query &= Q(works__type=int(show_type))
works = []
for team in Team.objects(query).filter(works__1__exists=True).order_by(
'update_time', '-age')[:limit]:
teamDic = team.dic()
teamDic['leader'] = {
'id': str(team.leader.id),
'name': team.leader.name,
'nickname': team.leader.nickname,
'avatar_url': team.leader.avatar_url
}
teamDic['cover'] = teamDic.get('cover', '')
teamDic['project_name'] = teamDic.get('project_name', '')
teamDic['dev_plan'] = teamDic.get('dev_plan', '')
[
teamDic.pop(key, None) for key in [
'assets', 'awards', 'azure_keys', 'scores', 'templates',
'members'
]
]
#
# teamDic['works'] = []
#
# for work in team.works:
# teamDic['works'].append(to_dic(work))
works.append(teamDic)
# works.sort(lambda a, b: int(b["create_time"] - a["create_time"]))
# def proc_work(w):
# w.pop("create_time")
# w["id"] = str(w["id"])
# w["team_id"] = str(w["team_id"])
# w["hackathon_id"] = str(w["hackathon_id"])
# return w
return works
def get_team_show_list_by_user(self, user_id):
teams = Team.objects(members__match={
"user": user_id,
"status": TEAM_MEMBER_STATUS.APPROVED
}).all()
def get_team_show_detail(team):
dic = self.__team_detail(team)
dic["hackathon"] = team.hackathon.dic()
return dic
return [
get_team_show_detail(team) for team in teams
if not len(team.works) == 0
]
def get_team_source_code(self, team_id):
try:
team = Team.objects(id=team_id,
works__type=TEAM_SHOW_TYPE.SOURCE_CODE)
except ValidationError:
return None
if not team:
return None
return [w for w in team.works
if w.type == TEAM_SHOW_TYPE.SOURCE_CODE][0]
def query_team_awards(self, team_id):
team = self.__get_team_by_id(team_id)
if not team:
return []
awards = [
self.__award_with_detail(r, hackathon=team.hackathon)
for r in team.awards
]
awards.sort(lambda a, b: b.level - a.level)
return awards
def get_granted_awards(self, hackathon):
awards = []
team_id_with_awards = []
for team in Team.objects(hackathon=hackathon):
awards += team.awards
if not len(team.awards) == 0:
team_id_with_awards.append(team.id)
awards = [self.__award_with_detail(r) for r in awards]
awards.sort(lambda a, b: b["level"] - a["level"])
# find teams who are granted these awards
for award in awards:
award["team"] = []
for team_id in team_id_with_awards:
team = Team.objects(id=team_id).first()
if uuid.UUID(award["id"]) in team.awards:
award["team"].append(team.dic())
# len(awards) is equal to the number of all awards granted, so it's duplicated, remove duplicated items in JS.
return awards
def get_all_granted_awards(self, limit):
teams = Team.objects().all()
teams_with_awards = [team for team in teams if not team.awards == []]
teams_with_awards.sort(
key=lambda t:
(t.hackathon.id,
Hackathon.objects(id=t.hackathon.id, awards__id=t.awards[0]).
first().awards.get(id=t.awards[0]).level),
reverse=True) # sort by hackathon and then sort by award level.
teams_with_awards = teams_with_awards[0:int(limit)]
return [
self.__get_hackathon_and_show_detail(team)
for team in teams_with_awards
]
def grant_award_to_team(self, hackathon, context):
team = self.__get_team_by_id(context.team_id)
if not team:
return not_found("team not found")
award = [a for a in hackathon.awards if str(a.id) == context.award_id]
assert len(award) < 2
if not award:
return not_found("award not found")
award = award[0]
if team.hackathon.id != hackathon.id:
return precondition_failed("hackathon doesn't match")
team_award = [a for a in team.awards if str(a) == context.award_id]
assert len(team_award) < 2
if not team_award:
team.awards.append(uuid.UUID(context.award_id))
team.save()
return self.__award_with_detail(context.award_id)
def cancel_team_award(self, hackathon, team_id, award_id):
team = self.__get_team_by_id(team_id)
if not team:
return not_found()
for award in team.awards:
if str(award) == award_id:
team.awards.remove(award)
team.save()
break
return ok()
def send_email_azure(self, kwargs):
# team information
team = self.__get_team_by_id(kwargs["id"])
if not team:
return not_found("team not exists")
azure = team.azure
if not azure.strip():
if Azure.objects(status="0").count() == 0:
return ok("请联系管理员.")
azure_info = Azure.objects(status="0").first()
else:
azure_info = Azure.objects(account=azure).first()
if not azure_info:
return ok("请联系管理员!")
primary_emails = []
for i in range(0, len(team.members)):
mem = team.members[i]
resp = self.user_manager.user_display_info(mem.user)
primary_emails.append(resp['emails'][0]['email'])
Azure.objects(account=azure_info.account).update_one(status="1")
Team.objects(id=team.id).update_one(azure=azure_info.account)
sender = ''
email_title = ''
email_content = ''
return self.util.send_emails(sender, primary_emails, email_title,
email_content)
def __init__(self):
pass
def __award_with_detail(self, team_award, hackathon=None):
if not hackathon:
hackathon = g.hackathon
try:
award = [
a for a in hackathon.awards if str(a.id) == str(team_award)
][0]
except IndexError:
return None
return to_dic(award)
def __team_detail(self, team, user=None):
resp = team.dic()
resp["leader"] = self.user_manager.user_display_info(team.leader)
resp["member_count"] = team.members.filter(
status=TEAM_MEMBER_STATUS.APPROVED).count()
# all team action not allowed if frozen
resp["is_frozen"] = False
for i in range(0, len(team.members)):
mem = team.members[i]
resp["members"][i]["user"] = self.user_manager.user_display_info(
mem.user)
if user:
resp["is_admin"] = self.admin_manager.is_hackathon_admin(
team.hackathon.id, user.id)
resp["is_leader"] = team.leader == user
rel = team.members.filter(user=user)
resp["is_member"] = True if not rel == [] else False
return resp
def __generate_team_name(self, hackathon, user):
"""Generate a default team name by user name. It can be updated later by team leader"""
team_name = user.name
if Team.objects(hackathon=hackathon, name=team_name).first():
team_name = "%s (%s)" % (user.name, user.id)
return team_name
def __get_user_teams(self, user_id):
"""Get all teams of specific and related hackathon display info
:type user_id: int
:param user_id: User id to get teams. Cannot be None
:rtype: list
:return list of all teams as well as hackathon info
"""
return Team.objects(members__user=user_id).all()
def __get_team_by_id(self, team_id):
"""Get team by its primary key"""
try:
return Team.objects(id=team_id).first()
except ValidationError:
return None
def __get_valid_team_by_user(self, user_id, hackathon_id):
"""Get valid Team(Mongo-document) by user and hackathon
"valid" means user is approved. There might be other records where status=Init
Since foreign keys are defined in Team, one can access team or user through the return result directly
:rtype: Team
:return instance of Team
"""
return Team.objects(hackathon=hackathon_id,
members__match={
"user": user_id,
"status": TEAM_MEMBER_STATUS.APPROVED
}).first()
def __get_team_by_name(self, hackathon_id, team_name):
""" get user's team basic information stored on table 'team' based on team name
:type hackathon_id: int
:param hackathon_id: hackathon id for the team
:type team_name: str|unicode
:param team_name: name of the team
:rtype: Team
:return: instance of Team if team found otherwise None
"""
try:
return Team.objects(hackathon=hackathon_id, name=team_name).first()
except ValidationError:
return None
def __validate_team_permission(self, hackathon_id, team, user):
"""Validate current login user whether has proper right on specific team.
:type hackathon_id: int
:param hackathon_id: id of hackathon related to the team
:type team: Team
:param team: team to be checked
:type user: User
:param user: current login user
:raise: Forbidden if user is neither team leader, hackathon admin nor super admin
"""
self.log.debug(
"validate team permission on hackathon %s and team %s for user %s"
% (hackathon_id, team.name, user.id))
# check if team leader
if team.leader.id != user.id:
# check if hackathon admin
if not self.admin_manager.is_hackathon_admin(
hackathon_id, user.id):
# super permission is already checked in admin_manager.is_hackathon_admin
self.log.debug(
"Access denied for user [%s]%s trying to access team '%s' of hackathon %s "
% (user.id, user.name, team, hackathon_id))
raise Forbidden(
description="You don't have permission on team '%s'" %
team.name)
return
def __get_hackathon_and_show_detail(self, team):
team_dic = team.dic()
team_dic['leader'] = {
'id': str(team.leader.id),
'name': team.leader.name,
'nickname': team.leader.nickname,
'avatar_url': team.leader.avatar_url
}
team_dic['cover'] = team_dic.get('cover', '')
team_dic['project_name'] = team_dic.get('project_name', '')
team_dic['dev_plan'] = team_dic.get('dev_plan', '')
[
team_dic.pop(key, None) for key in [
'assets', 'awards', 'azure_keys', 'scores', 'templates',
'members'
]
]
team_dic["hackathon"] = hack_manager.get_hackathon_detail(
team.hackathon)
return team_dic
def __email_notify_dev_plan_submitted(self, team):
# send emails to all admins of this hackathon when one team dev plan is submitted.
admins = UserHackathon.objects(
hackathon=team.hackathon,
role=HACK_USER_TYPE.ADMIN).distinct("user")
email_title = self.util.safe_get_config(
"email.email_templates.dev_plan_submitted_notify.title", None)
file_name = self.util.safe_get_config(
"email.email_templates.dev_plan_submitted_notify.default_file_name",
None)
sender = self.util.safe_get_config("email.default_sender", "")
# todo remove receivers_forced
receivers_forced = self.util.safe_get_config("email.receivers_forced",
[])
try:
if email_title and file_name:
path = abspath("%s/.." % dirname(realpath(__file__)))
f = open(path + "/resources/email/" + file_name, "r")
email_content = f.read()
email_title = email_title % (team.name.encode("utf-8"))
email_content = email_content.replace(
"{{team_name}}", team.name.encode("utf-8"))
email_content = email_content.replace("{{team_id}}",
str(team.id))
email_content = email_content.replace(
"{{hackathon_name}}", team.hackathon.name.encode("utf-8"))
f.close()
else:
self.log.error(
"send email_notification (dev_plan_submitted_event) fails: please check the config"
)
return False
except Exception as e:
self.log.error(e)
return False
# isNotified: whether at least one admin has been notified by emails.
isNotified = False
for admin in admins:
isSent = False
primary_emails = [
email.email for email in admin.emails if email.primary_email
]
nonprimary_emails = [
email.email for email in admin.emails
if not email.primary_email
]
# send notification to all primary-mailboxes.
if not len(primary_emails) == 0:
isSent = self.util.send_emails(sender, primary_emails,
email_title, email_content)
# if fail to send emails to primary-mailboxes, sent email to one non-primary mailboxes.
if not isSent and not len(nonprimary_emails) == 0:
for nonpri_email in nonprimary_emails:
if self.util.send_emails(sender, [nonpri_email],
email_title, email_content):
isSent = True
break
isNotified = isNotified or isSent
# todo remove this code
self.util.send_emails(sender, receivers_forced, email_title,
email_content)
self.log.debug(team.name + ": dev_plan email notification result: " +
str(isNotified))
return isNotified
sys.path.append("..")
import uuid
import time
from flask import g
import threading
from mongoengine import Q, ValidationError
from os.path import realpath, abspath, dirname
from hackathon import Component, RequiredFeature
from hackathon.hmongo.models import Team, TeamMember, TeamScore, TeamWork, Hackathon, UserHackathon, to_dic
from hackathon.hackathon_response import not_found, bad_request, precondition_failed, ok, forbidden
from hackathon.constants import TEAM_MEMBER_STATUS, TEAM_SHOW_TYPE, HACK_USER_TYPE, HACKATHON_CONFIG
__all__ = ["TeamManager"]
hack_manager = RequiredFeature("hackathon_manager")
class TeamManager(Component):
"""Component to manage hackathon teams"""
user_manager = RequiredFeature("user_manager")
admin_manager = RequiredFeature("admin_manager")
register_manager = RequiredFeature("register_manager")
hackathon_template_manager = RequiredFeature("hackathon_template_manager")
def get_team_by_id(self, team_id):
team = self.__get_team_by_id(team_id)
# check whether it's anonymous user or not
user = None
if self.user_manager.validate_token():
class AlaudaDockerStarter(DockerExprStarter):
docker = RequiredFeature("alauda_docker_proxy")
def _get_docker_proxy(self):
return self.docker
class ExprStarter(Component):
"""Base for experiment starter"""
template_library = RequiredFeature("template_library")
def start_expr(self, context):
"""To start a new Experiment asynchronously
:type context: Context
:param context: the execution context.
"""
expr = Experiment(status=ExprStatus.INIT,
template=context.template,
user=context.user,
virtual_environments=[],
hackathon=context.hackathon)
expr.save()
template_content = self.template_library.load_template(context.template)
expr.status = ExprStatus.STARTING
expr.save()
# context contains complex object, we need create another serializable one with only simple fields
new_context = Context(template_content=template_content,
template_name=context.template.name,
hackathon_id=context.hackathon.id,
experiment_id=expr.id,
pre_alloc_enabled = context.pre_alloc_enabled)
if context.get("user", None):
new_context.user_id = context.user.id
if self._internal_start_expr(new_context):
new_context.experiment = expr
return new_context
self.rollback(new_context)
return None
def stop_expr(self, context):
"""Stop experiment asynchronously
:type context: Context
:param context: the execution context.
"""
return self._internal_stop_expr(context)
def rollback(self, context):
"""cancel/rollback a expr which is in error state
:type context: Context
:param context: the execution context.
"""
return self._internal_rollback(context)
def _internal_start_expr(self, context):
raise NotImplementedError()
def _internal_stop_expr(self, context):
raise NotImplementedError()
def _internal_rollback(self, context):
raise NotImplementedError()
def _on_virtual_environment_failed(self, context):
self.rollback(context)
def _on_virtual_environment_success(self, context):
expr = Experiment.objects(id=context.experiment_id).no_dereference() \
.only("status", "virtual_environments").first()
if all(ve.status == ExprEnvStatus.RUNNING for ve in expr.virtual_environments):
expr.status = ExprStatus.RUNNING
expr.save()
self._on_expr_started(context)
self._hooks_on_virtual_environment_success(context)
def _on_virtual_environment_stopped(self, context):
expr = Experiment.objects(id=context.experiment_id).no_dereference() \
.only("status", "virtual_environments").first()
ve = expr.virtual_environments.get(name=context.virtual_environment_name)
ve.status = ExprEnvStatus.STOPPED
if all(ve.status == ExprEnvStatus.STOPPED for ve in expr.virtual_environments):
expr.status = ExprStatus.STOPPED
expr.save()
def _on_virtual_environment_unexpected_error(self, context):
self.log.warn("experiment unexpected error: " + context.experiment_id)
expr = Experiment.objects(id=context.experiment_id).no_dereference() \
.only("status", "virtual_environments").first()
if "virtual_environment_name" in context:
expr.virtual_environments.get(name=context.virtual_environment_name).status = ExprEnvStatus.UNEXPECTED_ERROR
expr.save()
def _hooks_on_virtual_environment_success(self, context):
pass
def _on_expr_started(self, context):
# send notice
pass
import sys
sys.path.append("..")
import time
from flask import g, request
from flask_restful import reqparse
from hackathon import RequiredFeature, Component
from hackathon.decorators import hackathon_name_required, token_required, admin_privilege_required
from hackathon.health import report_health
from hackathon.hackathon_response import bad_request, not_found
from .hackathon_resource import HackathonResource
hackathon_manager = RequiredFeature("hackathon_manager")
user_manager = RequiredFeature("user_manager")
user_profile_manager = RequiredFeature("user_profile_manager")
register_manager = RequiredFeature("register_manager")
hackathon_template_manager = RequiredFeature("hackathon_template_manager")
template_library = RequiredFeature("template_library")
team_manager = RequiredFeature("team_manager")
expr_manager = RequiredFeature("expr_manager")
admin_manager = RequiredFeature("admin_manager")
guacamole = RequiredFeature("guacamole")
docker_host_manager = RequiredFeature("docker_host_manager")
util = RequiredFeature("util")
"""Resources for OHP itself"""
