def get_total_status(request):
dcs = {}
with open(settings.LITE_DB_PATH, "rb") as fd:
dcs = json.load(fd)
datas = []
for key in dcs:
galaxy = sdk.GalaxySDK(dcs[key]["master"])
response = galaxy.get_real_time_status()
builder = http.ResponseBuilder()
status = pb2dict.protobuf_to_dict(response)
datas.append(status);
builder = http.ResponseBuilder()
return builder.ok(data = {"status":datas}).build_json()
def job_permission_wrapper(request, *args, **kwds):
builder = http.ResponseBuilder()
id = request.GET.get('id', None)
if not id:
return builder.error('id is required').build_json()
master_addr = request.GET.get('master', None)
if not master_addr:
return builder.error('master is required').build_json()
try:
galaxy_job_it = models.GalaxyJob.objects.filter(
job_id=int(id), group__galaxy_master=master_addr)[:1]
if not galaxy_job_it:
return builder.error("job %s does not exist in db" %
id).build_json()
member = models.GroupMember.objects.filter(
group__id=galaxy_job_it[0].group.id,
user_name=request.user.username)[:1]
if not member:
return builder.error("you have no permission to kill job %s" %
id).build_json()
request.job = galaxy_job_it[0]
return func(request, *args, **kwds)
except:
LOG.exception('fail to get permission')
return builder.error('fail to kill job %s' % id).build_json()
def api_auth_wrapper(request, *args, **kwds):
res = http.ResponseBuilder()
if request.user.is_authenticated():
return func(request, *args, **kwds)
return res.error("%s?service=%s" %
(settings.UUAP_CAS_SERVER, settings.MY_HOST),
status=-2).build_json()
def list_service(request):
"""
get current user's service list
"""
builder = http.ResponseBuilder()
master_addr = request.GET.get('master', None)
if not master_addr:
return builder.error('master is required').build_json()
groupmember_list = models.GroupMember.objects.filter(
user_name=request.user.username, group__galaxy_master=master_addr)
group_id_list = []
for group_m in groupmember_list:
group_id_list.append(group_m.group.id)
db_jobs = models.GalaxyJob.objects.filter(group__id__in=group_id_list)
jobs_dict = {}
for db_job in db_jobs:
jobs_dict[db_job.job_id] = db_job
client = wrapper.Galaxy(master_addr, settings.GALAXY_CLIENT_BIN)
status, jobs = client.list_jobs()
if not status:
return builder.error('fail to list jobs').build_json()
ret = []
for job in jobs:
if job.job_id not in jobs_dict and not request.user.is_superuser:
continue
job.trace = job.trace.__dict__
ret.append(job.__dict__)
return builder.ok(data=ret).build_json()
def get_conf(request):
builder = http.ResponseBuilder()
home = [{
"nodeName": "服务列表",
"href": "#service",
"nodestyle": ["level1"],
"children": []
}, {
"nodeName": "集群信息",
"href": "#cluster",
"nodestyle": ["level1"],
"children": []
}, {
"nodeName": "配置集群",
"href": "#setup",
"nodestyle": ["level1"],
"children": []
}]
service = [{
"nodeName":
"Tera",
"nodestyle": ["level1"],
"children": [{
"nodestyle": ["level2"],
"subpage": "status",
"nodeName": "服务状态",
"children": []
}]
}]
return builder.ok(data={'home': home, 'service': service}).build_json()
def get_task_status(request):
builder = http.ResponseBuilder()
id = request.GET.get('id', None)
agent = request.GET.get('agent', None)
master_addr = request.GET.get('master', None)
if not master_addr:
return builder.error('master is required').build_json()
galaxy = wrapper.Galaxy(master_addr, settings.GALAXY_CLIENT_BIN)
tasklist = []
if id:
status, tasklist = galaxy.list_task_by_job_id(id)
if not status:
return builder.error("fail to get task list")\
.build_json()
if agent:
status, tasklist = galaxy.list_task_by_host(agent)
if not status:
return builder.error("fail to get task list")\
.build_json()
statics = {"RUNNING": 0, "DEPLOYING": 0, "ERROR": 0}
for task in tasklist:
task['mem_used'] = str_pretty(task['mem_used'])
task['mem_limit'] = str_pretty(task['mem_limit'])
task['cpu_used'] = "%0.2f" % (task['cpu_used'])
if task['status'] in statics:
statics[task['status']] += 1
if task['agent_addr']:
task['path'] = "http://" + task['agent_addr'].split(
":")[0] + ":8181/data/galaxy/" + str(task['id'])
return builder.ok(data={
'needInit': False,
'taskList': tasklist,
'statics': statics
}).build_json()
def create_service(request):
"""
create a service
"""
builder = http.ResponseBuilder()
master_addr = request.GET.get('master', None)
if not master_addr:
return builder.error('master is required').build_json()
galaxy = wrapper.Galaxy(master_addr, settings.GALAXY_CLIENT_BIN)
try:
ret = helper.validate_init_service_group_req(request)
status, output = galaxy.create_task(
ret['name'],
ret['pkg_src'],
ret['start_cmd'],
ret['replicate_count'],
ret['memory_limit'] * 1024 * 1024 * 1024,
ret['cpu_share'],
deploy_step_size=ret['deploy_step_size'],
one_task_per_host=ret['one_task_per_host'],
restrict_tags=ret['tag'])
if not status:
return builder.error('fail create task').build_json()
return builder.ok().build_json()
except Exception as e:
return builder.error(str(e)).build_json()
def user_wrapper(request, *args, **kwds):
builder = http.ResponseBuilder()
user_id = request.GET.get("user", None) or request.POST.get(
"user", None)
if not user_id:
return builder.error("user id required").build_json()
request.user_id = user_id
return func(request, *args, **kwds)
def job_permission_required_wrapper(request, *args, **kwds):
res = http.ResponseBuilder()
job_id = request.GET.get('id', None)
if not job_id:
return res.error("id is required").build_json()
master = request.GET.get('master', None)
if not master:
return res.error('master is required').build_json()
def service_name_wrapper(req, *args, **kwds):
builder = http.ResponseBuilder()
service_name = req.GET.get('serviceName', None) or req.POST.get(
'serviceName', None)
if not service_name:
return builder.error("serviceName is required").build_json()
req.service_name = service_name
return func(req, *args, **kwds)
def render_tpl(req, context, tpl_path):
render = http.ResponseBuilder()
media_url = req.build_absolute_uri("/statics")
root_url = req.build_absolute_uri("/")
context["media_url"] = media_url
context["root_url"] = root_url
return render.set_content(context)\
.add_req(req)\
.build_tmpl(tpl_path)
def get_real_time_status(request):
galaxy = sdk.GalaxySDK(request.master)
response = galaxy.get_real_time_status()
builder = http.ResponseBuilder()
status = pb2dict.protobuf_to_dict(response)
return builder.ok(data={
"status": status,
"dc": request.data_center,
"master": request.master
}).build_json()
def kill_service(request):
builder = http.ResponseBuilder()
id = request.GET.get('id', None)
master_addr = request.GET.get('master', None)
try:
galaxy = wrapper.Galaxy(master_addr, settings.GALAXY_CLIENT_BIN)
galaxy.kill_job(int(id))
return builder.ok().build_json()
except:
LOG.exception("fail to kill job %s" % id)
return builder.error('fail to kill job %s' % id).build_json()
def task_group_id_wrapper(req, *args, **kwds):
builder = http.ResponseBuilder()
task_group_id = req.GET.get("taskGroupId", None) or req.POST.get(
"taskGroupId", None)
if not task_group_id:
return builder.error("taskGroupId is required").build_json()
try:
req.task_group_id = int(task_group_id)
except:
LOG.exception("fail to convert task group id %s" % task_group_id)
return builder.error("taskGroupId is invalide").build_json()
return func(req, *args, **kwds)
def kill_service(request):
builder = http.ResponseBuilder()
id = request.GET.get('id', None)
if not id:
return builder.error('id is required').build_json()
master_addr = request.GET.get('master', None)
if not master_addr:
return builder.error('master is required').build_json()
galaxy = wrapper.Galaxy(master_addr, settings.GALAXY_CLIENT_BIN)
galaxy.kill_job(int(id))
return builder.ok().build_json()
def service_id_wrapper(request, *args, **kwds):
builder = http.ResponseBuilder()
service_id = request.GET.get("service", None) or request.POST.get(
"service", None)
if not service_id:
return builder.error("service is required").build_json()
try:
request.service_id = int(service_id)
except Exception as e:
LOG.exception("fail to convert service to int")
return builder.error("fail to convert service to int for %s" % e)
return func(request, *args, **kwds)
def list_tag(req):
builder = http.ResponseBuilder()
master_addr = req.GET.get('master', None)
if not master_addr:
return builder.error('master is required').build_json()
galaxy = wrapper.Galaxy(master_addr, settings.GALAXY_CLIENT_BIN)
try:
ret = galaxy.list_tag()
return builder.ok(data=ret).build_json()
except:
return builder.error('fail to tag agent ').build_json()
def get_status(req):
builder = http.ResponseBuilder()
master_addr = req.GET.get('master', None)
if not master_addr:
return builder.error('master is required').build_json()
client = wrapper.Galaxy(master_addr, settings.GALAXY_CLIENT_BIN)
machine_list = client.list_node()
ret = []
total_node_num = 0
total_cpu_num = 0
total_cpu_allocated = 0
total_cpu_used = 0
total_mem_used = 0
total_mem_num = 0
total_mem_allocated = 0
total_task_num = 0
total_node_num = len(machine_list)
for machine in machine_list:
total_cpu_num += machine.cpu_share
total_mem_num += machine.mem_share
total_task_num += machine.task_num
total_cpu_allocated += machine.cpu_allocated
total_mem_allocated += machine.mem_allocated
total_mem_used += machine.mem_used
total_cpu_used += machine.cpu_used
machine.mem_share = str_pretty(machine.mem_share)
machine.mem_allocated = str_pretty(machine.mem_allocated)
machine.cpu_used = '%0.1f' % machine.cpu_allocated
machine.cpu_allocated = '%0.1f' % machine.cpu_allocated
ret.append(machine.__dict__)
mem_usage_p = 0
cpu_usage_p = 0
if total_mem_num:
mem_usage_p = "%0.1f" % (100 * total_mem_used / total_mem_num)
if total_cpu_num:
cpu_usage_p = "%0.1f" % (100 * total_cpu_used / total_cpu_num)
return builder.ok(
data={
'machinelist': ret,
'total_node_num': total_node_num,
'total_mem_allocated': str_pretty(total_mem_allocated),
'total_cpu_allocated': "%0.0f" % total_cpu_allocated,
'total_cpu_num': total_cpu_num,
'total_mem_num': str_pretty(total_mem_num),
'total_task_num': total_task_num,
'total_mem_used': str_pretty(total_mem_used),
'total_cpu_used': "%0.0f" % total_cpu_used,
'mem_usage_p': mem_usage_p,
'cpu_usage_p': cpu_usage_p
}).build_json()
def squery(request):
builder = http.ResponseBuilder()
if request.has_err:
return builder.error(request.err).build_json()
context, pb_req, ok = sql_to_mdt(request.db, request.sql, request.limit)
if not ok:
return builder.error("fail to parse sql").build_json()
ftrace = fsdk.FtraceSDK(request.trace)
resultset, ok = ftrace.make_req(pb_req)
if not ok:
return builder.error("fail to parse sql").build_json()
proc_func = PROCESSOR_MAP[request.db][context["table"]]
datas= proc_func(resultset, context["fields"], request.limit)
return builder.ok(data = {"dc":request.data_center,"master":request.master,"trace":request.trace,"datas":datas, "tpl":gen_tpl(context["fields"])}).build_json()
def set_password(req):
builder = http.ResponseBuilder()
username = req.user.username
agent_addr = req.POST.get('agent', None)
if not agent_addr:
return builder.error("agent is required").build_json()
password = req.POST.get('password', None)
if not password:
return builder.error('password is required').build_json()
g_agent = agent.Agent(agent_addr)
ret = g_agent.set_password(username, password)
if ret != 0:
return builder.error("fail to set password for %s" %
agent_addr).build_json()
return builder.ok(data={}).build_json()
def add_tag(req):
builder = http.ResponseBuilder()
master_addr = req.GET.get('master', None)
if not master_addr:
return builder.error('master is required').build_json()
tag = req.POST.get('tag', None)
if not tag:
return builder.error('tag is required').build_json()
agent_list = req.POST.get('agentList', "")
agent_set = set(agent_list.splitlines())
galaxy = wrapper.Galaxy(master_addr, settings.GALAXY_CLIENT_BIN)
try:
status = galaxy.tag_agent(tag, agent_set)
if status:
return builder.ok().build_json()
return builder.error('fail to tag agent ').build_json()
except:
return builder.error('fail to tag agent ').build_json()
def update_service(request):
builder = http.ResponseBuilder()
id = request.GET.get('id', None)
if not id:
return builder.error('id is required').build_json()
master_addr = request.GET.get('master', None)
if not master_addr:
return builder.error('master is required').build_json()
replicate = request.GET.get('replicate', None)
if not replicate:
return builder.error('replicate is required').build_json()
galaxy = wrapper.Galaxy(master_addr, settings.GALAXY_CLIENT_BIN)
status = galaxy.update_job(id, replicate)
if status:
return builder.ok().build_json()
else:
return builder.error('fail to kill job').build_json()
def list_service(request):
"""
get current user's service list
"""
builder = http.ResponseBuilder()
master_addr = request.GET.get('master', None)
if not master_addr:
return builder.error('master is required').build_json()
client = wrapper.Galaxy(master_addr, settings.GALAXY_CLIENT_BIN)
status, jobs = client.list_jobs()
LOG.info(status)
if not status:
return builder.error('fail to list jobs').build_json()
ret = []
for job in jobs:
ret.append(job.__dict__)
return builder.ok(data=ret).build_json()
def login(request):
username = request.POST.get('username', None)
response_builder = http.ResponseBuilder()
if not username:
return response_builder.add_params({"error":"username is required"})\
.add_req(request)\
.build_tmpl("login.html")
password = request.POST.get('password', None)
if not password:
return response_builder.add_params({"error":"password is required"})\
.add_req(request)\
.build_tmpl("login.html")
user = auth.authenticate(username=username, password=password)
if user and user.is_active:
login(request, user)
shortcuts.redirect("/")
else:
return response_builder.add_params({"error":"user %s does not exist "%username})\
.add_req(request)\
.build_tmpl("login.html")
def get_job_sched_history(request):
builder = http.ResponseBuilder()
id = request.GET.get('id', None)
master_addr = request.GET.get('master', None)
if not master_addr:
return builder.error('master is required').build_json()
galaxy = wrapper.Galaxy(master_addr, settings.GALAXY_CLIENT_BIN)
if not id:
return builder.error("id is required").build_json()
status, tasklist = galaxy.job_history(id)
for task in tasklist:
task['mem_used'] = str_pretty(task['mem_used'])
task['mem_limit'] = str_pretty(task['mem_limit'])
task['cpu_used'] = "%0.2f" % (task['cpu_limit'] * task['cpu_used'])
return builder.ok(data={
'needInit': False,
'taskList': tasklist
}).build_json()
def get_job_sched_history(request):
builder = http.ResponseBuilder()
id = request.GET.get('id', None)
master_addr = request.GET.get('master', None)
if not master_addr:
return builder.error('master is required').build_json()
galaxy = wrapper.Galaxy(master_addr, settings.GALAXY_CLIENT_BIN)
if not id:
return builder.error("id is required").build_json()
status, tasklist = galaxy.job_history(id)
for task in tasklist:
task['mem_used'] = str_pretty(task['mem_used'])
task['mem_limit'] = str_pretty(task['mem_limit'])
task['cpu_used'] = "%0.2f" % (task['cpu_used'])
if task['agent_addr']:
task['path'] = "http://" + task['agent_addr'].split(
":")[0] + ":8181/gc/" + task['gc_path'].replace(
"/home/galaxy/agent/work_dir/gc/", "")
return builder.ok(data={
'needInit': False,
'taskList': tasklist
}).build_json()
def update_service(request):
builder = http.ResponseBuilder()
id = request.GET.get('id', None)
if not id:
return builder.error('id is required').build_json()
master_addr = request.GET.get('master', None)
if not master_addr:
return builder.error('master is required').build_json()
replicate = request.GET.get('replicate', None)
if not replicate:
return builder.error('replicate is required').build_json()
status, stat = views.get_group_quota_stat(request.job.group, {})
old_job_meta = json.loads(request.job.meta)
replicate_num = int(replicate)
if replicate_num > old_job_meta['replicate_count']:
new_add = replicate_num - old_job_meta['replicate_count']
cpu_total_require = new_add * old_job_meta['cpu_share']
mem_total_require = old_job_meta[
'memory_limit'] * 1024 * 1024 * 1024 * new_add
if cpu_total_require > stat['total_cpu_left']:
return builder.error(
'cpu %s exceeds the left cpu quota %s' %
(cpu_total_require, stat['total_cpu_left'])).build_json()
if mem_total_require > stat['total_mem_left']:
return builder.error(
'mem %s exceeds the left mem %s' %
(mem_total_require, stat['total_mem_left'])).build_json()
old_job_meta['replicate_count'] = replicate_num
request.job.meta = json.dumps(old_job_meta)
request.job.save()
galaxy = wrapper.Galaxy(master_addr, settings.GALAXY_CLIENT_BIN)
status = galaxy.update_job(id, replicate)
if status:
return builder.ok().build_json()
else:
return builder.error('fail to kill job').build_json()
def post_wrapper(request,*args,**kwds):
res = http.ResponseBuilder()
if request.method != 'POST':
return res.error("post is required").build_json()
return func(request,*args,**kwds)
def create_service(request):
"""
create a service
"""
builder = http.ResponseBuilder()
data_str = request.POST.get("data", None)
if not data_str:
return builder.error('data is required').build_json()
data = json.loads(data_str)
LOG.info(data)
group_id = data.get('groupId', None)
if not group_id:
return builder.error('groupId is required').build_json()
try:
#判断是否有组权限
group_member_iterator = models.GroupMember.objects.filter(
user_name=request.user.username, group__id=int(group_id))
group_member = None
for gm in group_member_iterator:
group_member = gm
if not group_member:
return builder.error('group with %s does not exist' %
group_id).build_json()
if group_member.group.max_cpu_limit and data[
'cpuLimit'] > group_member.group.max_cpu_limit:
return builder.error(
'cpu limit exceeds the max cpu limit %d' %
group_member.group.max_cpu_limit).build_json()
status, stat = views.get_group_quota_stat(group_member.group, {})
cpu_total_require = data['replicate'] * data['cpuShare']
mem_total_require = data['memoryLimit'] * 1024 * 1024 * 1024 * data[
'replicate']
if cpu_total_require > stat['total_cpu_left']:
return builder.error(
'cpu %s exceeds the left cpu quota %s' %
(cpu_total_require, stat['total_cpu_left'])).build_json()
if mem_total_require > stat['total_mem_left']:
return builder.error(
'mem %s exceeds the left mem %s' %
(mem_total_require, stat['total_mem_left'])).build_json()
conf = None
if data['setMonitor']:
conf = helper.build_monitor_conf(data)
galaxy = wrapper.Galaxy(gm.group.galaxy_master,
settings.GALAXY_CLIENT_BIN)
tags = []
if 'tag' in data and data['tag'].strip():
tags.append(data['tag'].strip())
status, output = galaxy.create_task(
data['name'],
data['pkgSrc'],
data['startCmd'],
data['replicate'],
data['memoryLimit'] * 1024 * 1024 * 1024,
cpu_soft_limit=data['cpuShare'],
cpu_limit=data['cpuLimit'],
deploy_step_size=data['deployStepSize'],
one_task_per_host=data['oneTaskPerHost'],
restrict_tags=tags,
conf=conf)
if not status:
return builder.error('fail create task').build_json()
galaxy_job = models.GalaxyJob(group=group_member.group,
job_id=int(output),
meta=json.dumps(data))
galaxy_job.save()
return builder.ok().build_json()
except Exception as e:
return builder.error(str(e)).build_json()
def index(request):
response_builder = http.ResponseBuilder()
return response_builder.ok().build_json()
