def get_total_status(request): dcs = {} with open(settings.LITE_DB_PATH, "rb") as fd: dcs = json.load(fd) datas = [] for key in dcs: galaxy = sdk.GalaxySDK(dcs[key]["master"]) response = galaxy.get_real_time_status() builder = http.ResponseBuilder() status = pb2dict.protobuf_to_dict(response) datas.append(status); builder = http.ResponseBuilder() return builder.ok(data = {"status":datas}).build_json()
def job_permission_wrapper(request, *args, **kwds): builder = http.ResponseBuilder() id = request.GET.get('id', None) if not id: return builder.error('id is required').build_json() master_addr = request.GET.get('master', None) if not master_addr: return builder.error('master is required').build_json() try: galaxy_job_it = models.GalaxyJob.objects.filter( job_id=int(id), group__galaxy_master=master_addr)[:1] if not galaxy_job_it: return builder.error("job %s does not exist in db" % id).build_json() member = models.GroupMember.objects.filter( group__id=galaxy_job_it[0].group.id, user_name=request.user.username)[:1] if not member: return builder.error("you have no permission to kill job %s" % id).build_json() request.job = galaxy_job_it[0] return func(request, *args, **kwds) except: LOG.exception('fail to get permission') return builder.error('fail to kill job %s' % id).build_json()
def api_auth_wrapper(request, *args, **kwds): res = http.ResponseBuilder() if request.user.is_authenticated(): return func(request, *args, **kwds) return res.error("%s?service=%s" % (settings.UUAP_CAS_SERVER, settings.MY_HOST), status=-2).build_json()
def list_service(request): """ get current user's service list """ builder = http.ResponseBuilder() master_addr = request.GET.get('master', None) if not master_addr: return builder.error('master is required').build_json() groupmember_list = models.GroupMember.objects.filter( user_name=request.user.username, group__galaxy_master=master_addr) group_id_list = [] for group_m in groupmember_list: group_id_list.append(group_m.group.id) db_jobs = models.GalaxyJob.objects.filter(group__id__in=group_id_list) jobs_dict = {} for db_job in db_jobs: jobs_dict[db_job.job_id] = db_job client = wrapper.Galaxy(master_addr, settings.GALAXY_CLIENT_BIN) status, jobs = client.list_jobs() if not status: return builder.error('fail to list jobs').build_json() ret = [] for job in jobs: if job.job_id not in jobs_dict and not request.user.is_superuser: continue job.trace = job.trace.__dict__ ret.append(job.__dict__) return builder.ok(data=ret).build_json()
def get_conf(request): builder = http.ResponseBuilder() home = [{ "nodeName": "服务列表", "href": "#service", "nodestyle": ["level1"], "children": [] }, { "nodeName": "集群信息", "href": "#cluster", "nodestyle": ["level1"], "children": [] }, { "nodeName": "配置集群", "href": "#setup", "nodestyle": ["level1"], "children": [] }] service = [{ "nodeName": "Tera", "nodestyle": ["level1"], "children": [{ "nodestyle": ["level2"], "subpage": "status", "nodeName": "服务状态", "children": [] }] }] return builder.ok(data={'home': home, 'service': service}).build_json()
def get_task_status(request): builder = http.ResponseBuilder() id = request.GET.get('id', None) agent = request.GET.get('agent', None) master_addr = request.GET.get('master', None) if not master_addr: return builder.error('master is required').build_json() galaxy = wrapper.Galaxy(master_addr, settings.GALAXY_CLIENT_BIN) tasklist = [] if id: status, tasklist = galaxy.list_task_by_job_id(id) if not status: return builder.error("fail to get task list")\ .build_json() if agent: status, tasklist = galaxy.list_task_by_host(agent) if not status: return builder.error("fail to get task list")\ .build_json() statics = {"RUNNING": 0, "DEPLOYING": 0, "ERROR": 0} for task in tasklist: task['mem_used'] = str_pretty(task['mem_used']) task['mem_limit'] = str_pretty(task['mem_limit']) task['cpu_used'] = "%0.2f" % (task['cpu_used']) if task['status'] in statics: statics[task['status']] += 1 if task['agent_addr']: task['path'] = "http://" + task['agent_addr'].split( ":")[0] + ":8181/data/galaxy/" + str(task['id']) return builder.ok(data={ 'needInit': False, 'taskList': tasklist, 'statics': statics }).build_json()
def create_service(request): """ create a service """ builder = http.ResponseBuilder() master_addr = request.GET.get('master', None) if not master_addr: return builder.error('master is required').build_json() galaxy = wrapper.Galaxy(master_addr, settings.GALAXY_CLIENT_BIN) try: ret = helper.validate_init_service_group_req(request) status, output = galaxy.create_task( ret['name'], ret['pkg_src'], ret['start_cmd'], ret['replicate_count'], ret['memory_limit'] * 1024 * 1024 * 1024, ret['cpu_share'], deploy_step_size=ret['deploy_step_size'], one_task_per_host=ret['one_task_per_host'], restrict_tags=ret['tag']) if not status: return builder.error('fail create task').build_json() return builder.ok().build_json() except Exception as e: return builder.error(str(e)).build_json()
def user_wrapper(request, *args, **kwds): builder = http.ResponseBuilder() user_id = request.GET.get("user", None) or request.POST.get( "user", None) if not user_id: return builder.error("user id required").build_json() request.user_id = user_id return func(request, *args, **kwds)
def job_permission_required_wrapper(request, *args, **kwds): res = http.ResponseBuilder() job_id = request.GET.get('id', None) if not job_id: return res.error("id is required").build_json() master = request.GET.get('master', None) if not master: return res.error('master is required').build_json()
def service_name_wrapper(req, *args, **kwds): builder = http.ResponseBuilder() service_name = req.GET.get('serviceName', None) or req.POST.get( 'serviceName', None) if not service_name: return builder.error("serviceName is required").build_json() req.service_name = service_name return func(req, *args, **kwds)
def render_tpl(req, context, tpl_path): render = http.ResponseBuilder() media_url = req.build_absolute_uri("/statics") root_url = req.build_absolute_uri("/") context["media_url"] = media_url context["root_url"] = root_url return render.set_content(context)\ .add_req(req)\ .build_tmpl(tpl_path)
def get_real_time_status(request): galaxy = sdk.GalaxySDK(request.master) response = galaxy.get_real_time_status() builder = http.ResponseBuilder() status = pb2dict.protobuf_to_dict(response) return builder.ok(data={ "status": status, "dc": request.data_center, "master": request.master }).build_json()
def kill_service(request): builder = http.ResponseBuilder() id = request.GET.get('id', None) master_addr = request.GET.get('master', None) try: galaxy = wrapper.Galaxy(master_addr, settings.GALAXY_CLIENT_BIN) galaxy.kill_job(int(id)) return builder.ok().build_json() except: LOG.exception("fail to kill job %s" % id) return builder.error('fail to kill job %s' % id).build_json()
def task_group_id_wrapper(req, *args, **kwds): builder = http.ResponseBuilder() task_group_id = req.GET.get("taskGroupId", None) or req.POST.get( "taskGroupId", None) if not task_group_id: return builder.error("taskGroupId is required").build_json() try: req.task_group_id = int(task_group_id) except: LOG.exception("fail to convert task group id %s" % task_group_id) return builder.error("taskGroupId is invalide").build_json() return func(req, *args, **kwds)
def kill_service(request): builder = http.ResponseBuilder() id = request.GET.get('id', None) if not id: return builder.error('id is required').build_json() master_addr = request.GET.get('master', None) if not master_addr: return builder.error('master is required').build_json() galaxy = wrapper.Galaxy(master_addr, settings.GALAXY_CLIENT_BIN) galaxy.kill_job(int(id)) return builder.ok().build_json()
def service_id_wrapper(request, *args, **kwds): builder = http.ResponseBuilder() service_id = request.GET.get("service", None) or request.POST.get( "service", None) if not service_id: return builder.error("service is required").build_json() try: request.service_id = int(service_id) except Exception as e: LOG.exception("fail to convert service to int") return builder.error("fail to convert service to int for %s" % e) return func(request, *args, **kwds)
def list_tag(req): builder = http.ResponseBuilder() master_addr = req.GET.get('master', None) if not master_addr: return builder.error('master is required').build_json() galaxy = wrapper.Galaxy(master_addr, settings.GALAXY_CLIENT_BIN) try: ret = galaxy.list_tag() return builder.ok(data=ret).build_json() except: return builder.error('fail to tag agent ').build_json()
def get_status(req): builder = http.ResponseBuilder() master_addr = req.GET.get('master', None) if not master_addr: return builder.error('master is required').build_json() client = wrapper.Galaxy(master_addr, settings.GALAXY_CLIENT_BIN) machine_list = client.list_node() ret = [] total_node_num = 0 total_cpu_num = 0 total_cpu_allocated = 0 total_cpu_used = 0 total_mem_used = 0 total_mem_num = 0 total_mem_allocated = 0 total_task_num = 0 total_node_num = len(machine_list) for machine in machine_list: total_cpu_num += machine.cpu_share total_mem_num += machine.mem_share total_task_num += machine.task_num total_cpu_allocated += machine.cpu_allocated total_mem_allocated += machine.mem_allocated total_mem_used += machine.mem_used total_cpu_used += machine.cpu_used machine.mem_share = str_pretty(machine.mem_share) machine.mem_allocated = str_pretty(machine.mem_allocated) machine.cpu_used = '%0.1f' % machine.cpu_allocated machine.cpu_allocated = '%0.1f' % machine.cpu_allocated ret.append(machine.__dict__) mem_usage_p = 0 cpu_usage_p = 0 if total_mem_num: mem_usage_p = "%0.1f" % (100 * total_mem_used / total_mem_num) if total_cpu_num: cpu_usage_p = "%0.1f" % (100 * total_cpu_used / total_cpu_num) return builder.ok( data={ 'machinelist': ret, 'total_node_num': total_node_num, 'total_mem_allocated': str_pretty(total_mem_allocated), 'total_cpu_allocated': "%0.0f" % total_cpu_allocated, 'total_cpu_num': total_cpu_num, 'total_mem_num': str_pretty(total_mem_num), 'total_task_num': total_task_num, 'total_mem_used': str_pretty(total_mem_used), 'total_cpu_used': "%0.0f" % total_cpu_used, 'mem_usage_p': mem_usage_p, 'cpu_usage_p': cpu_usage_p }).build_json()
def squery(request): builder = http.ResponseBuilder() if request.has_err: return builder.error(request.err).build_json() context, pb_req, ok = sql_to_mdt(request.db, request.sql, request.limit) if not ok: return builder.error("fail to parse sql").build_json() ftrace = fsdk.FtraceSDK(request.trace) resultset, ok = ftrace.make_req(pb_req) if not ok: return builder.error("fail to parse sql").build_json() proc_func = PROCESSOR_MAP[request.db][context["table"]] datas= proc_func(resultset, context["fields"], request.limit) return builder.ok(data = {"dc":request.data_center,"master":request.master,"trace":request.trace,"datas":datas, "tpl":gen_tpl(context["fields"])}).build_json()
def set_password(req): builder = http.ResponseBuilder() username = req.user.username agent_addr = req.POST.get('agent', None) if not agent_addr: return builder.error("agent is required").build_json() password = req.POST.get('password', None) if not password: return builder.error('password is required').build_json() g_agent = agent.Agent(agent_addr) ret = g_agent.set_password(username, password) if ret != 0: return builder.error("fail to set password for %s" % agent_addr).build_json() return builder.ok(data={}).build_json()
def add_tag(req): builder = http.ResponseBuilder() master_addr = req.GET.get('master', None) if not master_addr: return builder.error('master is required').build_json() tag = req.POST.get('tag', None) if not tag: return builder.error('tag is required').build_json() agent_list = req.POST.get('agentList', "") agent_set = set(agent_list.splitlines()) galaxy = wrapper.Galaxy(master_addr, settings.GALAXY_CLIENT_BIN) try: status = galaxy.tag_agent(tag, agent_set) if status: return builder.ok().build_json() return builder.error('fail to tag agent ').build_json() except: return builder.error('fail to tag agent ').build_json()
def update_service(request): builder = http.ResponseBuilder() id = request.GET.get('id', None) if not id: return builder.error('id is required').build_json() master_addr = request.GET.get('master', None) if not master_addr: return builder.error('master is required').build_json() replicate = request.GET.get('replicate', None) if not replicate: return builder.error('replicate is required').build_json() galaxy = wrapper.Galaxy(master_addr, settings.GALAXY_CLIENT_BIN) status = galaxy.update_job(id, replicate) if status: return builder.ok().build_json() else: return builder.error('fail to kill job').build_json()
def list_service(request): """ get current user's service list """ builder = http.ResponseBuilder() master_addr = request.GET.get('master', None) if not master_addr: return builder.error('master is required').build_json() client = wrapper.Galaxy(master_addr, settings.GALAXY_CLIENT_BIN) status, jobs = client.list_jobs() LOG.info(status) if not status: return builder.error('fail to list jobs').build_json() ret = [] for job in jobs: ret.append(job.__dict__) return builder.ok(data=ret).build_json()
def login(request): username = request.POST.get('username', None) response_builder = http.ResponseBuilder() if not username: return response_builder.add_params({"error":"username is required"})\ .add_req(request)\ .build_tmpl("login.html") password = request.POST.get('password', None) if not password: return response_builder.add_params({"error":"password is required"})\ .add_req(request)\ .build_tmpl("login.html") user = auth.authenticate(username=username, password=password) if user and user.is_active: login(request, user) shortcuts.redirect("/") else: return response_builder.add_params({"error":"user %s does not exist "%username})\ .add_req(request)\ .build_tmpl("login.html")
def get_job_sched_history(request): builder = http.ResponseBuilder() id = request.GET.get('id', None) master_addr = request.GET.get('master', None) if not master_addr: return builder.error('master is required').build_json() galaxy = wrapper.Galaxy(master_addr, settings.GALAXY_CLIENT_BIN) if not id: return builder.error("id is required").build_json() status, tasklist = galaxy.job_history(id) for task in tasklist: task['mem_used'] = str_pretty(task['mem_used']) task['mem_limit'] = str_pretty(task['mem_limit']) task['cpu_used'] = "%0.2f" % (task['cpu_limit'] * task['cpu_used']) return builder.ok(data={ 'needInit': False, 'taskList': tasklist }).build_json()
def get_job_sched_history(request): builder = http.ResponseBuilder() id = request.GET.get('id', None) master_addr = request.GET.get('master', None) if not master_addr: return builder.error('master is required').build_json() galaxy = wrapper.Galaxy(master_addr, settings.GALAXY_CLIENT_BIN) if not id: return builder.error("id is required").build_json() status, tasklist = galaxy.job_history(id) for task in tasklist: task['mem_used'] = str_pretty(task['mem_used']) task['mem_limit'] = str_pretty(task['mem_limit']) task['cpu_used'] = "%0.2f" % (task['cpu_used']) if task['agent_addr']: task['path'] = "http://" + task['agent_addr'].split( ":")[0] + ":8181/gc/" + task['gc_path'].replace( "/home/galaxy/agent/work_dir/gc/", "") return builder.ok(data={ 'needInit': False, 'taskList': tasklist }).build_json()
def update_service(request): builder = http.ResponseBuilder() id = request.GET.get('id', None) if not id: return builder.error('id is required').build_json() master_addr = request.GET.get('master', None) if not master_addr: return builder.error('master is required').build_json() replicate = request.GET.get('replicate', None) if not replicate: return builder.error('replicate is required').build_json() status, stat = views.get_group_quota_stat(request.job.group, {}) old_job_meta = json.loads(request.job.meta) replicate_num = int(replicate) if replicate_num > old_job_meta['replicate_count']: new_add = replicate_num - old_job_meta['replicate_count'] cpu_total_require = new_add * old_job_meta['cpu_share'] mem_total_require = old_job_meta[ 'memory_limit'] * 1024 * 1024 * 1024 * new_add if cpu_total_require > stat['total_cpu_left']: return builder.error( 'cpu %s exceeds the left cpu quota %s' % (cpu_total_require, stat['total_cpu_left'])).build_json() if mem_total_require > stat['total_mem_left']: return builder.error( 'mem %s exceeds the left mem %s' % (mem_total_require, stat['total_mem_left'])).build_json() old_job_meta['replicate_count'] = replicate_num request.job.meta = json.dumps(old_job_meta) request.job.save() galaxy = wrapper.Galaxy(master_addr, settings.GALAXY_CLIENT_BIN) status = galaxy.update_job(id, replicate) if status: return builder.ok().build_json() else: return builder.error('fail to kill job').build_json()
def post_wrapper(request,*args,**kwds): res = http.ResponseBuilder() if request.method != 'POST': return res.error("post is required").build_json() return func(request,*args,**kwds)
def create_service(request): """ create a service """ builder = http.ResponseBuilder() data_str = request.POST.get("data", None) if not data_str: return builder.error('data is required').build_json() data = json.loads(data_str) LOG.info(data) group_id = data.get('groupId', None) if not group_id: return builder.error('groupId is required').build_json() try: #判断是否有组权限 group_member_iterator = models.GroupMember.objects.filter( user_name=request.user.username, group__id=int(group_id)) group_member = None for gm in group_member_iterator: group_member = gm if not group_member: return builder.error('group with %s does not exist' % group_id).build_json() if group_member.group.max_cpu_limit and data[ 'cpuLimit'] > group_member.group.max_cpu_limit: return builder.error( 'cpu limit exceeds the max cpu limit %d' % group_member.group.max_cpu_limit).build_json() status, stat = views.get_group_quota_stat(group_member.group, {}) cpu_total_require = data['replicate'] * data['cpuShare'] mem_total_require = data['memoryLimit'] * 1024 * 1024 * 1024 * data[ 'replicate'] if cpu_total_require > stat['total_cpu_left']: return builder.error( 'cpu %s exceeds the left cpu quota %s' % (cpu_total_require, stat['total_cpu_left'])).build_json() if mem_total_require > stat['total_mem_left']: return builder.error( 'mem %s exceeds the left mem %s' % (mem_total_require, stat['total_mem_left'])).build_json() conf = None if data['setMonitor']: conf = helper.build_monitor_conf(data) galaxy = wrapper.Galaxy(gm.group.galaxy_master, settings.GALAXY_CLIENT_BIN) tags = [] if 'tag' in data and data['tag'].strip(): tags.append(data['tag'].strip()) status, output = galaxy.create_task( data['name'], data['pkgSrc'], data['startCmd'], data['replicate'], data['memoryLimit'] * 1024 * 1024 * 1024, cpu_soft_limit=data['cpuShare'], cpu_limit=data['cpuLimit'], deploy_step_size=data['deployStepSize'], one_task_per_host=data['oneTaskPerHost'], restrict_tags=tags, conf=conf) if not status: return builder.error('fail create task').build_json() galaxy_job = models.GalaxyJob(group=group_member.group, job_id=int(output), meta=json.dumps(data)) galaxy_job.save() return builder.ok().build_json() except Exception as e: return builder.error(str(e)).build_json()
def index(request): response_builder = http.ResponseBuilder() return response_builder.ok().build_json()