def get_job_event(self, jobid, start_time, end_time, limit=100): data_list, status = self.ftrace.simple_query( "baidu.galaxy", "JobEvent", jobid, int(start_time * 1000000), int(end_time * 1000000), limit=limit ) if not status: logger.error("fail to query job stat") return [], False events = [] job_event = log_pb2.JobEvent() job_desc = master_pb2.JobDescriptor() for data in data_list: for d in data.data_list: job_event.ParseFromString(d) job_desc.ParseFromString(job_event.desc) data = pb2dict.protobuf_to_dict(job_event) data["desc"] = pb2dict.protobuf_to_dict(job_desc) data["state"] = master_pb2.JobState.Name(data["state"]) data["level"] = log_pb2.TraceLevel.Name(job_event.level) data["update_state"] = master_pb2.JobUpdateState.Name(data["update_state"]) data["time"] = datetime.datetime.fromtimestamp(data["time"] / 1000000).strftime("%Y-%m-%d %H:%M:%S") events.append(data) logger.info( "query job %s event from %s to %s , count %d" % ( jobid, datetime.datetime.fromtimestamp(start_time).strftime("%Y-%m-%d %H:%M:%S"), datetime.datetime.fromtimestamp(end_time).strftime("%Y-%m-%d %H:%M:%S"), len(events), ) ) return events, True
def get_job_event(self, jobid, start_time, end_time, limit=100): data_list, status = self.ftrace.simple_query("baidu.galaxy", "JobEvent", jobid, int(start_time * 1000000), int(end_time * 1000000), limit=limit) if not status: logger.error("fail to query job stat") return [], False events = [] job_event = log_pb2.JobEvent() job_desc = master_pb2.JobDescriptor() for data in data_list: for d in data.data_list: job_event.ParseFromString(d) job_desc.ParseFromString(job_event.desc) data = pb2dict.protobuf_to_dict(job_event) data["desc"] = pb2dict.protobuf_to_dict(job_desc) data["state"] = master_pb2.JobState.Name(data["state"]) data["level"] = log_pb2.TraceLevel.Name(job_event.level) data["update_state"] = master_pb2.JobUpdateState.Name( data["update_state"]) data["time"] = datetime.datetime.fromtimestamp( data["time"] / 1000000).strftime("%Y-%m-%d %H:%M:%S") events.append(data) logger.info( "query job %s event from %s to %s , count %d" % (jobid, datetime.datetime.fromtimestamp(start_time).strftime( "%Y-%m-%d %H:%M:%S"), datetime.datetime.fromtimestamp( end_time).strftime("%Y-%m-%d %H:%M:%S"), len(events))) return events, True
def get_real_time_status(request): galaxy = sdk.GalaxySDK(request.master) response = galaxy.get_real_time_status() builder = http.ResponseBuilder() status = pb2dict.protobuf_to_dict(response) return builder.ok(data={ "status": status, "dc": request.data_center, "master": request.master }).build_json()
def job_all(request): galaxy = sdk.GalaxySDK(request.master) jobs, status = galaxy.get_all_job() job_dicts = [] for job in jobs: job_dict = pb2dict.protobuf_to_dict(job) job_dict['state'] = master_pb2.JobState.Name(job_dict['state']) job_dicts.append(job_dict) return util.render_tpl(request, {"jobs":job_dicts, "dc":request.data_center, "master":request.master, "trace":request.trace}, "index.html")
def get_total_status(request): dcs = {} with open(settings.LITE_DB_PATH, "rb") as fd: dcs = json.load(fd) datas = [] for key in dcs: galaxy = sdk.GalaxySDK(dcs[key]["master"]) response = galaxy.get_real_time_status() builder = http.ResponseBuilder() status = pb2dict.protobuf_to_dict(response) datas.append(status); builder = http.ResponseBuilder() return builder.ok(data = {"status":datas}).build_json()
def job_event_processor(resultset, fields=[], limit=100): if not fields: return [] job_event = log_pb2.JobEvent() events = [] for result in resultset: for d in result.data_list: job_event.ParseFromString(d) data = pb2dict.protobuf_to_dict(job_event) data["state"] = master_pb2.JobState.Name(data["state"]) data["level"] = log_pb2.TraceLevel.Name(job_event.level) data["update_state"] = master_pb2.JobUpdateState.Name(data["update_state"]) events.append(data_filter(data, fields)) return events
def get_real_time_status(request): galaxy = sdk.GalaxySDK(request.master) response = galaxy.get_real_time_status() builder = http.ResponseBuilder() status = pb2dict.protobuf_to_dict(response) return builder.ok(data = {"status":status,"dc":request.data_center,"master":request.master}).build_json()