def get_job_event(self, jobid, start_time, end_time, limit=100):
data_list, status = self.ftrace.simple_query(
"baidu.galaxy", "JobEvent", jobid, int(start_time * 1000000), int(end_time * 1000000), limit=limit
)
if not status:
logger.error("fail to query job stat")
return [], False
events = []
job_event = log_pb2.JobEvent()
job_desc = master_pb2.JobDescriptor()
for data in data_list:
for d in data.data_list:
job_event.ParseFromString(d)
job_desc.ParseFromString(job_event.desc)
data = pb2dict.protobuf_to_dict(job_event)
data["desc"] = pb2dict.protobuf_to_dict(job_desc)
data["state"] = master_pb2.JobState.Name(data["state"])
data["level"] = log_pb2.TraceLevel.Name(job_event.level)
data["update_state"] = master_pb2.JobUpdateState.Name(data["update_state"])
data["time"] = datetime.datetime.fromtimestamp(data["time"] / 1000000).strftime("%Y-%m-%d %H:%M:%S")
events.append(data)
logger.info(
"query job %s event from %s to %s , count %d"
% (
jobid,
datetime.datetime.fromtimestamp(start_time).strftime("%Y-%m-%d %H:%M:%S"),
datetime.datetime.fromtimestamp(end_time).strftime("%Y-%m-%d %H:%M:%S"),
len(events),
)
)
return events, True
def get_job_event(self, jobid, start_time, end_time, limit=100):
data_list, status = self.ftrace.simple_query("baidu.galaxy",
"JobEvent",
jobid,
int(start_time * 1000000),
int(end_time * 1000000),
limit=limit)
if not status:
logger.error("fail to query job stat")
return [], False
events = []
job_event = log_pb2.JobEvent()
job_desc = master_pb2.JobDescriptor()
for data in data_list:
for d in data.data_list:
job_event.ParseFromString(d)
job_desc.ParseFromString(job_event.desc)
data = pb2dict.protobuf_to_dict(job_event)
data["desc"] = pb2dict.protobuf_to_dict(job_desc)
data["state"] = master_pb2.JobState.Name(data["state"])
data["level"] = log_pb2.TraceLevel.Name(job_event.level)
data["update_state"] = master_pb2.JobUpdateState.Name(
data["update_state"])
data["time"] = datetime.datetime.fromtimestamp(
data["time"] / 1000000).strftime("%Y-%m-%d %H:%M:%S")
events.append(data)
logger.info(
"query job %s event from %s to %s , count %d" %
(jobid, datetime.datetime.fromtimestamp(start_time).strftime(
"%Y-%m-%d %H:%M:%S"), datetime.datetime.fromtimestamp(
end_time).strftime("%Y-%m-%d %H:%M:%S"), len(events)))
return events, True
def get_real_time_status(request):
galaxy = sdk.GalaxySDK(request.master)
response = galaxy.get_real_time_status()
builder = http.ResponseBuilder()
status = pb2dict.protobuf_to_dict(response)
return builder.ok(data={
"status": status,
"dc": request.data_center,
"master": request.master
}).build_json()
def job_all(request):
galaxy = sdk.GalaxySDK(request.master)
jobs, status = galaxy.get_all_job()
job_dicts = []
for job in jobs:
job_dict = pb2dict.protobuf_to_dict(job)
job_dict['state'] = master_pb2.JobState.Name(job_dict['state'])
job_dicts.append(job_dict)
return util.render_tpl(request, {"jobs":job_dicts,
"dc":request.data_center,
"master":request.master,
"trace":request.trace}, "index.html")
def job_all(request):
galaxy = sdk.GalaxySDK(request.master)
jobs, status = galaxy.get_all_job()
job_dicts = []
for job in jobs:
job_dict = pb2dict.protobuf_to_dict(job)
job_dict['state'] = master_pb2.JobState.Name(job_dict['state'])
job_dicts.append(job_dict)
return util.render_tpl(request, {"jobs":job_dicts,
"dc":request.data_center,
"master":request.master,
"trace":request.trace}, "index.html")
def get_total_status(request):
dcs = {}
with open(settings.LITE_DB_PATH, "rb") as fd:
dcs = json.load(fd)
datas = []
for key in dcs:
galaxy = sdk.GalaxySDK(dcs[key]["master"])
response = galaxy.get_real_time_status()
builder = http.ResponseBuilder()
status = pb2dict.protobuf_to_dict(response)
datas.append(status);
builder = http.ResponseBuilder()
return builder.ok(data = {"status":datas}).build_json()
def get_total_status(request):
dcs = {}
with open(settings.LITE_DB_PATH, "rb") as fd:
dcs = json.load(fd)
datas = []
for key in dcs:
galaxy = sdk.GalaxySDK(dcs[key]["master"])
response = galaxy.get_real_time_status()
builder = http.ResponseBuilder()
status = pb2dict.protobuf_to_dict(response)
datas.append(status);
builder = http.ResponseBuilder()
return builder.ok(data = {"status":datas}).build_json()
def job_event_processor(resultset, fields=[], limit=100):
if not fields:
return []
job_event = log_pb2.JobEvent()
events = []
for result in resultset:
for d in result.data_list:
job_event.ParseFromString(d)
data = pb2dict.protobuf_to_dict(job_event)
data["state"] = master_pb2.JobState.Name(data["state"])
data["level"] = log_pb2.TraceLevel.Name(job_event.level)
data["update_state"] = master_pb2.JobUpdateState.Name(data["update_state"])
events.append(data_filter(data, fields))
return events
def job_event_processor(resultset, fields=[], limit=100):
if not fields:
return []
job_event = log_pb2.JobEvent()
events = []
for result in resultset:
for d in result.data_list:
job_event.ParseFromString(d)
data = pb2dict.protobuf_to_dict(job_event)
data["state"] = master_pb2.JobState.Name(data["state"])
data["level"] = log_pb2.TraceLevel.Name(job_event.level)
data["update_state"] = master_pb2.JobUpdateState.Name(data["update_state"])
events.append(data_filter(data, fields))
return events
def get_real_time_status(request):
galaxy = sdk.GalaxySDK(request.master)
response = galaxy.get_real_time_status()
builder = http.ResponseBuilder()
status = pb2dict.protobuf_to_dict(response)
return builder.ok(data = {"status":status,"dc":request.data_center,"master":request.master}).build_json()
