def update_workflow_state(self, name, state, target_state,
transaction_state, uuid, forked_from_uuid):
msg = service_pb2.UpdateWorkflowStateRequest(
auth_info=self._auth_info,
workflow_name=name,
state=state.value,
target_state=target_state.value,
transaction_state=transaction_state.value,
uuid=uuid,
forked_from_uuid=forked_from_uuid
)
try:
response = self._client.UpdateWorkflowState(
request=msg, metadata=self._get_metadata())
if response.status.code != common_pb2.STATUS_SUCCESS:
logging.error(
'update_workflow_state request error: %s',
response.status.msg)
return response
except Exception as e:
logging.error('workflow %s update_workflow_state request error: %s'
, name, repr(e))
return service_pb2.UpdateWorkflowStateResponse(
status=common_pb2.Status(
code=common_pb2.STATUS_UNKNOWN_ERROR,
msg=repr(e)))
def check_connection(self, request, context):
with self._app.app_context():
_, party = self.check_auth_info(request.auth_info, context)
logging.debug('received check_connection from %s',
party.domain_name)
return service_pb2.CheckConnectionResponse(
status=common_pb2.Status(code=common_pb2.STATUS_SUCCESS))
def get_workflow(self, request):
with self._app.app_context():
project, party = self.check_auth_info(request.auth_info)
workflow = Workflow.query.filter_by(name=request.workflow_name,
project_id=project.id).first()
assert workflow is not None
config = workflow.get_config()
self._filter_workflow(config, [
common_pb2.Variable.PEER_READABLE,
common_pb2.Variable.PEER_WRITABLE
])
# job details
jobs = [
service_pb2.JobDetail(name=job.name,
state=job.get_state_for_front())
for job in workflow.get_jobs()
]
# fork info
forked_from = ''
if workflow.forked_from:
forked_from = Workflow.query.get(workflow.forked_from).name
return service_pb2.GetWorkflowResponse(
name=request.workflow_name,
status=common_pb2.Status(code=common_pb2.STATUS_SUCCESS),
config=config,
jobs=jobs,
state=workflow.state.value,
target_state=workflow.target_state.value,
transaction_state=workflow.transaction_state.value,
forkable=workflow.forkable,
forked_from=forked_from,
reuse_job_names=workflow.get_reuse_job_names(),
peer_reuse_job_names=workflow.get_peer_reuse_job_names(),
fork_proposal_config=workflow.get_fork_proposal_config())
def update_workflow_state(self, request):
with self._app.app_context():
project, party = self.check_auth_info(request.auth_info)
logging.debug('received update_workflow_state from %s: %s',
party.domain_name, request)
name = request.workflow_name
state = WorkflowState(request.state)
target_state = WorkflowState(request.target_state)
transaction_state = TransactionState(request.transaction_state)
workflow = Workflow.query.filter_by(name=request.workflow_name,
project_id=project.id).first()
if workflow is None:
assert state == WorkflowState.NEW
assert target_state == WorkflowState.READY
workflow = Workflow(name=name,
project_id=project.id,
state=state,
target_state=target_state,
transaction_state=transaction_state)
db.session.add(workflow)
db.session.commit()
db.session.refresh(workflow)
workflow.update_state(state, target_state, transaction_state)
db.session.commit()
return service_pb2.UpdateWorkflowStateResponse(
status=common_pb2.Status(code=common_pb2.STATUS_SUCCESS),
transaction_state=workflow.transaction_state.value)
def CheckConnection(self, request, context):
try:
return self._server.check_connection(request)
except Exception as e:
return service_pb2.CheckConnectionResponse(
status=common_pb2.Status(
code=common_pb2.STATUS_UNKNOWN_ERROR,
msg=repr(e)))
def get_job_kibana(self, request, context):
with self._app.app_context():
job = self._check_metrics_public(request, context)
try:
metrics = Kibana.remote_query(job,
json.loads(request.json_args))
except UnauthorizedException as ua_e:
return service_pb2.GetJobKibanaResponse(
status=common_pb2.Status(
code=common_pb2.STATUS_UNAUTHORIZED, msg=ua_e.message))
except InvalidArgumentException as ia_e:
return service_pb2.GetJobKibanaResponse(
status=common_pb2.Status(
code=common_pb2.STATUS_INVALID_ARGUMENT,
msg=ia_e.message))
return service_pb2.GetJobKibanaResponse(
status=common_pb2.Status(code=common_pb2.STATUS_SUCCESS),
metrics=json.dumps(metrics))
def check_connection(self):
msg = service_pb2.CheckConnectionRequest(
auth_info=service_pb2.FedAuthInfo(
federation_name=self._federation.federation_name,
sender_name=self._federation.self_name,
receiver_name=self._receiver.name,
auth_token=self._receiver.sender_auth_token))
try:
return self._client.CheckConnection(msg).status
except Exception as e:
return common_pb2.Status(code=common_pb2.STATUS_UNKNOWN_ERROR,
msg=repr(e))
def get_job_metrics(self, request, context):
with self._app.app_context():
project, party = self.check_auth_info(request.auth_info, context)
job = Job.query.filter_by(name=request.job_name,
project_id=project.id).first()
assert job is not None, f'Job {request.job_name} not found'
workflow = job.workflow
if not workflow.metric_is_public:
raise UnauthorizedException('Metric is private!')
metrics = JobMetricsBuilder(job).plot_metrics()
return service_pb2.GetJobMetricsResponse(
status=common_pb2.Status(code=common_pb2.STATUS_SUCCESS),
metrics=json.dumps(metrics))
def check_job_ready(self, request, context):
with self._app.app_context():
project, _ = self.check_auth_info(request.auth_info, context)
job = db.session.query(Job).filter_by(
name=request.job_name, project_id=project.id).first()
assert job is not None, \
f'Job {request.job_name} not found'
with get_session(db.get_engine()) as session:
is_ready = JobService(session).is_ready(job)
return service_pb2.CheckJobReadyResponse(
status=common_pb2.Status(code=common_pb2.STATUS_SUCCESS),
is_ready=is_ready)
def check_connection(self):
msg = service_pb2.CheckConnectionRequest(auth_info=self._auth_info)
try:
response = self._client.CheckConnection(
request=msg, metadata=self._get_metadata())
if response.status.code != common_pb2.STATUS_SUCCESS:
logging.debug('check_connection request error: %s',
response.status.msg)
return response.status
except Exception as e:
logging.debug('check_connection request error: %s', repr(e))
return common_pb2.Status(code=common_pb2.STATUS_UNKNOWN_ERROR,
msg=repr(e))
def get_workflow(self, name):
msg = service_pb2.GetWorkflowRequest(auth_info=self._auth_info,
workflow_name=name)
try:
response = self._client.GetWorkflow(request=msg,
metadata=self._get_metadata())
if response.status.code != common_pb2.STATUS_SUCCESS:
logging.error('get_workflow request error: %s',
response.status.msg)
return response
except Exception as e:
logging.error('get_workflow request error: %s', repr(e))
return service_pb2.GetWorkflowResponse(status=common_pb2.Status(
code=common_pb2.STATUS_UNKNOWN_ERROR, msg=repr(e)))
def check_connection(self):
msg = service_pb2.CheckConnectionRequest(
auth_info=service_pb2.ProjAuthInfo(
project_name=self._project.project_name,
sender_name=self._project.self_name,
receiver_name=self._receiver.name,
auth_token=self._receiver.sender_auth_token))
try:
response = self._client.CheckConnection(
request=msg, metadata=self._get_metadata())
return response.status
except Exception as e:
return common_pb2.Status(code=common_pb2.STATUS_UNKNOWN_ERROR,
msg=repr(e))
def get_job_events(self, request, context):
with self._app.app_context():
project, party = self.check_auth_info(request.auth_info, context)
job = Job.query.filter_by(name=request.job_name,
project_id=project.id).first
assert job is not None, \
f'Job {request.job_name} not found'
result = es.query_events('filebeat-*', job.name,
'fedlearner-operator', request.start_time,
int(time.time() *
1000))[:request.max_lines][::-1]
return service_pb2.GetJobEventsResponse(
status=common_pb2.Status(code=common_pb2.STATUS_SUCCESS),
logs=result)
def update_workflow(self, request, context):
with self._app.app_context():
project, party = self.check_auth_info(request.auth_info, context)
workflow = Workflow.query.filter_by(name=request.workflow_name,
project_id=project.id).first()
assert workflow is not None, "Workflow not found"
config = workflow.get_config()
_merge_workflow_config(config, request.config,
[common_pb2.Variable.PEER_WRITABLE])
workflow.set_config(config)
db.session.commit()
self._filter_workflow(config, [
common_pb2.Variable.PEER_READABLE,
common_pb2.Variable.PEER_WRITABLE
])
return service_pb2.UpdateWorkflowResponse(
status=common_pb2.Status(code=common_pb2.STATUS_SUCCESS),
workflow_name=request.workflow_name,
config=config)
def get_job_events(self, job_name, start_time, max_lines):
msg = service_pb2.GetJobMetricsRequest(
auth_info=self._auth_info,
job_name=job_name,
start_time=start_time,
max_lines=max_lines)
try:
response = self._client.GetJobMetrics(
request=msg, metadata=self._get_metadata())
if response.status.code != common_pb2.STATUS_SUCCESS:
logging.error(
'get_job_events request error: %s',
response.status.msg)
return response
except Exception as e:
logging.error('get_job_events request error: %s', repr(e))
return service_pb2.GetJobMetricsResponse(
status=common_pb2.Status(
code=common_pb2.STATUS_UNKNOWN_ERROR,
msg=repr(e)))
def get_workflow(self, request, context):
with self._app.app_context():
project, party = self.check_auth_info(request.auth_info, context)
workflow = Workflow.query.filter_by(name=request.workflow_name,
project_id=project.id).first()
assert workflow is not None, 'Workflow not found'
config = workflow.get_config()
config = self._filter_workflow(config, [
common_pb2.Variable.PEER_READABLE,
common_pb2.Variable.PEER_WRITABLE
])
# job details
jobs = [
service_pb2.JobDetail(
name=job.name,
state=job.get_state_for_frontend(),
pods=json.dumps(
job.get_pods_for_frontend(include_private_info=False)))
for job in workflow.get_jobs()
]
# fork info
forked_from = ''
if workflow.forked_from:
forked_from = Workflow.query.get(workflow.forked_from).name
return service_pb2.GetWorkflowResponse(
name=request.workflow_name,
status=common_pb2.Status(code=common_pb2.STATUS_SUCCESS),
config=config,
jobs=jobs,
state=workflow.state.value,
target_state=workflow.target_state.value,
transaction_state=workflow.transaction_state.value,
forkable=workflow.forkable,
forked_from=forked_from,
create_job_flags=workflow.get_create_job_flags(),
peer_create_job_flags=workflow.get_peer_create_job_flags(),
fork_proposal_config=workflow.get_fork_proposal_config(),
uuid=workflow.uuid,
metric_is_public=workflow.metric_is_public)
def get_job_metrics(self, request, context):
with self._app.app_context():
project, party = self.check_auth_info(request.auth_info, context)
workflow = Workflow.query.filter_by(
name=request.workflow_name).first()
assert workflow is not None, \
f'Workflow {request.workflow_name} not found'
if not workflow.metric_is_public:
raise UnauthorizedException('Metric is private!')
job = None
for i in workflow.get_jobs():
if i.get_config().name == request.job_name:
job = i
break
assert job is not None, f'Job {request.job_name} not found'
metrics = JobMetricsBuilder(job).plot_metrics()
return service_pb2.GetJobMetricsResponse(
status=common_pb2.Status(code=common_pb2.STATUS_SUCCESS),
metrics=json.dumps(metrics))
def wrapper(*args, **kwargs):
try:
return f(*args, **kwargs)
except grpc.RpcError as e:
return resp_class(status=common_pb2.Status(
code=common_pb2.STATUS_UNKNOWN_ERROR, msg=repr(e)))
def ping(self, request, context):
return service_pb2.PingResponse(
status=common_pb2.Status(code=common_pb2.STATUS_SUCCESS),
msg='Pong!')
def check_connection(self, request):
if self.check_auth_info(request.auth_info):
return service_pb2.CheckConnectionResponse(
status=common_pb2.Status(code=common_pb2.STATUS_SUCCESS))
return service_pb2.CheckConnectionResponse(status=common_pb2.Status(
code=common_pb2.STATUS_UNAUTHORIZED))
