def create_sparkapplication(
self,
json_object: dict,
namespace: str = SPARKOPERATOR_NAMESPACE) -> dict:
""" create sparkapp
Args:
json_object (dict): json object of config
namespace (str, optional): namespace to submit.
Raises:
ApiException
Returns:
dict: resp of k8s
"""
try:
logging.debug('create sparkapp json is %s', json_object)
return self.crds.create_namespaced_custom_object(
group=SPARKOPERATOR_CUSTOM_GROUP,
version=SPARKOPERATOR_CUSTOM_VERSION,
namespace=namespace,
plural=CrdKind.SPARK_APPLICATION.value,
body=json_object)
except ApiException as err:
if err.status == 409:
raise ResourceConflictException(message=err.reason)
if err.status == 400:
raise InvalidArgumentException(details=err.reason)
raise InternalException(details=err.body)
def remote_query(job, args):
Kibana._check_remote_args(args)
if args['type'] == 'Ratio':
panel = Kibana._create_ratio_visualization(job, args)['params']
else:
assert args['type'] == 'Numeric'
panel = Kibana._create_numeric_visualization(job, args)['params']
if 'query' in args and args['query']:
panel['filter']['query'] += ' and ({})'.format(args['query'])
st, et = Kibana._parse_start_end_time(args, use_now=False)
req = {
'timerange': {
'timezone': Envs.TZ.zone,
'min': st,
'max': et
},
'panels': [panel]
}
res = requests.post(os.path.join(Envs.KIBANA_SERVICE_ADDRESS,
'api/metrics/vis/data'),
json=req,
headers={'kbn-xsrf': 'true'})
try:
res.raise_for_status()
res = res.json()
data = res['undefined']['series'][0]['data']
data = list(map(lambda x: [x[0], x[1] or 0], data))
return data
except Exception as e: # pylint: disable=broad-except
raise InternalException(repr(e))
def get(self, participant_id, job_id):
parser = reqparse.RequestParser()
parser.add_argument('start_time',
type=int,
location='args',
required=False,
help='project_id must be timestamp')
parser.add_argument('max_lines',
type=int,
location='args',
required=True,
help='max_lines is required')
data = parser.parse_args()
start_time = data['start_time']
max_lines = data['max_lines']
job = _get_job(job_id)
if start_time is None:
start_time = job.workflow.start_at
workflow = job.workflow
project_config = workflow.project.get_config()
party = project_config.participants[participant_id]
client = RpcClient(project_config, party)
resp = client.get_job_events(job_name=job.name,
start_time=start_time,
max_lines=max_lines)
if resp.status.code != common_pb2.STATUS_SUCCESS:
raise InternalException(resp.status.msg)
peer_events = MessageToDict(resp.logs,
preserving_proto_field_name=True,
including_default_value_fields=True)
return {'data': peer_events}
def delete_sparkapplication(self,
name: str,
namespace: str = SPARKOPERATOR_NAMESPACE
) -> dict:
""" delete sparkapp
Args:
name (str): sparkapp name
namespace (str, optional): namespace to delete.
Raises:
ApiException
Returns:
dict: resp of k8s
"""
try:
return self.crds.delete_namespaced_custom_object(
group=SPARKOPERATOR_CUSTOM_GROUP,
version=SPARKOPERATOR_CUSTOM_VERSION,
namespace=namespace,
plural=CrdKind.SPARK_APPLICATION.value,
name=name,
body=client.V1DeleteOptions())
except ApiException as err:
if err.status == 404:
raise NotFoundException()
raise InternalException(details=err.body)
def get(self, workflow_uuid, participant_id, job_name):
parser = reqparse.RequestParser()
parser.add_argument('start_time', type=int, location='args',
required=False,
help='project_id must be timestamp')
parser.add_argument('max_lines', type=int, location='args',
required=True,
help='max_lines is required')
data = parser.parse_args()
start_time = data['start_time']
max_lines = data['max_lines']
workflow = Workflow.query.filter_by(uuid=workflow_uuid).first()
if workflow is None:
raise NotFoundException(
f'Failed to find workflow: {workflow_uuid}')
if start_time is None:
start_time = workflow.start_at
project_config = workflow.project.get_config()
party = project_config.participants[participant_id]
client = RpcClient(project_config, party)
resp = client.get_job_events(job_name=job_name,
start_time=start_time,
max_lines=max_lines)
if resp.status.code != common_pb2.STATUS_SUCCESS:
raise InternalException(resp.status.msg)
peer_events = MessageToDict(
resp,
preserving_proto_field_name=True,
including_default_value_fields=True)['logs']
return {'data': peer_events}
def is_peer_job_inheritance_matched(workflow):
# TODO: Move it to workflow service
if workflow.forked_from is None:
return True
job_flags = workflow.get_create_job_flags()
peer_job_flags = workflow.get_peer_create_job_flags()
job_defs = workflow.get_config().job_definitions
project = workflow.project
if project is None:
return True
project_config = project.get_config()
# TODO: Fix for multi-peer
client = RpcClient(project_config, project_config.participants[0])
parent_workflow = db.session.query(Workflow).get(workflow.forked_from)
resp = client.get_workflow(parent_workflow.name)
if resp.status.code != common_pb2.STATUS_SUCCESS:
emit_counter('get_workflow_failed', 1)
raise InternalException(resp.status.msg)
peer_job_defs = resp.config.job_definitions
for i, job_def in enumerate(job_defs):
if job_def.is_federated:
for j, peer_job_def in enumerate(peer_job_defs):
if job_def.name == peer_job_def.name:
if job_flags[i] != peer_job_flags[j]:
return False
return True
def get(self, workflow_uuid, participant_id, job_name):
parser = reqparse.RequestParser()
parser.add_argument('type', type=str, location='args',
required=True,
choices=('Ratio', 'Numeric'),
help='Visualization type is required. Choices: '
'Rate, Ratio, Numeric, Time, Timer')
parser.add_argument('interval', type=str, location='args',
default='',
help='Time bucket interval length, '
'defaults to be automated by Kibana.')
parser.add_argument('x_axis_field', type=str, location='args',
default='tags.event_time',
help='Time field (X axis) is required.')
parser.add_argument('query', type=str, location='args',
help='Additional query string to the graph.')
parser.add_argument('start_time', type=int, location='args',
default=-1,
help='Earliest <x_axis_field> time of data.'
'Unix timestamp in secs.')
parser.add_argument('end_time', type=int, location='args',
default=-1,
help='Latest <x_axis_field> time of data.'
'Unix timestamp in secs.')
# Ratio visualization
parser.add_argument('numerator', type=str, location='args',
help='Numerator is required in Ratio '
'visualization. '
'A query string similar to args::query.')
parser.add_argument('denominator', type=str, location='args',
help='Denominator is required in Ratio '
'visualization. '
'A query string similar to args::query.')
# Numeric visualization
parser.add_argument('aggregator', type=str, location='args',
default='Average',
choices=('Average', 'Sum', 'Max', 'Min', 'Variance',
'Std. Deviation', 'Sum of Squares'),
help='Aggregator type is required in Numeric and '
'Timer visualization.')
parser.add_argument('value_field', type=str, location='args',
help='The field to be aggregated on is required '
'in Numeric visualization.')
args = parser.parse_args()
workflow = Workflow.query.filter_by(uuid=workflow_uuid).first()
if workflow is None:
raise NotFoundException(
f'Failed to find workflow: {workflow_uuid}')
project_config = workflow.project.get_config()
party = project_config.participants[participant_id]
client = RpcClient(project_config, party)
resp = client.get_job_kibana(job_name, json.dumps(args))
if resp.status.code != common_pb2.STATUS_SUCCESS:
raise InternalException(resp.status.msg)
metrics = json.loads(resp.metrics)
# metrics is a list of 2-element lists,
# each 2-element list is a [x, y] pair.
return {'data': metrics}
def get(self, workflow_id):
workflow = _get_workflow(workflow_id)
project_config = workflow.project.get_config()
peer_workflows = {}
for party in project_config.participants:
client = RpcClient(project_config, party)
resp = client.get_workflow(workflow.name)
if resp.status.code != common_pb2.STATUS_SUCCESS:
raise InternalException()
peer_workflows[party.name] = MessageToDict(
resp,
preserving_proto_field_name=True,
including_default_value_fields=True)
return {'data': peer_workflows}, HTTPStatus.OK
def get(self, participant_id, job_id):
job = _get_job(job_id)
workflow = job.workflow
project_config = workflow.project.get_config()
party = project_config.participants[participant_id]
client = RpcClient(project_config, party)
resp = client.get_job_metrics(job.name)
if resp.status.code != common_pb2.STATUS_SUCCESS:
raise InternalException(resp.status.msg)
metrics = json.loads(resp.metrics)
# Metrics is a list of dict. Each dict can be rendered by frontend with
# mpld3.draw_figure('figure1', json)
return {'data': metrics}
def get(self, workflow_id, participant_id, job_name):
workflow = Workflow.query.filter_by(id=workflow_id).first()
if workflow is None:
raise NotFoundException()
project_config = workflow.project.get_config()
party = project_config.participants[participant_id]
client = RpcClient(project_config, party)
resp = client.get_job_metrics(workflow.name, job_name)
if resp.status.code != common_pb2.STATUS_SUCCESS:
raise InternalException(resp.status.msg)
metrics = json.loads(resp.metrics)
# Metrics is a list of dict. Each dict can be rendered by frontend with
# mpld3.draw_figure('figure1', json)
return {'data': metrics}
def get(self, workflow_id):
workflow = _get_workflow(workflow_id)
project_config = workflow.project.get_config()
peer_workflows = {}
for party in project_config.participants:
client = RpcClient(project_config, party)
# TODO(xiangyxuan): use uuid to identify the workflow
resp = client.get_workflow(workflow.name)
if resp.status.code != common_pb2.STATUS_SUCCESS:
raise InternalException(resp.status.msg)
peer_workflow = MessageToDict(resp,
preserving_proto_field_name=True,
including_default_value_fields=True)
for job in peer_workflow['jobs']:
if 'pods' in job:
job['pods'] = json.loads(job['pods'])
peer_workflows[party.name] = peer_workflow
return {'data': peer_workflows}, HTTPStatus.OK
def patch(self, workflow_id):
parser = reqparse.RequestParser()
parser.add_argument('config', type=dict, required=True,
help='new config for peer')
data = parser.parse_args()
config_proto = dict_to_workflow_definition(data['config'])
workflow = _get_workflow(workflow_id)
project_config = workflow.project.get_config()
peer_workflows = {}
for party in project_config.participants:
client = RpcClient(project_config, party)
resp = client.update_workflow(
workflow.name, config_proto)
if resp.status.code != common_pb2.STATUS_SUCCESS:
raise InternalException(resp.status.msg)
peer_workflows[party.name] = MessageToDict(
resp,
preserving_proto_field_name=True,
including_default_value_fields=True)
return {'data': peer_workflows}, HTTPStatus.OK