def put(self, workflow_id):
parser = reqparse.RequestParser()
parser.add_argument('config',
type=dict,
required=True,
help='config is empty')
parser.add_argument('forkable',
type=bool,
required=True,
help='forkable is empty')
parser.add_argument('comment')
data = parser.parse_args()
workflow = _get_workflow(workflow_id)
if workflow.config:
raise ResourceConflictException(
'Resetting workflow is not allowed')
workflow.comment = data['comment']
workflow.forkable = data['forkable']
workflow.set_config(dict_to_workflow_definition(data['config']))
workflow.update_target_state(WorkflowState.READY)
db.session.commit()
logging.info('update workflow %d target_state to %s', workflow.id,
workflow.target_state)
return {'data': workflow.to_dict()}, HTTPStatus.OK
def post(self):
parser = reqparse.RequestParser()
parser.add_argument('name', required=True, help='name is empty')
parser.add_argument('comment')
parser.add_argument('config',
type=dict,
required=True,
help='config is empty')
data = parser.parse_args()
name = data['name']
comment = data['comment']
config = data['config']
if WorkflowTemplate.query.filter_by(name=name).first() is not None:
raise ResourceConflictException(
'Workflow template {} already exists'.format(name))
# form to proto buffer
template_proto = dict_to_workflow_definition(config)
group_template = WorkflowTemplate.query.filter_by(
group_alias=template_proto.group_alias).first()
if group_template is not None:
group_proto = group_template.get_config()
if not (check_group_match(group_proto, template_proto)
or check_group_same(group_proto, template_proto)):
raise InvalidArgumentException(
'The group is not matched with existing groups.')
template = WorkflowTemplate(name=name,
comment=comment,
group_alias=template_proto.group_alias)
template.set_config(template_proto)
db.session.add(template)
db.session.commit()
logging.info('Inserted a workflow_template to db')
return {'data': template.to_dict()}, HTTPStatus.CREATED
def create_sparkapplication(
self,
json_object: dict,
namespace: str = SPARKOPERATOR_NAMESPACE) -> dict:
""" create sparkapp
Args:
json_object (dict): json object of config
namespace (str, optional): namespace to submit.
Raises:
ApiException
Returns:
dict: resp of k8s
"""
try:
logging.debug('create sparkapp json is %s', json_object)
return self.crds.create_namespaced_custom_object(
group=SPARKOPERATOR_CUSTOM_GROUP,
version=SPARKOPERATOR_CUSTOM_VERSION,
namespace=namespace,
plural=CrdKind.SPARK_APPLICATION.value,
body=json_object)
except ApiException as err:
if err.status == 409:
raise ResourceConflictException(message=err.reason)
if err.status == 400:
raise InvalidArgumentException(details=err.reason)
raise InternalException(details=err.body)
def post(self):
parser = reqparse.RequestParser()
parser.add_argument('name', required=True, help='name is empty')
parser.add_argument('comment')
parser.add_argument('config',
type=dict,
required=True,
help='config is empty')
parser.add_argument('editor_info', type=dict, default={})
parser.add_argument('kind', type=int, default=0)
data = parser.parse_args()
name = data['name']
comment = data['comment']
config = data['config']
editor_info = data['editor_info']
kind = data['kind']
if WorkflowTemplate.query.filter_by(name=name).first() is not None:
raise ResourceConflictException(
'Workflow template {} already exists'.format(name))
template_proto, editor_info_proto = _check_config_and_editor_info(
config, editor_info)
template_proto = _format_template_with_yaml_editor(
template_proto, editor_info_proto)
template = WorkflowTemplate(name=name,
comment=comment,
group_alias=template_proto.group_alias,
is_left=template_proto.is_left,
kind=kind)
template.set_config(template_proto)
template.set_editor_info(editor_info_proto)
db.session.add(template)
db.session.commit()
logging.info('Inserted a workflow_template to db')
result = template.to_dict()
return {'data': result}, HTTPStatus.CREATED
def post(self):
parser = reqparse.RequestParser()
parser.add_argument('username',
required=True,
help='username is empty')
parser.add_argument('password',
required=True,
help='password is empty')
parser.add_argument('role', required=True, help='role is empty')
parser.add_argument('name', required=True, help='name is empty')
parser.add_argument('email', required=True, help='email is empty')
data = parser.parse_args()
username = data['username']
password = base64decode(data['password'])
role = data['role']
name = data['name']
email = data['email']
check_password_format(password)
if User.query.filter_by(username=username).first() is not None:
raise ResourceConflictException(
'user {} already exists'.format(username))
user = User(username=username,
role=role,
name=name,
email=email,
state=State.ACTIVE)
user.set_password(password)
db.session.add(user)
db.session.commit()
return {'data': user.to_dict()}, HTTPStatus.CREATED
def post(self):
parser = reqparse.RequestParser()
parser.add_argument('name', required=True, help='name is empty')
parser.add_argument('comment')
parser.add_argument('config', type=dict, required=True,
help='config is empty')
data = parser.parse_args()
name = data['name']
comment = data['comment']
config = data['config']
if 'group_alias' not in config:
raise InvalidArgumentException(details={
'config.group_alias': 'config.group_alias is required'})
if 'is_left' not in config:
raise InvalidArgumentException(
details={'config.is_left': 'config.is_left is required'})
if WorkflowTemplate.query.filter_by(name=name).first() is not None:
raise ResourceConflictException(
'Workflow template {} already exists'.format(name))
# form to proto buffer
template_proto = dict_to_workflow_definition(config)
template = WorkflowTemplate(name=name,
comment=comment,
group_alias=template_proto.group_alias,
is_left=template_proto.is_left)
template.set_config(template_proto)
db.session.add(template)
db.session.commit()
logging.info('Inserted a workflow_template to db')
return {'data': template.to_dict()}, HTTPStatus.CREATED
def put(self, template_id):
parser = reqparse.RequestParser()
parser.add_argument('name', required=True, help='name is empty')
parser.add_argument('comment')
parser.add_argument('config',
type=dict,
required=True,
help='config is empty')
data = parser.parse_args()
name = data['name']
comment = data['comment']
config = data['config']
tmp = WorkflowTemplate.query.filter_by(name=name).first()
if tmp is not None and tmp.id != template_id:
raise ResourceConflictException(
'Workflow template {} already exists'.format(name))
template = WorkflowTemplate.query.filter_by(id=template_id).first()
if template is None:
raise NotFoundException()
template_proto = _check_config(config)
template.set_config(template_proto)
template.name = name
template.comment = comment
template.group_alias = template_proto.group_alias
template.is_left = template_proto.is_left
db.session.commit()
return {'data': template.to_dict()}, HTTPStatus.OK
def stop(self):
if self.status == JobStatus.STOPPED:
raise ResourceConflictException('Job has stopped')
self.status = JobStatus.STOPPED
self._set_snapshot_flapp()
self._set_snapshot_pods()
self._k8s_client.delete_flapp(self._project_adapter.
get_namespace(), self.name)
def post(self):
parser = reqparse.RequestParser()
parser.add_argument('name', required=True, help='name is empty')
parser.add_argument('project_id', type=int, required=True,
help='project_id is empty')
# TODO: should verify if the config is compatible with
# workflow template
parser.add_argument('config', type=dict, required=True,
help='config is empty')
parser.add_argument('forkable', type=bool, required=True,
help='forkable is empty')
parser.add_argument('forked_from', type=int, required=False,
help='fork from base workflow')
parser.add_argument('reuse_job_names', type=list, required=False,
location='json', help='fork and inherit jobs')
parser.add_argument('peer_reuse_job_names', type=list,
required=False, location='json',
help='peer fork and inherit jobs')
parser.add_argument('fork_proposal_config', type=dict, required=False,
help='fork and edit peer config')
parser.add_argument('comment')
data = parser.parse_args()
name = data['name']
if Workflow.query.filter_by(name=name).first() is not None:
raise ResourceConflictException(
'Workflow {} already exists.'.format(name))
# form to proto buffer
template_proto = dict_to_workflow_definition(data['config'])
workflow = Workflow(name=name, comment=data['comment'],
project_id=data['project_id'],
forkable=data['forkable'],
forked_from=data['forked_from'],
state=WorkflowState.NEW,
target_state=WorkflowState.READY,
transaction_state=TransactionState.READY)
if workflow.forked_from is not None:
fork_config = dict_to_workflow_definition(
data['fork_proposal_config'])
# TODO: more validations
if len(fork_config.job_definitions) != \
len(template_proto.job_definitions):
raise InvalidArgumentException(
'Forked workflow\'s template does not match base workflow')
workflow.set_fork_proposal_config(fork_config)
workflow.set_reuse_job_names(data['reuse_job_names'])
workflow.set_peer_reuse_job_names(data['peer_reuse_job_names'])
workflow.set_config(template_proto)
db.session.add(workflow)
db.session.commit()
logging.info('Inserted a workflow to db')
scheduler.wakeup(workflow.id)
return {'data': workflow.to_dict()}, HTTPStatus.CREATED
def post(self):
parser = reqparse.RequestParser()
parser.add_argument('name', required=True, help='name is empty')
parser.add_argument('comment')
parser.add_argument('config',
type=dict,
required=True,
help='config is empty')
data = parser.parse_args()
name = data['name']
comment = data['comment']
config = data['config']
# TODO: format check
if 'group_alias' not in config:
raise InvalidArgumentException(
details={
'config.group_alias': 'config.group_alias is required'
})
if 'is_left' not in config:
raise InvalidArgumentException(
details={'config.is_left': 'config.is_left is required'})
if WorkflowTemplate.query.filter_by(name=name).first() is not None:
raise ResourceConflictException(
'Workflow template {} already exists'.format(name))
# form to proto buffer
template_proto = dict_to_workflow_definition(config)
for index, job_def in enumerate(template_proto.job_definitions):
# pod label name must be no more than 63 characters.
# workflow.uuid is 20 characters, pod name suffix such as
# '-follower-master-0' is less than 19 characters, so the
# job name must be no more than 24
if len(job_def.name) > 24:
raise InvalidArgumentException(
details={
'config.job_definitions':
'job_name:{} must be no more than 24 characters'
})
# limit from k8s
if not re.match('[a-z0-9-]*', job_def.name):
raise InvalidArgumentException(
details={
f'config.job_definitions[{index}].job_name':
'Only letters(a-z), numbers(0-9) '
'and dashes(-) are supported.'
})
template = WorkflowTemplate(name=name,
comment=comment,
group_alias=template_proto.group_alias,
is_left=template_proto.is_left)
template.set_config(template_proto)
db.session.add(template)
db.session.commit()
logging.info('Inserted a workflow_template to db')
return {'data': template.to_dict()}, HTTPStatus.CREATED
def post(self):
parser = reqparse.RequestParser()
parser.add_argument('username',
required=True,
help='username is empty')
parser.add_argument('password',
required=True,
help='password is empty')
data = parser.parse_args()
username = data['username']
password = data['password']
if User.query.filter_by(username=username).first() is not None:
raise ResourceConflictException(
'user {} already exists'.format(username))
user = User(username=username)
user.set_password(password)
db.session.add(user)
db.session.commit()
return {'username': user.username}, HTTPStatus.CREATED
def put(self, template_id):
parser = reqparse.RequestParser()
parser.add_argument('name', required=True, help='name is empty')
parser.add_argument('comment')
parser.add_argument('config',
type=dict,
required=True,
help='config is empty')
parser.add_argument('editor_info', type=dict, default={})
parser.add_argument('kind', type=int, default=0)
data = parser.parse_args()
name = data['name']
comment = data['comment']
config = data['config']
editor_info = data['editor_info']
kind = data['kind']
tmp = WorkflowTemplate.query.filter_by(name=name).first()
if tmp is not None and tmp.id != template_id:
raise ResourceConflictException(
'Workflow template {} already exists'.format(name))
template = WorkflowTemplate.query.filter_by(id=template_id).first()
if template is None:
raise NotFoundException()
template_proto, editor_info_proto = _check_config_and_editor_info(
config, editor_info)
template_proto = _format_template_with_yaml_editor(
template_proto, editor_info_proto)
template.set_config(template_proto)
template.set_editor_info(editor_info_proto)
template.name = name
template.comment = comment
template.group_alias = template_proto.group_alias
template.is_left = template_proto.is_left
template.kind = kind
db.session.commit()
result = template.to_dict()
return {'data': result}, HTTPStatus.OK
def post(self):
parser = reqparse.RequestParser()
parser.add_argument('name', required=True, help='name is empty')
parser.add_argument('project_id', type=int, required=True,
help='project_id is empty')
# TODO: should verify if the config is compatible with
# workflow template
parser.add_argument('config', type=dict, required=True,
help='config is empty')
parser.add_argument('forkable', type=bool, required=True,
help='forkable is empty')
parser.add_argument('forked_from', type=int, required=False,
help='forkable is empty')
parser.add_argument('comment')
data = parser.parse_args()
name = data['name']
if Workflow.query.filter_by(name=name).first() is not None:
raise ResourceConflictException(
'Workflow {} already exists.'.format(name))
# form to proto buffer
template_proto = dict_to_workflow_definition(data['config'])
workflow = Workflow(name=name, comment=data['comment'],
project_id=data['project_id'],
forkable=data['forkable'],
forked_from=data['forked_from'],
state=WorkflowState.NEW,
target_state=WorkflowState.READY,
transaction_state=TransactionState.READY)
workflow.set_config(template_proto)
db.session.add(workflow)
db.session.commit()
logging.info('Inserted a workflow to db')
scheduler.wakeup(workflow.id)
return {'data': workflow.to_dict()}, HTTPStatus.CREATED
def put(self, workflow_id):
parser = reqparse.RequestParser()
parser.add_argument('config', type=dict, required=True,
help='config is empty')
parser.add_argument('forkable', type=bool, required=True,
help='forkable is empty')
parser.add_argument('create_job_flags', type=list, required=False,
location='json',
help='flags in common.CreateJobFlag')
parser.add_argument(
'batch_update_interval',
type=int,
required=False,
help='interval time for cronjob of workflow in minute')
parser.add_argument('comment')
data = parser.parse_args()
workflow = _get_workflow(workflow_id)
if workflow.config:
raise ResourceConflictException(
'Resetting workflow is not allowed')
batch_update_interval = data['batch_update_interval']
if batch_update_interval:
start_or_stop_cronjob(batch_update_interval, workflow)
workflow.comment = data['comment']
workflow.forkable = data['forkable']
workflow.set_config(dict_to_workflow_definition(data['config']))
workflow.set_create_job_flags(data['create_job_flags'])
workflow.update_target_state(WorkflowState.READY)
db.session.commit()
scheduler.wakeup(workflow_id)
logging.info('update workflow %d target_state to %s',
workflow.id, workflow.target_state)
return {'data': workflow.to_dict()}, HTTPStatus.OK
def post(self):
parser = reqparse.RequestParser()
parser.add_argument('name', required=True, help='name is empty')
parser.add_argument('project_id',
type=int,
required=True,
help='project_id is empty')
# TODO: should verify if the config is compatible with
# workflow template
parser.add_argument('config',
type=dict,
required=True,
help='config is empty')
parser.add_argument('forkable',
type=bool,
required=True,
help='forkable is empty')
parser.add_argument('forked_from',
type=int,
required=False,
help='fork from base workflow')
parser.add_argument('create_job_flags',
type=list,
required=False,
location='json',
help='flags in common.CreateJobFlag')
parser.add_argument('peer_create_job_flags',
type=list,
required=False,
location='json',
help='peer flags in common.CreateJobFlag')
parser.add_argument('fork_proposal_config',
type=dict,
required=False,
help='fork and edit peer config')
parser.add_argument('comment')
data = parser.parse_args()
name = data['name']
if Workflow.query.filter_by(name=name).first() is not None:
raise ResourceConflictException(
'Workflow {} already exists.'.format(name))
# form to proto buffer
template_proto = dict_to_workflow_definition(data['config'])
workflow = Workflow(
name=name,
# 20 bytes
# a DNS-1035 label must start with an
# alphabetic character. substring uuid[:19] has
# no collision in 10 million draws
uuid=f'u{uuid4().hex[:19]}',
comment=data['comment'],
project_id=data['project_id'],
forkable=data['forkable'],
forked_from=data['forked_from'],
state=WorkflowState.NEW,
target_state=WorkflowState.READY,
transaction_state=TransactionState.READY)
workflow.set_create_job_flags(data['create_job_flags'])
if workflow.forked_from is not None:
fork_config = dict_to_workflow_definition(
data['fork_proposal_config'])
# TODO: more validations
if len(fork_config.job_definitions) != \
len(template_proto.job_definitions):
raise InvalidArgumentException(
'Forked workflow\'s template does not match base workflow')
workflow.set_fork_proposal_config(fork_config)
# TODO: check that federated jobs have
# same reuse policy on both sides
workflow.set_peer_create_job_flags(data['peer_create_job_flags'])
workflow.set_config(template_proto)
db.session.add(workflow)
db.session.commit()
logging.info('Inserted a workflow to db')
scheduler.wakeup(workflow.id)
return {'data': workflow.to_dict()}, HTTPStatus.CREATED
def run(self):
if self.status == JobStatus.STARTED:
raise ResourceConflictException('Job has been started')
self.status = JobStatus.STARTED
self._k8s_client.create_flapp(self._project_adapter.
get_namespace(), self.yaml)