def post(self):
parser = reqparse.RequestParser()
parser.add_argument('name', required=True, help='name is empty')
parser.add_argument('project_id', type=int, required=True,
help='project_id is empty')
# TODO: should verify if the config is compatible with
# workflow template
parser.add_argument('config', type=dict, required=True,
help='config is empty')
parser.add_argument('forkable', type=bool, required=True,
help='forkable is empty')
parser.add_argument('forked_from', type=int, required=False,
help='fork from base workflow')
parser.add_argument('reuse_job_names', type=list, required=False,
location='json', help='fork and inherit jobs')
parser.add_argument('peer_reuse_job_names', type=list,
required=False, location='json',
help='peer fork and inherit jobs')
parser.add_argument('fork_proposal_config', type=dict, required=False,
help='fork and edit peer config')
parser.add_argument('comment')
data = parser.parse_args()
name = data['name']
if Workflow.query.filter_by(name=name).first() is not None:
raise ResourceConflictException(
'Workflow {} already exists.'.format(name))
# form to proto buffer
template_proto = dict_to_workflow_definition(data['config'])
workflow = Workflow(name=name, comment=data['comment'],
project_id=data['project_id'],
forkable=data['forkable'],
forked_from=data['forked_from'],
state=WorkflowState.NEW,
target_state=WorkflowState.READY,
transaction_state=TransactionState.READY)
if workflow.forked_from is not None:
fork_config = dict_to_workflow_definition(
data['fork_proposal_config'])
# TODO: more validations
if len(fork_config.job_definitions) != \
len(template_proto.job_definitions):
raise InvalidArgumentException(
'Forked workflow\'s template does not match base workflow')
workflow.set_fork_proposal_config(fork_config)
workflow.set_reuse_job_names(data['reuse_job_names'])
workflow.set_peer_reuse_job_names(data['peer_reuse_job_names'])
workflow.set_config(template_proto)
db.session.add(workflow)
db.session.commit()
logging.info('Inserted a workflow to db')
scheduler.wakeup(workflow.id)
return {'data': workflow.to_dict()}, HTTPStatus.CREATED
def put(self, workflow_id):
parser = reqparse.RequestParser()
parser.add_argument('config',
type=dict,
required=True,
help='config is empty')
parser.add_argument('forkable',
type=bool,
required=True,
help='forkable is empty')
parser.add_argument('comment')
data = parser.parse_args()
workflow = _get_workflow(workflow_id)
if workflow.config:
raise ResourceConflictException(
'Resetting workflow is not allowed')
workflow.comment = data['comment']
workflow.forkable = data['forkable']
workflow.set_config(dict_to_workflow_definition(data['config']))
workflow.update_target_state(WorkflowState.READY)
db.session.commit()
logging.info('update workflow %d target_state to %s', workflow.id,
workflow.target_state)
return {'data': workflow.to_dict()}, HTTPStatus.OK
def test_dict_to_workflow_definition(self):
config = {
'variables': [{
'name': 'code',
'value': '{"asdf.py": "asdf"}',
'value_type': 'CODE'
}]
}
proto = dict_to_workflow_definition(config)
self.assertTrue(isinstance(proto.variables[0].value, str))
def put(self, workflow_id):
parser = reqparse.RequestParser()
parser.add_argument('config', type=dict, required=True,
help='config is empty')
parser.add_argument('forkable', type=bool, required=True,
help='forkable is empty')
parser.add_argument('comment')
data = parser.parse_args()
workflow = _get_workflow(workflow_id)
workflow.comment = data['comment']
workflow.forkable = data['forkable']
workflow.set_config(dict_to_workflow_definition(data['config']))
workflow.prepare()
db.session.commit()
logging.info('update workflow %d target_state to %s',
workflow.id, workflow.target_state)
return {'data': workflow.to_dict()}, HTTPStatus.OK
def patch(self, workflow_id):
parser = reqparse.RequestParser()
parser.add_argument('target_state', type=str, required=False,
default=None, help='target_state is empty')
parser.add_argument('forkable', type=bool)
parser.add_argument('config', type=dict, required=False,
default=None, help='updated config')
data = parser.parse_args()
workflow = _get_workflow(workflow_id)
forkable = data['forkable']
if forkable is not None:
workflow.forkable = forkable
db.session.commit()
target_state = data['target_state']
if target_state:
try:
db.session.refresh(workflow)
workflow.update_target_state(WorkflowState[target_state])
db.session.commit()
logging.info('updated workflow %d target_state to %s',
workflow.id, workflow.target_state)
scheduler.wakeup(workflow.id)
except ValueError as e:
raise InvalidArgumentException(details=str(e)) from e
config = data['config']
if config:
try:
db.session.refresh(workflow)
if workflow.target_state != WorkflowState.INVALID or \
workflow.state not in \
[WorkflowState.READY, WorkflowState.STOPPED]:
raise NoAccessException('Cannot edit running workflow')
config_proto = dict_to_workflow_definition(data['config'])
workflow.set_config(config_proto)
db.session.commit()
except ValueError as e:
raise InvalidArgumentException(details=str(e)) from e
return {'data': workflow.to_dict()}, HTTPStatus.OK
def patch(self, workflow_id):
parser = reqparse.RequestParser()
parser.add_argument('config', type=dict, required=True,
help='new config for peer')
data = parser.parse_args()
config_proto = dict_to_workflow_definition(data['config'])
workflow = _get_workflow(workflow_id)
project_config = workflow.project.get_config()
peer_workflows = {}
for party in project_config.participants:
client = RpcClient(project_config, party)
resp = client.update_workflow(
workflow.name, config_proto)
if resp.status.code != common_pb2.STATUS_SUCCESS:
raise InternalException(resp.status.msg)
peer_workflows[party.name] = MessageToDict(
resp,
preserving_proto_field_name=True,
including_default_value_fields=True)
return {'data': peer_workflows}, HTTPStatus.OK
def post(self):
parser = reqparse.RequestParser()
parser.add_argument('name', required=True, help='name is empty')
parser.add_argument('project_id', type=int, required=True,
help='project_id is empty')
# TODO: should verify if the config is compatible with
# workflow template
parser.add_argument('config', type=dict, required=True,
help='config is empty')
parser.add_argument('forkable', type=bool, required=True,
help='forkable is empty')
parser.add_argument('forked_from', type=int, required=False,
help='forkable is empty')
parser.add_argument('comment')
data = parser.parse_args()
name = data['name']
if Workflow.query.filter_by(name=name).first() is not None:
raise ResourceConflictException(
'Workflow {} already exists.'.format(name))
# form to proto buffer
template_proto = dict_to_workflow_definition(data['config'])
workflow = Workflow(name=name, comment=data['comment'],
project_id=data['project_id'],
forkable=data['forkable'],
forked_from=data['forked_from'],
state=WorkflowState.NEW,
target_state=WorkflowState.READY,
transaction_state=TransactionState.READY)
workflow.set_config(template_proto)
db.session.add(workflow)
db.session.commit()
logging.info('Inserted a workflow to db')
scheduler.wakeup(workflow.id)
return {'data': workflow.to_dict()}, HTTPStatus.CREATED
def put(self, workflow_id):
parser = reqparse.RequestParser()
parser.add_argument('config', type=dict, required=True,
help='config is empty')
parser.add_argument('forkable', type=bool, required=True,
help='forkable is empty')
parser.add_argument('create_job_flags', type=list, required=False,
location='json',
help='flags in common.CreateJobFlag')
parser.add_argument(
'batch_update_interval',
type=int,
required=False,
help='interval time for cronjob of workflow in minute')
parser.add_argument('comment')
data = parser.parse_args()
workflow = _get_workflow(workflow_id)
if workflow.config:
raise ResourceConflictException(
'Resetting workflow is not allowed')
batch_update_interval = data['batch_update_interval']
if batch_update_interval:
start_or_stop_cronjob(batch_update_interval, workflow)
workflow.comment = data['comment']
workflow.forkable = data['forkable']
workflow.set_config(dict_to_workflow_definition(data['config']))
workflow.set_create_job_flags(data['create_job_flags'])
workflow.update_target_state(WorkflowState.READY)
db.session.commit()
scheduler.wakeup(workflow_id)
logging.info('update workflow %d target_state to %s',
workflow.id, workflow.target_state)
return {'data': workflow.to_dict()}, HTTPStatus.OK
def patch(self, workflow_id):
parser = reqparse.RequestParser()
parser.add_argument('target_state', type=str, required=False,
default=None, help='target_state is empty')
parser.add_argument('state', type=str, required=False,
default=None, help='state is empty')
parser.add_argument('forkable', type=bool)
parser.add_argument('metric_is_public', type=bool)
parser.add_argument('config', type=dict, required=False,
default=None, help='updated config')
data = parser.parse_args()
workflow = _get_workflow(workflow_id)
forkable = data['forkable']
if forkable is not None:
workflow.forkable = forkable
db.session.flush()
metric_is_public = data['metric_is_public']
if metric_is_public is not None:
workflow.metric_is_public = metric_is_public
db.session.flush()
target_state = data['target_state']
if target_state:
try:
if WorkflowState[target_state] == WorkflowState.RUNNING:
for job in workflow.owned_jobs:
try:
generate_job_run_yaml(job)
# TODO: check if peer variables is valid
except RuntimeError as e:
raise ValueError(
f'Invalid Variable when try '
f'to format the job {job.name}:{str(e)}')
workflow.update_target_state(WorkflowState[target_state])
db.session.flush()
logging.info('updated workflow %d target_state to %s',
workflow.id, workflow.target_state)
scheduler.wakeup(workflow.id)
except ValueError as e:
raise InvalidArgumentException(details=str(e)) from e
state = data['state']
if state:
try:
assert state == 'INVALID', \
'Can only set state to INVALID for invalidation'
workflow.invalidate()
db.session.flush()
logging.info('invalidate workflow %d', workflow.id)
except ValueError as e:
raise InvalidArgumentException(details=str(e)) from e
config = data['config']
if config:
try:
if workflow.target_state != WorkflowState.INVALID or \
workflow.state not in \
[WorkflowState.READY, WorkflowState.STOPPED]:
raise NoAccessException('Cannot edit running workflow')
config_proto = dict_to_workflow_definition(data['config'])
workflow.set_config(config_proto)
db.session.flush()
except ValueError as e:
raise InvalidArgumentException(details=str(e)) from e
db.session.commit()
return {'data': workflow.to_dict()}, HTTPStatus.OK
def post(self):
parser = reqparse.RequestParser()
parser.add_argument('name', required=True, help='name is empty')
parser.add_argument('project_id',
type=int,
required=True,
help='project_id is empty')
# TODO: should verify if the config is compatible with
# workflow template
parser.add_argument('config',
type=dict,
required=True,
help='config is empty')
parser.add_argument('forkable',
type=bool,
required=True,
help='forkable is empty')
parser.add_argument('forked_from',
type=int,
required=False,
help='fork from base workflow')
parser.add_argument('create_job_flags',
type=list,
required=False,
location='json',
help='flags in common.CreateJobFlag')
parser.add_argument('peer_create_job_flags',
type=list,
required=False,
location='json',
help='peer flags in common.CreateJobFlag')
parser.add_argument('fork_proposal_config',
type=dict,
required=False,
help='fork and edit peer config')
parser.add_argument('comment')
data = parser.parse_args()
name = data['name']
if Workflow.query.filter_by(name=name).first() is not None:
raise ResourceConflictException(
'Workflow {} already exists.'.format(name))
# form to proto buffer
template_proto = dict_to_workflow_definition(data['config'])
workflow = Workflow(
name=name,
# 20 bytes
# a DNS-1035 label must start with an
# alphabetic character. substring uuid[:19] has
# no collision in 10 million draws
uuid=f'u{uuid4().hex[:19]}',
comment=data['comment'],
project_id=data['project_id'],
forkable=data['forkable'],
forked_from=data['forked_from'],
state=WorkflowState.NEW,
target_state=WorkflowState.READY,
transaction_state=TransactionState.READY)
workflow.set_create_job_flags(data['create_job_flags'])
if workflow.forked_from is not None:
fork_config = dict_to_workflow_definition(
data['fork_proposal_config'])
# TODO: more validations
if len(fork_config.job_definitions) != \
len(template_proto.job_definitions):
raise InvalidArgumentException(
'Forked workflow\'s template does not match base workflow')
workflow.set_fork_proposal_config(fork_config)
# TODO: check that federated jobs have
# same reuse policy on both sides
workflow.set_peer_create_job_flags(data['peer_create_job_flags'])
workflow.set_config(template_proto)
db.session.add(workflow)
db.session.commit()
logging.info('Inserted a workflow to db')
scheduler.wakeup(workflow.id)
return {'data': workflow.to_dict()}, HTTPStatus.CREATED
def patch(self, workflow_id):
parser = reqparse.RequestParser()
parser.add_argument('target_state', type=str, required=False,
default=None, help='target_state is empty')
parser.add_argument('state',
type=str,
required=False,
help='state is empty')
parser.add_argument('forkable', type=bool)
parser.add_argument('metric_is_public', type=bool)
parser.add_argument('config',
type=dict,
required=False,
help='updated config')
parser.add_argument('create_job_flags', type=list, required=False,
location='json',
help='flags in common.CreateJobFlag')
parser.add_argument('batch_update_interval',
type=int,
required=False,
help='interval for restart workflow in minute')
data = parser.parse_args()
workflow = _get_workflow(workflow_id)
# start workflow every interval time
batch_update_interval = data['batch_update_interval']
if batch_update_interval:
start_or_stop_cronjob(batch_update_interval, workflow)
forkable = data['forkable']
if forkable is not None:
workflow.forkable = forkable
db.session.flush()
metric_is_public = data['metric_is_public']
if metric_is_public is not None:
workflow.metric_is_public = metric_is_public
db.session.flush()
target_state = data['target_state']
if target_state:
try:
if WorkflowState[target_state] == WorkflowState.RUNNING:
for job in workflow.owned_jobs:
try:
generate_job_run_yaml(job)
# TODO: check if peer variables is valid
except Exception as e: # pylint: disable=broad-except
raise ValueError(
f'Invalid Variable when try '
f'to format the job {job.name}:{str(e)}')
workflow.update_target_state(WorkflowState[target_state])
db.session.flush()
logging.info('updated workflow %d target_state to %s',
workflow.id, workflow.target_state)
except ValueError as e:
raise InvalidArgumentException(details=str(e)) from e
state = data['state']
if state:
try:
assert state == 'INVALID', \
'Can only set state to INVALID for invalidation'
workflow.invalidate()
db.session.flush()
logging.info('invalidate workflow %d', workflow.id)
except ValueError as e:
raise InvalidArgumentException(details=str(e)) from e
config = data['config']
if config:
try:
if workflow.target_state != WorkflowState.INVALID or \
workflow.state not in \
[WorkflowState.READY, WorkflowState.STOPPED]:
raise NoAccessException('Cannot edit running workflow')
config_proto = dict_to_workflow_definition(data['config'])
workflow.set_config(config_proto)
db.session.flush()
except ValueError as e:
raise InvalidArgumentException(details=str(e)) from e
create_job_flags = data['create_job_flags']
if create_job_flags:
jobs = workflow.get_jobs()
if len(create_job_flags) != len(jobs):
raise InvalidArgumentException(
details='Number of job defs does not match number '
f'of create_job_flags {len(jobs)} '
f'vs {len(create_job_flags)}')
workflow.set_create_job_flags(create_job_flags)
flags = workflow.get_create_job_flags()
for i, job in enumerate(jobs):
if job.workflow_id == workflow.id:
job.is_disabled = flags[i] == \
common_pb2.CreateJobFlag.DISABLED
db.session.commit()
scheduler.wakeup(workflow.id)
return {'data': workflow.to_dict()}, HTTPStatus.OK
