def put(self, workflow_id): parser = reqparse.RequestParser() parser.add_argument('config', type=dict, required=True, help='config is empty') parser.add_argument('forkable', type=bool, required=True, help='forkable is empty') parser.add_argument('comment') data = parser.parse_args() workflow = _get_workflow(workflow_id) if workflow.config: raise ResourceConflictException( 'Resetting workflow is not allowed') workflow.comment = data['comment'] workflow.forkable = data['forkable'] workflow.set_config(dict_to_workflow_definition(data['config'])) workflow.update_target_state(WorkflowState.READY) db.session.commit() logging.info('update workflow %d target_state to %s', workflow.id, workflow.target_state) return {'data': workflow.to_dict()}, HTTPStatus.OK
def post(self): parser = reqparse.RequestParser() parser.add_argument('name', required=True, help='name is empty') parser.add_argument('comment') parser.add_argument('config', type=dict, required=True, help='config is empty') data = parser.parse_args() name = data['name'] comment = data['comment'] config = data['config'] if WorkflowTemplate.query.filter_by(name=name).first() is not None: raise ResourceConflictException( 'Workflow template {} already exists'.format(name)) # form to proto buffer template_proto = dict_to_workflow_definition(config) group_template = WorkflowTemplate.query.filter_by( group_alias=template_proto.group_alias).first() if group_template is not None: group_proto = group_template.get_config() if not (check_group_match(group_proto, template_proto) or check_group_same(group_proto, template_proto)): raise InvalidArgumentException( 'The group is not matched with existing groups.') template = WorkflowTemplate(name=name, comment=comment, group_alias=template_proto.group_alias) template.set_config(template_proto) db.session.add(template) db.session.commit() logging.info('Inserted a workflow_template to db') return {'data': template.to_dict()}, HTTPStatus.CREATED
def create_sparkapplication( self, json_object: dict, namespace: str = SPARKOPERATOR_NAMESPACE) -> dict: """ create sparkapp Args: json_object (dict): json object of config namespace (str, optional): namespace to submit. Raises: ApiException Returns: dict: resp of k8s """ try: logging.debug('create sparkapp json is %s', json_object) return self.crds.create_namespaced_custom_object( group=SPARKOPERATOR_CUSTOM_GROUP, version=SPARKOPERATOR_CUSTOM_VERSION, namespace=namespace, plural=CrdKind.SPARK_APPLICATION.value, body=json_object) except ApiException as err: if err.status == 409: raise ResourceConflictException(message=err.reason) if err.status == 400: raise InvalidArgumentException(details=err.reason) raise InternalException(details=err.body)
def post(self): parser = reqparse.RequestParser() parser.add_argument('name', required=True, help='name is empty') parser.add_argument('comment') parser.add_argument('config', type=dict, required=True, help='config is empty') parser.add_argument('editor_info', type=dict, default={}) parser.add_argument('kind', type=int, default=0) data = parser.parse_args() name = data['name'] comment = data['comment'] config = data['config'] editor_info = data['editor_info'] kind = data['kind'] if WorkflowTemplate.query.filter_by(name=name).first() is not None: raise ResourceConflictException( 'Workflow template {} already exists'.format(name)) template_proto, editor_info_proto = _check_config_and_editor_info( config, editor_info) template_proto = _format_template_with_yaml_editor( template_proto, editor_info_proto) template = WorkflowTemplate(name=name, comment=comment, group_alias=template_proto.group_alias, is_left=template_proto.is_left, kind=kind) template.set_config(template_proto) template.set_editor_info(editor_info_proto) db.session.add(template) db.session.commit() logging.info('Inserted a workflow_template to db') result = template.to_dict() return {'data': result}, HTTPStatus.CREATED
def post(self): parser = reqparse.RequestParser() parser.add_argument('username', required=True, help='username is empty') parser.add_argument('password', required=True, help='password is empty') parser.add_argument('role', required=True, help='role is empty') parser.add_argument('name', required=True, help='name is empty') parser.add_argument('email', required=True, help='email is empty') data = parser.parse_args() username = data['username'] password = base64decode(data['password']) role = data['role'] name = data['name'] email = data['email'] check_password_format(password) if User.query.filter_by(username=username).first() is not None: raise ResourceConflictException( 'user {} already exists'.format(username)) user = User(username=username, role=role, name=name, email=email, state=State.ACTIVE) user.set_password(password) db.session.add(user) db.session.commit() return {'data': user.to_dict()}, HTTPStatus.CREATED
def post(self): parser = reqparse.RequestParser() parser.add_argument('name', required=True, help='name is empty') parser.add_argument('comment') parser.add_argument('config', type=dict, required=True, help='config is empty') data = parser.parse_args() name = data['name'] comment = data['comment'] config = data['config'] if 'group_alias' not in config: raise InvalidArgumentException(details={ 'config.group_alias': 'config.group_alias is required'}) if 'is_left' not in config: raise InvalidArgumentException( details={'config.is_left': 'config.is_left is required'}) if WorkflowTemplate.query.filter_by(name=name).first() is not None: raise ResourceConflictException( 'Workflow template {} already exists'.format(name)) # form to proto buffer template_proto = dict_to_workflow_definition(config) template = WorkflowTemplate(name=name, comment=comment, group_alias=template_proto.group_alias, is_left=template_proto.is_left) template.set_config(template_proto) db.session.add(template) db.session.commit() logging.info('Inserted a workflow_template to db') return {'data': template.to_dict()}, HTTPStatus.CREATED
def put(self, template_id): parser = reqparse.RequestParser() parser.add_argument('name', required=True, help='name is empty') parser.add_argument('comment') parser.add_argument('config', type=dict, required=True, help='config is empty') data = parser.parse_args() name = data['name'] comment = data['comment'] config = data['config'] tmp = WorkflowTemplate.query.filter_by(name=name).first() if tmp is not None and tmp.id != template_id: raise ResourceConflictException( 'Workflow template {} already exists'.format(name)) template = WorkflowTemplate.query.filter_by(id=template_id).first() if template is None: raise NotFoundException() template_proto = _check_config(config) template.set_config(template_proto) template.name = name template.comment = comment template.group_alias = template_proto.group_alias template.is_left = template_proto.is_left db.session.commit() return {'data': template.to_dict()}, HTTPStatus.OK
def stop(self): if self.status == JobStatus.STOPPED: raise ResourceConflictException('Job has stopped') self.status = JobStatus.STOPPED self._set_snapshot_flapp() self._set_snapshot_pods() self._k8s_client.delete_flapp(self._project_adapter. get_namespace(), self.name)
def post(self): parser = reqparse.RequestParser() parser.add_argument('name', required=True, help='name is empty') parser.add_argument('project_id', type=int, required=True, help='project_id is empty') # TODO: should verify if the config is compatible with # workflow template parser.add_argument('config', type=dict, required=True, help='config is empty') parser.add_argument('forkable', type=bool, required=True, help='forkable is empty') parser.add_argument('forked_from', type=int, required=False, help='fork from base workflow') parser.add_argument('reuse_job_names', type=list, required=False, location='json', help='fork and inherit jobs') parser.add_argument('peer_reuse_job_names', type=list, required=False, location='json', help='peer fork and inherit jobs') parser.add_argument('fork_proposal_config', type=dict, required=False, help='fork and edit peer config') parser.add_argument('comment') data = parser.parse_args() name = data['name'] if Workflow.query.filter_by(name=name).first() is not None: raise ResourceConflictException( 'Workflow {} already exists.'.format(name)) # form to proto buffer template_proto = dict_to_workflow_definition(data['config']) workflow = Workflow(name=name, comment=data['comment'], project_id=data['project_id'], forkable=data['forkable'], forked_from=data['forked_from'], state=WorkflowState.NEW, target_state=WorkflowState.READY, transaction_state=TransactionState.READY) if workflow.forked_from is not None: fork_config = dict_to_workflow_definition( data['fork_proposal_config']) # TODO: more validations if len(fork_config.job_definitions) != \ len(template_proto.job_definitions): raise InvalidArgumentException( 'Forked workflow\'s template does not match base workflow') workflow.set_fork_proposal_config(fork_config) workflow.set_reuse_job_names(data['reuse_job_names']) workflow.set_peer_reuse_job_names(data['peer_reuse_job_names']) workflow.set_config(template_proto) db.session.add(workflow) db.session.commit() logging.info('Inserted a workflow to db') scheduler.wakeup(workflow.id) return {'data': workflow.to_dict()}, HTTPStatus.CREATED
def post(self): parser = reqparse.RequestParser() parser.add_argument('name', required=True, help='name is empty') parser.add_argument('comment') parser.add_argument('config', type=dict, required=True, help='config is empty') data = parser.parse_args() name = data['name'] comment = data['comment'] config = data['config'] # TODO: format check if 'group_alias' not in config: raise InvalidArgumentException( details={ 'config.group_alias': 'config.group_alias is required' }) if 'is_left' not in config: raise InvalidArgumentException( details={'config.is_left': 'config.is_left is required'}) if WorkflowTemplate.query.filter_by(name=name).first() is not None: raise ResourceConflictException( 'Workflow template {} already exists'.format(name)) # form to proto buffer template_proto = dict_to_workflow_definition(config) for index, job_def in enumerate(template_proto.job_definitions): # pod label name must be no more than 63 characters. # workflow.uuid is 20 characters, pod name suffix such as # '-follower-master-0' is less than 19 characters, so the # job name must be no more than 24 if len(job_def.name) > 24: raise InvalidArgumentException( details={ 'config.job_definitions': 'job_name:{} must be no more than 24 characters' }) # limit from k8s if not re.match('[a-z0-9-]*', job_def.name): raise InvalidArgumentException( details={ f'config.job_definitions[{index}].job_name': 'Only letters(a-z), numbers(0-9) ' 'and dashes(-) are supported.' }) template = WorkflowTemplate(name=name, comment=comment, group_alias=template_proto.group_alias, is_left=template_proto.is_left) template.set_config(template_proto) db.session.add(template) db.session.commit() logging.info('Inserted a workflow_template to db') return {'data': template.to_dict()}, HTTPStatus.CREATED
def post(self): parser = reqparse.RequestParser() parser.add_argument('username', required=True, help='username is empty') parser.add_argument('password', required=True, help='password is empty') data = parser.parse_args() username = data['username'] password = data['password'] if User.query.filter_by(username=username).first() is not None: raise ResourceConflictException( 'user {} already exists'.format(username)) user = User(username=username) user.set_password(password) db.session.add(user) db.session.commit() return {'username': user.username}, HTTPStatus.CREATED
def put(self, template_id): parser = reqparse.RequestParser() parser.add_argument('name', required=True, help='name is empty') parser.add_argument('comment') parser.add_argument('config', type=dict, required=True, help='config is empty') parser.add_argument('editor_info', type=dict, default={}) parser.add_argument('kind', type=int, default=0) data = parser.parse_args() name = data['name'] comment = data['comment'] config = data['config'] editor_info = data['editor_info'] kind = data['kind'] tmp = WorkflowTemplate.query.filter_by(name=name).first() if tmp is not None and tmp.id != template_id: raise ResourceConflictException( 'Workflow template {} already exists'.format(name)) template = WorkflowTemplate.query.filter_by(id=template_id).first() if template is None: raise NotFoundException() template_proto, editor_info_proto = _check_config_and_editor_info( config, editor_info) template_proto = _format_template_with_yaml_editor( template_proto, editor_info_proto) template.set_config(template_proto) template.set_editor_info(editor_info_proto) template.name = name template.comment = comment template.group_alias = template_proto.group_alias template.is_left = template_proto.is_left template.kind = kind db.session.commit() result = template.to_dict() return {'data': result}, HTTPStatus.OK
def post(self): parser = reqparse.RequestParser() parser.add_argument('name', required=True, help='name is empty') parser.add_argument('project_id', type=int, required=True, help='project_id is empty') # TODO: should verify if the config is compatible with # workflow template parser.add_argument('config', type=dict, required=True, help='config is empty') parser.add_argument('forkable', type=bool, required=True, help='forkable is empty') parser.add_argument('forked_from', type=int, required=False, help='forkable is empty') parser.add_argument('comment') data = parser.parse_args() name = data['name'] if Workflow.query.filter_by(name=name).first() is not None: raise ResourceConflictException( 'Workflow {} already exists.'.format(name)) # form to proto buffer template_proto = dict_to_workflow_definition(data['config']) workflow = Workflow(name=name, comment=data['comment'], project_id=data['project_id'], forkable=data['forkable'], forked_from=data['forked_from'], state=WorkflowState.NEW, target_state=WorkflowState.READY, transaction_state=TransactionState.READY) workflow.set_config(template_proto) db.session.add(workflow) db.session.commit() logging.info('Inserted a workflow to db') scheduler.wakeup(workflow.id) return {'data': workflow.to_dict()}, HTTPStatus.CREATED
def put(self, workflow_id): parser = reqparse.RequestParser() parser.add_argument('config', type=dict, required=True, help='config is empty') parser.add_argument('forkable', type=bool, required=True, help='forkable is empty') parser.add_argument('create_job_flags', type=list, required=False, location='json', help='flags in common.CreateJobFlag') parser.add_argument( 'batch_update_interval', type=int, required=False, help='interval time for cronjob of workflow in minute') parser.add_argument('comment') data = parser.parse_args() workflow = _get_workflow(workflow_id) if workflow.config: raise ResourceConflictException( 'Resetting workflow is not allowed') batch_update_interval = data['batch_update_interval'] if batch_update_interval: start_or_stop_cronjob(batch_update_interval, workflow) workflow.comment = data['comment'] workflow.forkable = data['forkable'] workflow.set_config(dict_to_workflow_definition(data['config'])) workflow.set_create_job_flags(data['create_job_flags']) workflow.update_target_state(WorkflowState.READY) db.session.commit() scheduler.wakeup(workflow_id) logging.info('update workflow %d target_state to %s', workflow.id, workflow.target_state) return {'data': workflow.to_dict()}, HTTPStatus.OK
def post(self): parser = reqparse.RequestParser() parser.add_argument('name', required=True, help='name is empty') parser.add_argument('project_id', type=int, required=True, help='project_id is empty') # TODO: should verify if the config is compatible with # workflow template parser.add_argument('config', type=dict, required=True, help='config is empty') parser.add_argument('forkable', type=bool, required=True, help='forkable is empty') parser.add_argument('forked_from', type=int, required=False, help='fork from base workflow') parser.add_argument('create_job_flags', type=list, required=False, location='json', help='flags in common.CreateJobFlag') parser.add_argument('peer_create_job_flags', type=list, required=False, location='json', help='peer flags in common.CreateJobFlag') parser.add_argument('fork_proposal_config', type=dict, required=False, help='fork and edit peer config') parser.add_argument('comment') data = parser.parse_args() name = data['name'] if Workflow.query.filter_by(name=name).first() is not None: raise ResourceConflictException( 'Workflow {} already exists.'.format(name)) # form to proto buffer template_proto = dict_to_workflow_definition(data['config']) workflow = Workflow( name=name, # 20 bytes # a DNS-1035 label must start with an # alphabetic character. substring uuid[:19] has # no collision in 10 million draws uuid=f'u{uuid4().hex[:19]}', comment=data['comment'], project_id=data['project_id'], forkable=data['forkable'], forked_from=data['forked_from'], state=WorkflowState.NEW, target_state=WorkflowState.READY, transaction_state=TransactionState.READY) workflow.set_create_job_flags(data['create_job_flags']) if workflow.forked_from is not None: fork_config = dict_to_workflow_definition( data['fork_proposal_config']) # TODO: more validations if len(fork_config.job_definitions) != \ len(template_proto.job_definitions): raise InvalidArgumentException( 'Forked workflow\'s template does not match base workflow') workflow.set_fork_proposal_config(fork_config) # TODO: check that federated jobs have # same reuse policy on both sides workflow.set_peer_create_job_flags(data['peer_create_job_flags']) workflow.set_config(template_proto) db.session.add(workflow) db.session.commit() logging.info('Inserted a workflow to db') scheduler.wakeup(workflow.id) return {'data': workflow.to_dict()}, HTTPStatus.CREATED
def run(self): if self.status == JobStatus.STARTED: raise ResourceConflictException('Job has been started') self.status = JobStatus.STARTED self._k8s_client.create_flapp(self._project_adapter. get_namespace(), self.yaml)