class PolicyRule(MetaCRUD): orm_meta = models_manage.PolicyRule _validate = [ crud.ColumnValidator(field='policy_id', validate_on=('create:M', 'update:M')), crud.ColumnValidator(field='rule_id', rule=BackRefValidator(Rule), validate_on=('create:M', 'update:M')), ]
class SubjectTarget(MetaCRUD): orm_meta = models_manage.SubjectTarget _validate = [ crud.ColumnValidator(field='subject_id', validate_on=('create:M', 'update:M')), crud.ColumnValidator(field='target_id', rule=BackRefValidator(Target), validate_on=('create:M', 'update:M')), ]
class ServiceScript(MetaCRUD): orm_meta = models_manage.ServiceScript _default_order = ['-id'] _validate = [ crud.ColumnValidator(field='service', rule=my_validator.LengthValidator(1, 63), validate_on=('create:M', 'update:O')), crud.ColumnValidator(field='content_type', rule_type='in', rule=['shell', 'sql'], validate_on=('create:O', 'update:O'), nullable=True), crud.ColumnValidator(field='content_field', rule=my_validator.LengthValidator(0, 63), validate_on=('create:O', 'update:O'), nullable=True), crud.ColumnValidator(field='endpoint_field', rule=my_validator.LengthValidator(0, 63), validate_on=('create:O', 'update:O'), nullable=True), crud.ColumnValidator(field='endpoint_include', rule=my_validator.LengthValidator(0, 255), validate_on=('create:O', 'update:O'), nullable=True), crud.ColumnValidator(field='created_by', validate_on=('create:O', 'update:O'), nullable=True), crud.ColumnValidator(field='created_time', validate_on=('create:O', 'update:O'), nullable=True), crud.ColumnValidator(field='updated_by', validate_on=('create:O', 'update:O'), nullable=True), crud.ColumnValidator(field='updated_time', validate_on=('create:O', 'update:O'), nullable=True), ] def _before_create(self, resource, validate): super()._before_create(resource, validate) if ('content_field' not in resource and 'endpoint_field' not in resource) or ('content_field' in resource and not resource['content_field'] and 'endpoint_field' in resource and not resource['endpoint_field']): raise exceptions.FieldRequired( attribute='content_field or endpoint_field') def _before_update(self, rid, resource, validate): super()._before_update(rid, resource, validate) if 'content_field' in resource and not resource[ 'content_field'] and 'endpoint_field' in resource and not resource[ 'endpoint_field']: raise exceptions.ValidationError( attribute='content_field and endpoint_field', msg=_('specify at least one field content'))
class BoxManage(MetaCRUD): orm_meta = models_manage.Box _default_order = ['-id'] _validate = [ crud.ColumnValidator(field='name', rule=my_validator.LengthValidator(1, 36), validate_on=('create:M', 'update:O')), crud.ColumnValidator(field='description', rule=my_validator.LengthValidator(0, 63), validate_on=('create:O', 'update:O'), nullable=True), crud.ColumnValidator(field='policy_id', rule=BackRefValidator(Policy), validate_on=('create:M', 'update:O')), crud.ColumnValidator(field='subject_id', rule=BackRefValidator(Subject), validate_on=('create:M', 'update:O')), crud.ColumnValidator(field='created_by', validate_on=('create:O', 'update:O'), nullable=True), crud.ColumnValidator(field='created_time', validate_on=('create:O', 'update:O'), nullable=True), crud.ColumnValidator(field='updated_by', validate_on=('create:O', 'update:O'), nullable=True), crud.ColumnValidator(field='updated_time', validate_on=('create:O', 'update:O'), nullable=True), ]
class Rule(crud.ResourceBase): orm_meta = models_manage.Rule _validate = [ crud.ColumnValidator(field='name', rule=my_validator.LengthValidator(1, 36), validate_on=['create:M', 'update:O']), crud.ColumnValidator(field='description', rule=my_validator.LengthValidator(0, 63), validate_on=('create:O', 'update:O'), nullable=True), crud.ColumnValidator(field='level', rule=validator.NumberValidator(int, range_min=0), validate_on=('create:M', 'update:O')), crud.ColumnValidator(field='effect_on', rule_type='in', rule=['param', 'script'], validate_on=('create:M', 'update:O')), crud.ColumnValidator(field='match_type', rule_type='in', rule=['filter', 'cli', 'sql', 'text', 'fulltext'], validate_on=('create:M', 'update:O')), crud.ColumnValidator(field='match_param_id', rule=BackRefValidator(MatchParam), validate_on=('create:O', 'update:O'), nullable=True), crud.ColumnValidator(field='enabled', rule_type='in', rule=[0, 1], validate_on=('create:M', 'update:O')), crud.ColumnValidator(field='match_value', rule=my_validator.LengthValidator(1, 512), validate_on=('create:M', 'update:O')), ]
class BoxManage(crud.ResourceBase): orm_meta = models_manage.Box _validate = [ crud.ColumnValidator(field='name', rule=my_validator.LengthValidator(1, 36), validate_on=['create:M', 'update:O']), crud.ColumnValidator(field='description', rule=my_validator.LengthValidator(0, 63), validate_on=['create:O', 'update:O'], nullable=True), crud.ColumnValidator(field='policy_id', rule=BackRefValidator(Policy), validate_on=('create:M', 'update:O')), crud.ColumnValidator(field='subject_id', rule=BackRefValidator(Subject), validate_on=('create:M', 'update:O')), ]
class MatchParam(crud.ResourceBase): orm_meta = models_manage.MatchParam _validate = [ crud.ColumnValidator(field='name', rule=my_validator.LengthValidator(1, 36), validate_on=['create:M', 'update:O']), crud.ColumnValidator(field='description', rule=my_validator.LengthValidator(0, 63), validate_on=['create:O', 'update:O'], nullable=True), crud.ColumnValidator(field='type', rule_type='in', rule=['regex', 'cli'], validate_on=('create:M', 'update:O')), crud.ColumnValidator(field='params', rule=validator.TypeValidator(dict), validate_on=('create:M', 'update:O')), ]
class ServiceScript(crud.ResourceBase): orm_meta = models_manage.ServiceScript _validate = [ crud.ColumnValidator(field='service', rule=my_validator.LengthValidator(1, 63), validate_on=['create:M', 'update:O']), crud.ColumnValidator(field='content_type', rule=my_validator.LengthValidator(1, 36), validate_on=['create:O', 'update:O'], nullable=True), crud.ColumnValidator(field='content_field', rule=my_validator.LengthValidator(1, 63), validate_on=['create:O', 'update:O'], nullable=True), crud.ColumnValidator(field='endpoint_field', rule=my_validator.LengthValidator(1, 63), validate_on=['create:O', 'update:O'], nullable=True), ]
class Target(crud.ResourceBase): orm_meta = models_manage.Target _validate = [ crud.ColumnValidator(field='name', rule=my_validator.LengthValidator(1, 36), validate_on=['create:M', 'update:O']), crud.ColumnValidator(field='enabled', rule_type='in', rule=[0, 1], validate_on=('create:M', 'update:O')), crud.ColumnValidator(field='args_scope', rule=my_validator.LengthValidator(1, 512), validate_on=('create:O', 'update:O'), nullable=True), crud.ColumnValidator(field='entity_scope', rule=my_validator.LengthValidator(1, 512), validate_on=('create:O', 'update:O'), nullable=True), ]
class Target(MetaCRUD): orm_meta = models_manage.Target _default_order = ['-id'] _validate = [ crud.ColumnValidator(field='name', rule=my_validator.LengthValidator(1, 36), validate_on=('create:M', 'update:O')), crud.ColumnValidator(field='enabled', rule_type='in', rule=[0, 1], validate_on=('create:M', 'update:O')), crud.ColumnValidator(field='args_scope', rule=my_validator.LengthValidator(0, 512), validate_on=('create:O', 'update:O'), nullable=True), crud.ColumnValidator(field='entity_scope', rule=my_validator.LengthValidator(0, 512), validate_on=('create:O', 'update:O'), nullable=True), crud.ColumnValidator(field='created_by', validate_on=('create:O', 'update:O'), nullable=True), crud.ColumnValidator(field='created_time', validate_on=('create:O', 'update:O'), nullable=True), crud.ColumnValidator(field='updated_by', validate_on=('create:O', 'update:O'), nullable=True), crud.ColumnValidator(field='updated_time', validate_on=('create:O', 'update:O'), nullable=True), ] def create(self, resource, validate=True, detail=False): '''make detail False as default, or it will return subjects.targets extra info[performance issue]''' return super().create(resource, validate=validate, detail=detail) def update(self, rid, resource, filters=None, validate=True, detail=False): '''make detail False as default, or it will return subjects.targets extra info[performance issue]''' return super().update(rid, resource, filters=filters, validate=validate, detail=detail) def delete(self, rid, filters=None, detail=True): ref = super().get(rid) if ref: names = '|'.join([i['name'] for i in ref['subjects']]) if names: raise exceptions.ConflictError(oid=rid, name=names) return super().delete(rid, filters, detail)
class Policy(crud.ResourceBase): orm_meta = models_manage.Policy _validate = [ crud.ColumnValidator(field='name', rule=my_validator.LengthValidator(1, 36), validate_on=['create:M', 'update:O']), crud.ColumnValidator(field='description', rule=my_validator.LengthValidator(0, 63), validate_on=('create:O', 'update:O'), nullable=True), crud.ColumnValidator(field='enabled', rule_type='in', rule=[0, 1], validate_on=('create:M', 'update:O')), crud.ColumnValidator(field='rules', rule=validator.TypeValidator(list), validate_on=('create:O', 'update:O'), orm_required=False) ] def _addtional_create(self, session, resource, created): if 'rules' in resource: refs = resource['rules'] for ref in refs: ref['policy_id'] = created['id'] PolicyRule(transaction=session).create(ref) def _addtional_update(self, session, rid, resource, before_updated, after_updated): if 'rules' in resource: refs = resource['rules'] PolicyRule(transaction=session).delete_all( filters={'policy_id': before_updated['id']}) for ref in refs: ref['policy_id'] = before_updated['id'] PolicyRule(transaction=session).create(ref)
class Box(resource.Box): ''' Rule Resource for CRUD ''' runall_rules = [ crud.ColumnValidator(field='operator', rule=my_validator.LengthValidator(0, 63), validate_on=['check:O'], nullable=True), crud.ColumnValidator(field='serviceName', rule=my_validator.LengthValidator(1, 255), validate_on=['check:M'], nullable=False), crud.ColumnValidator(field='servicePath', rule=my_validator.LengthValidator(0, 255), validate_on=['check:O'], nullable=True), crud.ColumnValidator(field='entityType', rule=my_validator.LengthValidator(0, 255), validate_on=['check:O'], nullable=True), crud.ColumnValidator(field='entityInstances', rule=my_validator.TypeValidator(list), validate_on=['check:M'], nullable=False), crud.ColumnValidator(field='inputParams', rule=my_validator.TypeValidator(list), validate_on=['check:M'], nullable=False), ] def _get_rules(self, data, boxes=None, without_subject_test=False): boxes = boxes if boxes is not None else self.list(filters={ 'policy.enabled': 1, 'subject.enabled': 1, 'enabled': 1 }) rules = {} hasher = hashlib.sha256() hasher.update(json.dumps(data).encode('utf-8')) digest = hasher.hexdigest() LOG.debug('scope test with data - %s ...', str(data)[:4096]) for b in boxes: LOG.info('scope test of box[%s - %s]', b['id'], b['name']) subject_included = False if without_subject_test: subject_included = True else: for target in b['subject']['targets']: target_included = True # target with the same data is cached key = 'scope/target/%s/data/%s' % (target['id'], digest) cached = cache.get(key, 10) if cache.validate(cached): target_included = cached LOG.debug('scope test of target[%s - %s]: %s', target['id'], target['name'], ('accepted' if cached else 'rejected')) else: LOG.debug('scope test of target[%s - %s]', target['id'], target['name']) if target['enabled']: # jsonscope match & exprscope match if target['args_scope']: target_included = scope.JsonScope(target['args_scope']).is_match(data) else: target_included = True if target_included: LOG.debug('args scope: accepted') if target['entity_scope']: target_included = scope.WeCubeScope(target['entity_scope']).is_match( data['entityInstances'], data.get('entityType', None)) else: target_included = True if target_included: LOG.debug('entity scope: accepted') else: LOG.debug('entity scope: rejected') else: LOG.debug('args scope: rejected') else: LOG.debug('target: disabled') target_included = False cache.set(key, target_included) # if any target are included, means subject is inlcuded if target_included: subject_included = True break if subject_included: # extend box rules(enabled) new_rules_map = {} for rule in b['policy']['rules']: if rule['enabled']: new_rules_map[rule['id']] = rule rules.update(new_rules_map) LOG.info('scope test of box[%s - %s]: accepted, rules: %s', b['id'], b['name'], list(new_rules_map.keys())) else: LOG.info('scope test of box[%s - %s]: rejected', b['id'], b['name']) return list(rules.values()) def _rule_grouping(self, rules): # {'filter': [r1, r2], 'cli': [r3], 'sql/text/fulltext/..': [rx...]} results = {} for r in rules: rs = results.setdefault(r['match_type'], []) rs.append(r) return results def simplerun(self, data, rid): '''run box[rid] rule check :param data: see function check :type data: dict :param rid: box id :type rid: any :raises exceptions.NotFoundError: if id of box not found :return: see function check :rtype: see function check ''' # check even box is diabled refs = self.list({'id': rid}) if len(refs) == 0: raise exceptions.NotFoundError(resource='Box') return self.check(data, boxes=refs, without_subject_test=True) def script_check(self, data, boxes=None): '''run boxes rule check :param data: see function check :type data: dict :param boxes: box ids :type boxes: list :return: see function check :rtype: see function check ''' # check box is enabled filters = {'policy.enabled': 1, 'subject.enabled': 1, 'enabled': 1} if boxes: filters['id'] = boxes refs = self.list(filters) results = self.check(data, boxes=refs, without_subject_test=False) associate_instances = data['entityInstances'] text_output = '' if results: table = texttable.Texttable(max_width=120) # { # 'lineno': [start, end], 'level': level of rule, # 'content': content, 'message': rule name, 'script_name': script name # } table.set_cols_align(["l", "c", "l", "l", "l"]) table.set_cols_valign(["m", "m", "m", "m", "m"]) table.header([_("Instance Ids"), _("Line"), _("Content"), _("Message"), _('Source Script')]) for ret in results: associate_ids = ','.join([(inst.get('displayName', '') or '#' + str(inst.get('id', ''))) for inst in associate_instances]) table.add_row([ associate_ids, '%s-%s' % (ret['lineno'][0], ret['lineno'][1]), ret['content'], ret['message'], ret['script_name'] ]) text_output = table.draw() return {'text': text_output, 'data': results} def plugin_check(self, data): '''run plugin params check :param data: input data { "operator": "admin", "serviceName": "qcloud/vm(resource)/create", "servicePath": "/qcloud/v1/vm/create", "entityType": "wecmdb:host_resource_instance", "entityInstances": [{"id": "xxx_xxxxxx"}], // inputs param for serviceName "inputParams": [ {"xml define prop": xxx}, {}, {} ] } :return: see funcion check :rtype: see funcion check ''' results = [] render_results = [] clean_data = crud.ColumnValidator.get_clean_data(self.runall_rules, data, 'check') service = clean_data['serviceName'] entity_instances = clean_data['entityInstances'] input_params = clean_data['inputParams'] for index, input_param in enumerate(input_params): associate_instances = [entity_instances[index] ] if len(input_params) == len(entity_instances) else entity_instances detect_data = { "operator": clean_data.get('operator', None), 'serviceName': service, 'servicePath': clean_data.get('servicePath', None), 'inputParams': input_param, 'scripts': ServiceScript().get_contents(service, input_param), 'entityType': clean_data.get('entityType', None), 'entityInstances': associate_instances } input_results = self.check(detect_data) results.extend(input_results) for input_result in input_results: render_results.append((associate_instances, input_result)) text_output = '' if render_results: table = texttable.Texttable(max_width=120) # { # 'lineno': [start, end], 'level': level of rule, # 'content': content, 'message': rule name, 'script_name': script name # } table.set_cols_align(["l", "c", "l", "l", "l"]) table.set_cols_valign(["m", "m", "m", "m", "m"]) table.header([_("Instance Ids"), _("Line"), _("Content"), _("Message"), _('Source Script')]) for associate_instances, ret in render_results: associate_ids = ','.join([(inst.get('displayName', '') or '#' + str(inst.get('id', ''))) for inst in associate_instances]) table.add_row([ associate_ids, '%s-%s' % (ret['lineno'][0], ret['lineno'][1]), ret['content'], ret['message'], ret['script_name'] ]) text_output = table.draw() return {'text': text_output, 'data': results} def check(self, data, boxes=None, without_subject_test=False, handover_match_params=None): '''check script & param with boxes, return dangerous contents & rule name :param data: data with param & script content { (Optional - JsonScope check)"operator": "admin", (Optional - JsonScope check)"serviceName": "qcloud/vm(resource)/create", (Optional - JsonScope check)"servicePath": "/qcloud/v1/vm/create", (Optional - JsonScope check)"inputParams": {...service input params}, (Must - script check)"scripts": [{"type": None/"sql"/"shell", "content": "...", "name": "filename"}], (Optional - EntityScope check)"entityType": "wecmdb:host_resource_instance", (Must - EntityScope check)"entityInstances": [{"id": "xxx_xxxxxx"}]} :type data: dict :param boxes: specific boxes if any, defaults to None, mean all boxes :type boxes: list of Box, optional :return: list of {'lineno': [start, end], 'level': level of rule, 'content': content, 'message': rule name, 'script_name': script name} :rtype: list ''' results = [] scripts = data['scripts'] rules = self._get_rules(data, boxes=boxes, without_subject_test=without_subject_test) rules = self._rule_grouping(rules) handover_match_params = MatchParam().list({'type': 'cli_handover' }) if handover_match_params is None else handover_match_params for item in scripts: script_name = item.get('name', '') or '' script_content = item.get('content', '') or '' script_type = item.get('type', None) for key, values in rules.items(): script_results = [] if not script_type: script_type = reader.guess(script_content) or 'shell' if key == 'cli' and script_type == 'shell': try: script_results = detector.BashCliDetector(script_content, values, handover_match_params).check() except Exception as e: LOG.exception(e) script_results = [] elif key == 'sql' and script_type == 'sql': script_results = detector.SqlDetector(script_content, values).check() elif key == 'text': script_results = detector.LineTextDetector(script_content, values).check() elif key == 'fulltext': script_results = detector.FullTextDetector(script_content, values).check() for r in script_results: r['script_name'] = script_name results.extend(script_results) # check filter rules global json_results = detector.JsonFilterDetector(data, rules.get('filter', [])).check() for r in json_results: r['script_name'] = '' results.extend(json_results) return results
# coding=utf-8 from __future__ import absolute_import from talos.db import crud from talos.db import converter from wecubek8s.db import validator cluster_rules = [ crud.ColumnValidator(field='name', rule=validator.LengthValidator(1, 255), validate_on=['check:M'], nullable=False), crud.ColumnValidator(field='correlation_id', rule=validator.LengthValidator(1, 64), validate_on=['check:M'], nullable=False), crud.ColumnValidator(field='api_server', rule=validator.LengthValidator(1, 255), validate_on=['check:M'], nullable=False), crud.ColumnValidator(field='token', rule=validator.LengthValidator(1, 2048), validate_on=['check:M'], nullable=False), crud.ColumnValidator(field='metric_host', rule=validator.LengthValidator(0, 63), validate_on=['check:O'], nullable=False), crud.ColumnValidator(field='metric_port', rule=validator.LengthValidator(0, 20),
class Box(object): data_rules = [ crud.ColumnValidator(field='requestId', rule=my_validator.LengthValidator(0, 63), validate_on=['check:O'], nullable=True), crud.ColumnValidator(field='operator', rule=my_validator.LengthValidator(0, 63), validate_on=['check:O'], nullable=True), crud.ColumnValidator(field='serviceName', rule=my_validator.LengthValidator(1, 255), validate_on=['check:M'], nullable=False), crud.ColumnValidator(field='servicePath', rule=my_validator.LengthValidator(0, 255), validate_on=['check:O'], nullable=True), crud.ColumnValidator(field='entityInstances', rule=my_validator.TypeValidator(list), validate_on=['check:M'], nullable=False), crud.ColumnValidator(field='inputs', rule=my_validator.TypeValidator(list), validate_on=['check:M'], nullable=False), ] def check(self, data): ''' input data: { "requestId": "request-001", //仅异步调用需要用到 "operator": "admin", //操作人 "serviceName": "a/b(c)/d" "servicePath": "a/b/run" "entityInstances": [{"id": "xxx_xxxxxx"}] "inputs": [ {"callbackParameter": "", "xml define prop": xxx}, {}, {} ] } ''' from wecube_plugins_itsdangerous.apps.processor import api as processor_api results = [] box = processor_api.Box() clean_data = crud.ColumnValidator.get_clean_data( self.data_rules, data, 'check') service = clean_data['serviceName'] entity_instances = clean_data['entityInstances'] input_params = clean_data['inputs'] for input_param in input_params: detect_data = { 'serviceName': service, 'servicePath': clean_data.get('servicePath', None), 'inputParams': input_param, 'scripts': processor_api.ServiceScript().get_contents( service, input_param), 'entityInstances': entity_instances } input_results = box.check(detect_data) results.append({ 'is_danger': len(input_results) > 0, 'details': input_results }) return results
class Rule(MetaCRUD): orm_meta = models_manage.Rule _default_order = ['-id'] _validate = [ crud.ColumnValidator(field='name', rule=my_validator.LengthValidator(1, 36), validate_on=('create:M', 'update:O')), crud.ColumnValidator(field='description', rule=my_validator.LengthValidator(0, 63), validate_on=('create:O', 'update:O'), nullable=True), crud.ColumnValidator(field='level', rule_type='in', rule=['critical', 'high', 'medium', 'low'], validate_on=('create:M', 'update:O')), crud.ColumnValidator(field='effect_on', rule_type='in', rule=['param', 'script'], validate_on=('create:M', 'update:O')), crud.ColumnValidator(field='match_type', rule_type='in', rule=['filter', 'cli', 'sql', 'text', 'fulltext'], validate_on=('create:M', 'update:O')), crud.ColumnValidator(field='match_param_id', rule=BackRefValidator(MatchParam), validate_on=('create:O', 'update:O'), nullable=True), crud.ColumnValidator(field='enabled', rule_type='in', rule=[0, 1], validate_on=('create:M', 'update:O')), crud.ColumnValidator(field='match_value', rule=my_validator.LengthValidator(0, 512), validate_on=('create:M', 'update:O')), crud.ColumnValidator(field='created_by', validate_on=('create:O', 'update:O'), nullable=True), crud.ColumnValidator(field='created_time', validate_on=('create:O', 'update:O'), nullable=True), crud.ColumnValidator(field='updated_by', validate_on=('create:O', 'update:O'), nullable=True), crud.ColumnValidator(field='updated_time', validate_on=('create:O', 'update:O'), nullable=True), ] def create(self, resource, validate=True, detail=False): '''make detail False as default, or it will return policies.rules extra info[performance issue]''' return super().create(resource, validate=validate, detail=detail) def update(self, rid, resource, filters=None, validate=True, detail=False): '''make detail False as default, or it will return policies.rules extra info[performance issue]''' return super().update(rid, resource, filters=filters, validate=validate, detail=detail) def _addtional_validation(self, resource): if resource['effect_on'] == 'param' and resource['match_type'] not in ( 'filter', ): raise exceptions.ValidationError( attribute='match_type', msg=_( '%(value)s is not acceptable, use %(fixed_value)s instead') % { 'value': resource['match_type'], 'fixed_value': 'filter' }) if resource['effect_on'] == 'script' and resource[ 'match_type'] not in ('cli', 'sql', 'text', 'fulltext'): raise exceptions.ValidationError( attribute='match_type', msg=_( '%(value)s is not acceptable, use %(fixed_value)s instead') % { 'value': resource['match_type'], 'fixed_value': ('cli', 'sql', 'text', 'fulltext') }) if resource['effect_on'] == 'script' and resource[ 'match_type'] == 'cli' and not resource['match_param_id']: raise exceptions.FieldRequired(attribute='match_param_id') def _addtional_create(self, session, resource, created): self._addtional_validation(created) def _addtional_update(self, session, rid, resource, before_updated, after_updated): self._addtional_validation(after_updated) def delete(self, rid, filters=None, detail=True): ref = super().get(rid) if ref: names = '|'.join([i['name'] for i in ref['policies']]) if names: raise exceptions.ConflictError(oid=rid, name=names) return super().delete(rid, filters, detail)
class Subject(MetaCRUD): orm_meta = models_manage.Subject _default_order = ['-id'] _validate = [ crud.ColumnValidator(field='name', rule=my_validator.LengthValidator(1, 36), validate_on=('create:M', 'update:O')), crud.ColumnValidator(field='description', rule=my_validator.LengthValidator(0, 63), validate_on=('create:O', 'update:O'), nullable=True), crud.ColumnValidator(field='enabled', rule_type='in', rule=[0, 1], validate_on=('create:M', 'update:O')), crud.ColumnValidator(field='created_by', validate_on=('create:O', 'update:O'), nullable=True), crud.ColumnValidator(field='created_time', validate_on=('create:O', 'update:O'), nullable=True), crud.ColumnValidator(field='updated_by', validate_on=('create:O', 'update:O'), nullable=True), crud.ColumnValidator(field='updated_time', validate_on=('create:O', 'update:O'), nullable=True), crud.ColumnValidator(field='targets', rule=validator.TypeValidator(list), validate_on=('create:O', 'update:O'), orm_required=False) ] def _addtional_create(self, session, resource, created): if 'targets' in resource: refs = resource['targets'] reduce_refs = list(set(refs)) reduce_refs.sort(key=refs.index) for ref in reduce_refs: new_ref = {} new_ref['subject_id'] = created['id'] new_ref['target_id'] = ref SubjectTarget(transaction=session).create(new_ref) def _addtional_update(self, session, rid, resource, before_updated, after_updated): if 'targets' in resource: refs = resource['targets'] old_refs = [ result['target_id'] for result in SubjectTarget(session=session).list( filters={'subject_id': before_updated['id']}) ] create_refs = list(set(refs) - set(old_refs)) create_refs.sort(key=refs.index) delete_refs = set(old_refs) - set(refs) if delete_refs: SubjectTarget(transaction=session).delete_all( filters={ 'subject_id': before_updated['id'], 'target_id': { 'in': list(delete_refs) } }) for ref in create_refs: new_ref = {} new_ref['subject_id'] = before_updated['id'] new_ref['target_id'] = ref SubjectTarget(transaction=session).create(new_ref) def delete(self, rid, filters=None, detail=True): with self.transaction() as session: refs = BoxManage(session=session).list({'subject_id': rid}) if refs: names = '|'.join([i['name'] for i in refs]) raise exceptions.ConflictError(oid=rid, name=names) SubjectTarget(transaction=session).delete_all( filters={'subject_id': rid}) return super().delete(rid, filters, detail)
class Plugin(BaseController): allow_methods = ('POST', ) _param_rules = [ crud.ColumnValidator(field='requestId', rule=validator.LengthValidator(1, 255), validate_on=['check:O'], nullable=True), crud.ColumnValidator(field='operator', rule=validator.LengthValidator(1, 255), validate_on=['check:O'], nullable=True), crud.ColumnValidator(field='inputs', rule=validator.TypeValidator(list), validate_on=['check:M'], nullable=False), ] def __init__(self, action=None) -> None: super().__init__() self._default_action = action or 'process' def on_post(self, req, resp, **kwargs): self._validate_method(req) self._validate_data(req) resp.json = self.process_post(req, req.json, **kwargs) resp.status = falcon.HTTP_200 def validate_item(self, item_index, item): # do return crud.ColumnValidator.get_clean_data(rules, item, 'check') return item def process(self, reqid, operator, item_index, item, **kwargs): raise NotImplementedError() def process_post(self, req, data, **kwargs): result = { 'resultCode': '0', 'resultMessage': 'success', 'results': { 'outputs': [] } } is_item_error = False error_indexes = [] try: clean_data = crud.ColumnValidator.get_clean_data( self._param_rules, data, 'check') reqid = clean_data.get('requestId', None) or 'N/A' operator = clean_data.get('operator', None) or 'N/A' for idx, item in enumerate(clean_data['inputs']): single_result = { 'callbackParameter': item.get('callbackParameter', None), 'errorCode': '0', 'errorMessage': 'success' } try: validate_item_func = getattr( self, 'validate_item_' + self._default_action, self.validate_item) clean_item = validate_item_func(idx, item) process_func = getattr(self, self._default_action, self.process) process_result = process_func(reqid, operator, idx, clean_item, **kwargs) if process_result: single_result.update(process_result) result['results']['outputs'].append(single_result) except Exception as e: LOG.exception(e) single_result['errorCode'] = '1' single_result['errorMessage'] = str(e) result['results']['outputs'].append(single_result) is_item_error = True error_indexes.append(str(idx + 1)) except Exception as e: LOG.exception(e) result['resultCode'] = '1' result['resultMessage'] = str(e) if is_item_error: result['resultCode'] = '1' result['resultMessage'] = _( 'Fail to process [%(num)s] record, detail error in the data block' ) % dict(num=','.join(error_indexes)) return result
class PackageFromImage(controller.Controller): allow_methods = ('POST', ) name = 'artifacts.plugins.fromimage' param_rules = [ crud.ColumnValidator(field='requestId', rule=validator.LengthValidator(0, 255), validate_on=['check:O'], nullable=True), crud.ColumnValidator(field='operator', rule=validator.LengthValidator(0, 255), validate_on=['check:O'], nullable=True), crud.ColumnValidator(field='inputs', rule=validator.TypeValidator(list), validate_on=['check:M'], nullable=False), ] input_rules = [ crud.ColumnValidator(field='callbackParameter', rule=validator.LengthValidator(0, 255), validate_on=['check:O'], nullable=True), crud.ColumnValidator(field='unit_design', rule=validator.LengthValidator(1, 255), validate_on=['check:M'], nullable=False), crud.ColumnValidator(field='image_name', rule=validator.LengthValidator(1, 255), validate_on=['check:M'], nullable=False), crud.ColumnValidator(field='tag', rule=validator.LengthValidator(1, 255), validate_on=['check:M'], nullable=False), crud.ColumnValidator(field='namespace', rule=validator.LengthValidator(0, 255), validate_on=['check:O'], nullable=True), crud.ColumnValidator(field='md5', rule=validator.LengthValidator(0, 255), validate_on=['check:O'], nullable=True), crud.ColumnValidator(field='nexus_url', rule=validator.LengthValidator(0, 255), validate_on=['check:O'], nullable=True), crud.ColumnValidator(field='connector_port', rule=validator.LengthValidator(0, 255), validate_on=['check:O'], nullable=True), crud.ColumnValidator(field='baseline_package', rule=validator.LengthValidator(0, 255), validate_on=['check:O'], nullable=True), ] def on_post(self, req, resp, **kwargs): self._validate_method(req) self._validate_data(req) resp.json = self.create(req, req.json, **kwargs) resp.status = falcon.HTTP_200 def create(self, req, data): result = {'resultCode': '0', 'resultMessage': 'success', 'results': {'outputs': []}} is_error = False error_indexes = [] try: clean_data = crud.ColumnValidator.get_clean_data(self.param_rules, data, 'check') operator = clean_data.get('operator', None) or 'N/A' for idx, item in enumerate(clean_data['inputs']): single_result = { 'callbackParameter': item.get('callbackParameter', None), 'errorCode': '0', 'errorMessage': 'success', 'guid': None, 'deploy_package_url': None } try: clean_item = crud.ColumnValidator.get_clean_data(self.input_rules, item, 'check') package = plugin_api.Package().create_from_image_name(clean_item['image_name'], clean_item['tag'], clean_item.get('namespace', None), clean_item.get('md5', None), clean_item.get('nexus_url', None), clean_item.get('connector_port', None), clean_item['unit_design'], clean_item.get('baseline_package', None), operator) single_result.update(package) result['results']['outputs'].append(single_result) except Exception as e: single_result['errorCode'] = '1' single_result['errorMessage'] = str(e) result['results']['outputs'].append(single_result) is_error = True error_indexes.append(str(idx + 1)) except Exception as e: result['resultCode'] = '1' result['resultMessage'] = str(e) if is_error: result['resultCode'] = '1' result['resultMessage'] = _('Fail to %(action)s [%(num)s] record, detail error in the data block') % dict( action='process', num=','.join(error_indexes)) return result
class MatchParam(MetaCRUD): orm_meta = models_manage.MatchParam _default_order = ['-id'] _validate = [ crud.ColumnValidator(field='name', rule=my_validator.LengthValidator(1, 36), validate_on=('create:M', 'update:O')), crud.ColumnValidator(field='description', rule=my_validator.LengthValidator(0, 63), validate_on=('create:O', 'update:O'), nullable=True), crud.ColumnValidator(field='type', rule_type='in', rule=['regex', 'cli', 'cli_handover'], validate_on=('create:M', 'update:O')), crud.ColumnValidator(field='params', rule=validator.TypeValidator(dict), validate_on=('create:M', 'update:O')), crud.ColumnValidator(field='created_by', validate_on=('create:O', 'update:O'), nullable=True), crud.ColumnValidator(field='created_time', validate_on=('create:O', 'update:O'), nullable=True), crud.ColumnValidator(field='updated_by', validate_on=('create:O', 'update:O'), nullable=True), crud.ColumnValidator(field='updated_time', validate_on=('create:O', 'update:O'), nullable=True) ] _validate_params_cli = [ crud.ColumnValidator(field='name', rule=my_validator.LengthValidator(1, 36), validate_on=('create:M', )), crud.ColumnValidator(field='opt_strip_path', rule_type='in', rule=[True, False], validate_on=('create:O', ), nullable=True), crud.ColumnValidator(field='args', rule=validator.TypeValidator(list), validate_on=('create:M', )) ] _validate_params_cli_args = [ crud.ColumnValidator(field='name', rule=my_validator.LengthValidator(1, 36), validate_on=('create:M', )), crud.ColumnValidator(field='shortcut', rule=my_validator.LengthValidator(0, 255), validate_on=('create:O', ), nullable=True), crud.ColumnValidator( field='action', rule_type='in', rule=['store', 'store_true', 'store_false', 'count', 'append'], validate_on=('create:O', )), crud.ColumnValidator(field='convert_int', rule_type='in', rule=[True, False], validate_on=('create:O', ), nullable=True), crud.ColumnValidator(field='repeatable', rule=my_validator.RepeatableValidator(), validate_on=('create:O', ), nullable=True), ] def _addtional_create(self, session, resource, created): super()._addtional_create(session, resource, created) if created['type'] in ('cli', 'cli_handover'): params = created['params'] self.validate(params, 'create', rule=self._validate_params_cli) args = params['args'] for arg in args: self.validate(arg, 'create', rule=self._validate_params_cli_args) def _addtional_update(self, session, rid, resource, before_updated, after_updated): super()._addtional_update(session, rid, resource, before_updated, after_updated) if after_updated['type'] in ('cli', 'cli_handover'): params = after_updated['params'] self.validate(params, 'create', rule=self._validate_params_cli) args = params['args'] for arg in args: self.validate(arg, 'create', rule=self._validate_params_cli_args) def delete(self, rid, filters=None, detail=True): refs = Rule().list({'match_param_id': rid}) if refs: names = '|'.join([i['name'] for i in refs]) raise exceptions.ConflictError(oid=rid, name=names) return super().delete(rid, filters, detail)