def do_upgrade(obj: Union[Cluster, HostProvider], upgrade: Upgrade) -> dict: old_proto = obj.prototype check_license(obj.prototype.bundle) check_license(upgrade.bundle) ok, msg = check_upgrade(obj, upgrade) if not ok: return err('UPGRADE_ERROR', msg) log.info('upgrade %s version %s (upgrade #%s)', obj_ref(obj), old_proto.version, upgrade.id) if obj.prototype.type == 'cluster': new_proto = Prototype.objects.get(bundle=upgrade.bundle, type='cluster') elif obj.prototype.type == 'provider': new_proto = Prototype.objects.get(bundle=upgrade.bundle, type='provider') else: return err('UPGRADE_ERROR', 'can upgrade only cluster or host provider') with transaction.atomic(): obj.prototype = new_proto obj.before_upgrade['state'] = obj.state if upgrade.state_on_success: obj.state = upgrade.state_on_success obj.save() switch_config(obj, new_proto, old_proto) if obj.prototype.type == 'cluster': switch_services(upgrade, obj) elif obj.prototype.type == 'provider': switch_hosts(upgrade, obj) cm.issue.update_hierarchy_issues(obj) log.info('upgrade %s OK to version %s', obj_ref(obj), obj.prototype.version) cm.status_api.post_event( 'upgrade', obj.prototype.type, obj.id, 'version', str(obj.prototype.version) ) return {'id': obj.id, 'upgradable': bool(get_upgrade(obj))}
def check_services(): s = {} for p in StagePrototype.objects.filter(type='service'): if p.name in s: msg = 'There are more than one service with name {}' err('BUNDLE_ERROR', msg.format(p.name)) s[p.name] = p.version
def check_adcm_config(conf_file): warnings.simplefilter('error', ruyaml.error.ReusedAnchorWarning) schema_file = os.path.join(config.CODE_DIR, 'cm', 'adcm_schema.yaml') with open(schema_file, encoding='utf_8') as fd: rules = ruyaml.round_trip_load(fd) try: with open(conf_file, encoding='utf_8') as fd: data = cm.checker.round_trip_load(fd, version="1.1", allow_duplicate_keys=True) except (ruyaml.parser.ParserError, ruyaml.scanner.ScannerError, NotImplementedError) as e: err('STACK_LOAD_ERROR', f'YAML decode "{conf_file}" error: {e}') except ruyaml.error.ReusedAnchorWarning as e: err('STACK_LOAD_ERROR', f'YAML decode "{conf_file}" error: {e}') except ruyaml.constructor.DuplicateKeyError as e: msg = f'{e.context}\n{e.context_mark}\n{e.problem}\n{e.problem_mark}' err('STACK_LOAD_ERROR', f'Duplicate Keys error: {msg}') except ruyaml.composer.ComposerError as e: err('STACK_LOAD_ERROR', f'YAML Composer error: {e}') try: cm.checker.check(data, rules) return data except cm.checker.FormatError as e: args = '' if e.errors: for ee in e.errors: if 'Input data for' in ee.message: continue args += f'line {ee.line}: {ee}\n' err('INVALID_OBJECT_DEFINITION', f'"{conf_file}" line {e.line} error: {e}', args) return {}
def set_provider_config(provider_id, keys, value): try: provider = HostProvider.objects.get(id=provider_id) except HostProvider.DoesNotExist: msg = 'Host # {} does not exist' err('PROVIDER_NOT_FOUND', msg.format(provider_id)) return set_object_config(provider, keys, value)
def set_object_config(obj, keys, value): proto = obj.prototype try: spl = keys.split('/') key = spl[0] if len(spl) == 1: subkey = '' else: subkey = spl[1] pconf = PrototypeConfig.objects.get(prototype=proto, action=None, name=key, subname=subkey) except PrototypeConfig.DoesNotExist: msg = '{} does not has config key "{}/{}"' err('CONFIG_NOT_FOUND', msg.format(proto_ref(proto), key, subkey)) if pconf.type == 'group': msg = 'You can not update config group "{}" for {}' err('CONFIG_VALUE_ERROR', msg.format(key, obj_ref(obj))) check_config_type(proto, key, subkey, obj_to_dict(pconf, ('type', 'limits', 'option')), value) # if config_is_ro(obj, keys, pconf.limits): # msg = 'config key {} of {} is read only' # err('CONFIG_VALUE_ERROR', msg.format(key, ref)) replace_object_config(obj, key, subkey, value) if pconf.type == 'file': save_file_type(obj, key, subkey, value) log.info('update %s config %s/%s to "%s"', obj_ref(obj), key, subkey, value) return value
def check_component_constraint_definition(proto, name, conf): if not isinstance(conf, dict): return if 'constraint' not in conf: return const = conf['constraint'] ref = proto_ref(proto) def check_item(item): if isinstance(item, int): return elif item == '+': return elif item == 'odd': return else: msg = 'constraint item of component "{}" in {} should be only digit or "+" or "odd"' err('INVALID_COMPONENT_DEFINITION', msg.format(name, ref)) if not isinstance(const, list): msg = 'constraint of component "{}" in {} should be array' err('INVALID_COMPONENT_DEFINITION', msg.format(name, ref)) if len(const) > 2: msg = 'constraint of component "{}" in {} should have only 1 or 2 elements' err('INVALID_COMPONENT_DEFINITION', msg.format(name, ref)) check_item(const[0]) if len(const) > 1: check_item(const[1])
def set_host_config(host_id, keys, value): try: host = Host.objects.get(id=host_id) except Host.DoesNotExist: msg = 'Host # {} does not exist' err('HOST_NOT_FOUND', msg.format(host_id)) return set_object_config(host, keys, value)
def set_cluster_state(cluster_id, state): try: cluster = Cluster.objects.get(id=cluster_id) except Cluster.DoesNotExist: msg = 'Cluster # {} does not exist' err('CLUSTER_NOT_FOUND', msg.format(cluster_id)) return push_obj(cluster, state)
def set_host_state(host_id, state): try: host = Host.objects.get(id=host_id) except Host.DoesNotExist: msg = 'Host # {} does not exist' err('HOST_NOT_FOUND', msg.format(host_id)) return push_obj(host, state)
def log_check(job_id, group_data, check_data): try: job = JobLog.objects.get(id=job_id) if job.status != config.Job.RUNNING: err('JOB_NOT_FOUND', f'job #{job.id} has status "{job.status}", not "running"') except JobLog.DoesNotExist: err('JOB_NOT_FOUND', f'no job with id #{job_id}') group_title = group_data.pop('title') if group_title: group, _ = GroupCheckLog.objects.get_or_create(job_id=job_id, title=group_title) else: group = None check_data.update({'job_id': job_id, 'group': group}) cl = CheckLog.objects.create(**check_data) if group is not None: group_data.update({'group': group}) log_group_check(**group_data) ls, _ = LogStorage.objects.get_or_create(job=job, name='ansible', type='check', format='json') post_event('add_job_log', 'job', job_id, { 'id': ls.id, 'type': ls.type, 'name': ls.name, 'format': ls.format, }) return cl
def add_host(proto, provider, fqdn, desc='', lock=False): check_proto_type(proto, 'host') check_license(proto.bundle) if proto.bundle != provider.prototype.bundle: msg = 'Host prototype bundle #{} does not match with host provider bundle #{}' err('FOREIGN_HOST', msg.format(proto.bundle.id, provider.prototype.bundle.id)) spec, _, conf, attr = get_prototype_config(proto) event = Event() with transaction.atomic(): obj_conf = init_object_config(spec, conf, attr) host = Host(prototype=proto, provider=provider, fqdn=fqdn, config=obj_conf, description=desc) host.save() if lock: host.stack = ['created'] set_object_state(host, config.Job.LOCKED, event) process_file_type(host, spec, conf) cm.issue.save_issue(host) event.send_state() cm.status_api.post_event('create', 'host', host.id, 'provider', str(provider.id)) cm.status_api.load_service_map() return host
def check_task(action, selector, conf): obj, cluster, provider = get_action_context(action, selector) check_action_state(action, obj) iss = issue.get_issue(obj) if not issue.issue_to_bool(iss): err('TASK_ERROR', 'action has issues', iss) return obj, cluster, provider
def start_task(action_id, selector, conf, attr, hc, hosts, verbose): # pylint: disable=too-many-locals try: action = Action.objects.get(id=action_id) except Action.DoesNotExist: err('ACTION_NOT_FOUND') obj, cluster, provider = check_task(action, selector, conf) act_conf, spec = check_action_config(action, obj, conf, attr) host_map, delta = check_hostcomponentmap(cluster, action, hc) check_action_hosts(action, cluster, provider, hosts) old_hc = api.get_hc(cluster) if action.type not in ['task', 'job']: msg = f'unknown type "{action.type}" for action: {action}, {action.context}: {obj.name}' err('WRONG_ACTION_TYPE', msg) event = Event() task = prepare_task( action, obj, selector, act_conf, attr, spec, old_hc, delta, host_map, cluster, hosts, event, verbose ) event.send_state() run_task(task, event) event.send_state() log_rotation() return task
def get_action_context(action, selector): cluster = None provider = None if action.prototype.type == 'service': check_selector(selector, 'cluster') obj = check_service_task(selector['cluster'], action) cluster = obj.cluster elif action.prototype.type == 'component': check_selector(selector, 'cluster') obj = check_component_task(selector['cluster'], action) cluster = obj.cluster elif action.prototype.type == 'host': check_selector(selector, 'host') obj = check_host(selector['host'], selector) cluster = obj.cluster elif action.prototype.type == 'cluster': check_selector(selector, 'cluster') obj = check_cluster(selector['cluster']) cluster = obj elif action.prototype.type == 'provider': check_selector(selector, 'provider') obj = check_provider(selector['provider']) provider = obj elif action.prototype.type == 'adcm': check_selector(selector, 'adcm') obj = check_adcm(selector['adcm']) else: err('WRONG_ACTION_CONTEXT', f'unknown action context "{action.prototype.type}"') return obj, cluster, provider
def get_config_files(path, bundle_hash): conf_list = [] conf_types = [ ('config.yaml', 'yaml'), ('config.yml', 'yaml'), ('config.toml', 'toml'), ('config.json', 'json'), ] if not os.path.isdir(path): return err( 'STACK_LOAD_ERROR', 'no directory: {}'.format(path), status.HTTP_404_NOT_FOUND ) for root, _, files in os.walk(path): for conf_file, conf_type in conf_types: if conf_file in files: dirs = root.split('/') path = os.path.join('', *dirs[dirs.index(bundle_hash) + 1:]) conf_list.append((path, root + '/' + conf_file, conf_type)) break if not conf_list: msg = 'no config files in stack directory "{}"' return err('STACK_LOAD_ERROR', msg.format(path)) return conf_list
def set_cluster_config(cluster_id, keys, value): try: cluster = Cluster.objects.get(id=cluster_id) except Cluster.DoesNotExist: msg = 'Cluster # {} does not exist' err('CLUSTER_NOT_FOUND', msg.format(cluster_id)) return set_object_config(cluster, keys, value)
def read_definition(conf_file, conf_type): parsers = { 'toml': toml.load, 'yaml': yaml.safe_load, 'json': json.load } fn = parsers[conf_type] if os.path.isfile(conf_file): with open(conf_file) as fd: try: conf = fn(fd) except (toml.TomlDecodeError, IndexError) as e: err('STACK_LOAD_ERROR', 'TOML decode "{}" error: {}'.format(conf_file, e)) except yaml.parser.ParserError as e: err('STACK_LOAD_ERROR', 'YAML decode "{}" error: {}'.format(conf_file, e)) except yaml.composer.ComposerError as e: err('STACK_LOAD_ERROR', 'YAML decode "{}" error: {}'.format(conf_file, e)) except yaml.constructor.ConstructorError as e: err('STACK_LOAD_ERROR', 'YAML decode "{}" error: {}'.format(conf_file, e)) except yaml.scanner.ScannerError as e: err('STACK_LOAD_ERROR', 'YAML decode "{}" error: {}'.format(conf_file, e)) log.info('Read config file: "%s"', conf_file) return conf log.warning('Can not open config file: "%s"', conf_file) return {}
def process_limits(conf, name, subname): opt = {} if conf['type'] == 'option': opt = {'option': conf['option']} elif conf['type'] == 'variant': opt['source'] = check_variant(conf, name, subname) elif conf['type'] == 'integer' or conf['type'] == 'float': if 'min' in conf: opt['min'] = conf['min'] if 'max' in conf: opt['max'] = conf['max'] elif conf['type'] == 'structure': opt['yspec'] = get_yspec(proto, ref, bundle_hash, conf, name, subname) elif is_group(conf): if 'activatable' in conf: opt['activatable'] = conf['activatable'] opt['active'] = False if 'active' in conf: opt['active'] = conf['active'] if 'read_only' in conf and 'writable' in conf: key_ref = f'(config key "{name}/{subname}" of {ref})' msg = 'can not have "read_only" and "writable" simultaneously {}' err('INVALID_CONFIG_DEFINITION', msg.format(key_ref)) for label in ('read_only', 'writable'): if label in conf: opt[label] = conf[label] return opt
def check_action_hosts(action: Action, obj: ADCMEntity, cluster: Cluster, hosts: List[Host]): provider = None if obj.prototype.type == 'provider': provider = obj if not hosts: return if not action.partial_execution: err( 'TASK_ERROR', 'Only action with partial_execution permission can receive host list' ) if not isinstance(hosts, list): err('TASK_ERROR', 'Hosts should be array') for host_id in hosts: if not isinstance(host_id, int): err('TASK_ERROR', f'host id should be integer ({host_id})') host = Host.obj.get(id=host_id) if cluster and host.cluster != cluster: err('TASK_ERROR', f'host #{host_id} does not belong to cluster #{cluster.pk}') if provider and host.provider != provider: err( 'TASK_ERROR', f'host #{host_id} does not belong to host provider #{provider.pk}' )
def add_delta(delta, action, key, fqdn, host): service, comp = key.split('.') if not check_action_hc(action_hc, service, comp, action): msg = (f'no permission to "{action}" component "{comp}" of ' f'service "{service}" to/from hostcomponentmap') err('WRONG_ACTION_HC', msg) add_to_dict(delta[action], key, fqdn, host)
def copy_stage(bundle_hash, bundle_proto): bundle = copy_obj(bundle_proto, Bundle, ('name', 'version', 'edition', 'license_path', 'license_hash', 'description')) bundle.hash = bundle_hash check_license(bundle) if bundle.license_path: bundle.license = 'unaccepted' if check_license(bundle): bundle.license = 'accepted' try: bundle.save() except IntegrityError: msg = 'Bundle "{}" {} already installed' err('BUNDLE_ERROR', msg.format(bundle_proto.name, bundle_proto.version)) stage_prototypes = StagePrototype.objects.exclude(type='component') copy_stage_prototype(stage_prototypes, bundle) for sp in stage_prototypes: p = Prototype.objects.get(name=sp.name, type=sp.type, bundle=bundle) copy_stage_actons(StageAction.objects.filter(prototype=sp), p) copy_stage_config(StagePrototypeConfig.objects.filter(prototype=sp), p) copy_stage_component( StagePrototype.objects.filter(parent=sp, type='component'), sp, p, bundle) for se in StagePrototypeExport.objects.filter(prototype=sp): pe = PrototypeExport(prototype=p, name=se.name) pe.save() copy_stage_import(StagePrototypeImport.objects.filter(prototype=sp), p) copy_stage_sub_actons(bundle) copy_stage_upgrade(StageUpgrade.objects.all(), bundle) return bundle
def var_host_or(cluster, args): if not isinstance(args, list): err('CONFIG_VARIANT_ERROR', 'arguments of "or" predicate should be a list') if not args: return [] return sorted(list(set.union(*[set(a) for a in args])))
def var_host_get_component(cluster, args, service, func): if 'component' not in args: err('CONFIG_VARIANT_ERROR', f'no "component" argument for predicate "{func}"') return ServiceComponent.obj.get(cluster=cluster, service=service, prototype__name=args['component'])
def update_obj_config(obj_conf, conf, attr, desc=''): if not isinstance(attr, dict): err('INVALID_CONFIG_UPDATE', 'attr should be a map') obj = obj_conf.object if obj is None: err('INVALID_CONFIG_UPDATE', f'unknown object type "{obj_conf}"') group = None if isinstance(obj, GroupConfig): group = obj obj = group.object proto = obj.prototype else: proto = obj.prototype old_conf = ConfigLog.objects.get(obj_ref=obj_conf, id=obj_conf.current) if not attr: if old_conf.attr: attr = old_conf.attr new_conf = check_json_config(proto, group or obj, conf, old_conf.config, attr) with transaction.atomic(): cl = save_obj_config(obj_conf, new_conf, attr, desc) cm.issue.update_hierarchy_issues(obj) if hasattr(obj_conf, 'adcm'): prepare_social_auth(new_conf) if group is not None: cm.status_api.post_event('change_config', 'group-config', group.id, 'version', str(cl.id)) else: cm.status_api.post_event('change_config', proto.type, obj.id, 'version', str(cl.id)) return cl
def check_multi_bind(actual_import, cluster, service, export_cluster, export_service, cb_list=None): if actual_import.multibind: return if cb_list is None: cb_list = ClusterBind.objects.filter(cluster=cluster, service=service) for cb in cb_list: if cb.source_service: source_proto = cb.source_service.prototype else: source_proto = cb.source_cluster.prototype if export_service: if source_proto == export_service.prototype: msg = 'can not multi bind {} to {}' err('BIND_ERROR', msg.format(proto_ref(source_proto), obj_ref(cluster))) else: if source_proto == export_cluster.prototype: msg = 'can not multi bind {} to {}' err('BIND_ERROR', msg.format(proto_ref(source_proto), obj_ref(cluster)))
def accept_license(bundle): if not bundle.license_path: err('LICENSE_ERROR', 'This bundle has no license') if bundle.license == 'absent': err('LICENSE_ERROR', 'This bundle has no license') bundle.license = 'accepted' bundle.save()
def add_service_to_cluster(cluster, proto): check_proto_type(proto, 'service') check_license(proto.bundle) if not proto.shared: if cluster.prototype.bundle != proto.bundle: msg = '{} does not belong to bundle "{}" {}' err( 'SERVICE_CONFLICT', msg.format(proto_ref(proto), cluster.prototype.bundle.name, cluster.prototype.version), ) with transaction.atomic(): cs = ClusterObject.objects.create(cluster=cluster, prototype=proto) obj_conf = init_object_config(proto, cs) cs.config = obj_conf cs.save() add_components_to_service(cluster, cs) cm.issue.update_hierarchy_issues(cs) rbac.models.re_apply_object_policy(cluster) cm.status_api.post_event('add', 'service', cs.id, 'cluster', str(cluster.id)) load_service_map() log.info( f'service #{cs.id} {cs.prototype.name} is added to cluster #{cluster.id} {cluster.name}' ) return cs
def check_extra_keys(conf, acceptable, ref): if not isinstance(conf, dict): return for key in conf.keys(): if key not in acceptable: msg = 'Not allowed key "{}" in {} definition' err('INVALID_OBJECT_DEFINITION', msg.format(key, ref))
def save_upgrade(proto, conf): ref = proto_ref(proto) if not in_dict(conf, 'upgrade'): return if not isinstance(conf['upgrade'], list): msg = 'Upgrade definition of {} should be an array' err('INVALID_UPGRADE_DEFINITION', msg.format(ref)) for item in conf['upgrade']: allow = ('versions', 'from_edition', 'states', 'name', 'description') check_extra_keys(item, allow, 'upgrade of {}'.format(ref)) check_upgrade(proto, item) upg = StageUpgrade(name=item['name']) set_version(upg, item) dict_to_obj(item, 'description', upg) if 'states' in item: check_upgrade_states(proto, item) dict_to_obj(item['states'], 'available', upg) if 'available' in item['states']: upg.state_available = item['states']['available'] if 'on_success' in item['states']: upg.state_on_success = item['states']['on_success'] check_upgrade_edition(proto, item) if in_dict(item, 'from_edition'): upg.from_edition = item['from_edition'] upg.save()
def check_upgrade_edition(proto, conf): if 'from_edition' not in conf: return if isinstance(conf['from_edition'], str) and conf['from_edition'] == 'any': return if not isinstance(conf['from_edition'], list): msg = 'from_edition upgrade filed of {} should be array, not string' err('INVALID_UPGRADE_DEFINITION', msg.format(proto_ref(proto)))