def do_upgrade(obj: Union[Cluster, HostProvider], upgrade: Upgrade) -> dict:
old_proto = obj.prototype
check_license(obj.prototype.bundle)
check_license(upgrade.bundle)
ok, msg = check_upgrade(obj, upgrade)
if not ok:
return err('UPGRADE_ERROR', msg)
log.info('upgrade %s version %s (upgrade #%s)', obj_ref(obj), old_proto.version, upgrade.id)
if obj.prototype.type == 'cluster':
new_proto = Prototype.objects.get(bundle=upgrade.bundle, type='cluster')
elif obj.prototype.type == 'provider':
new_proto = Prototype.objects.get(bundle=upgrade.bundle, type='provider')
else:
return err('UPGRADE_ERROR', 'can upgrade only cluster or host provider')
with transaction.atomic():
obj.prototype = new_proto
obj.before_upgrade['state'] = obj.state
if upgrade.state_on_success:
obj.state = upgrade.state_on_success
obj.save()
switch_config(obj, new_proto, old_proto)
if obj.prototype.type == 'cluster':
switch_services(upgrade, obj)
elif obj.prototype.type == 'provider':
switch_hosts(upgrade, obj)
cm.issue.update_hierarchy_issues(obj)
log.info('upgrade %s OK to version %s', obj_ref(obj), obj.prototype.version)
cm.status_api.post_event(
'upgrade', obj.prototype.type, obj.id, 'version', str(obj.prototype.version)
)
return {'id': obj.id, 'upgradable': bool(get_upgrade(obj))}
def check_services():
s = {}
for p in StagePrototype.objects.filter(type='service'):
if p.name in s:
msg = 'There are more than one service with name {}'
err('BUNDLE_ERROR', msg.format(p.name))
s[p.name] = p.version
def check_adcm_config(conf_file):
warnings.simplefilter('error', ruyaml.error.ReusedAnchorWarning)
schema_file = os.path.join(config.CODE_DIR, 'cm', 'adcm_schema.yaml')
with open(schema_file, encoding='utf_8') as fd:
rules = ruyaml.round_trip_load(fd)
try:
with open(conf_file, encoding='utf_8') as fd:
data = cm.checker.round_trip_load(fd,
version="1.1",
allow_duplicate_keys=True)
except (ruyaml.parser.ParserError, ruyaml.scanner.ScannerError,
NotImplementedError) as e:
err('STACK_LOAD_ERROR', f'YAML decode "{conf_file}" error: {e}')
except ruyaml.error.ReusedAnchorWarning as e:
err('STACK_LOAD_ERROR', f'YAML decode "{conf_file}" error: {e}')
except ruyaml.constructor.DuplicateKeyError as e:
msg = f'{e.context}\n{e.context_mark}\n{e.problem}\n{e.problem_mark}'
err('STACK_LOAD_ERROR', f'Duplicate Keys error: {msg}')
except ruyaml.composer.ComposerError as e:
err('STACK_LOAD_ERROR', f'YAML Composer error: {e}')
try:
cm.checker.check(data, rules)
return data
except cm.checker.FormatError as e:
args = ''
if e.errors:
for ee in e.errors:
if 'Input data for' in ee.message:
continue
args += f'line {ee.line}: {ee}\n'
err('INVALID_OBJECT_DEFINITION',
f'"{conf_file}" line {e.line} error: {e}', args)
return {}
def set_provider_config(provider_id, keys, value):
try:
provider = HostProvider.objects.get(id=provider_id)
except HostProvider.DoesNotExist:
msg = 'Host # {} does not exist'
err('PROVIDER_NOT_FOUND', msg.format(provider_id))
return set_object_config(provider, keys, value)
def set_object_config(obj, keys, value):
proto = obj.prototype
try:
spl = keys.split('/')
key = spl[0]
if len(spl) == 1:
subkey = ''
else:
subkey = spl[1]
pconf = PrototypeConfig.objects.get(prototype=proto,
action=None,
name=key,
subname=subkey)
except PrototypeConfig.DoesNotExist:
msg = '{} does not has config key "{}/{}"'
err('CONFIG_NOT_FOUND', msg.format(proto_ref(proto), key, subkey))
if pconf.type == 'group':
msg = 'You can not update config group "{}" for {}'
err('CONFIG_VALUE_ERROR', msg.format(key, obj_ref(obj)))
check_config_type(proto, key, subkey,
obj_to_dict(pconf, ('type', 'limits', 'option')), value)
# if config_is_ro(obj, keys, pconf.limits):
# msg = 'config key {} of {} is read only'
# err('CONFIG_VALUE_ERROR', msg.format(key, ref))
replace_object_config(obj, key, subkey, value)
if pconf.type == 'file':
save_file_type(obj, key, subkey, value)
log.info('update %s config %s/%s to "%s"', obj_ref(obj), key, subkey,
value)
return value
def check_component_constraint_definition(proto, name, conf):
if not isinstance(conf, dict):
return
if 'constraint' not in conf:
return
const = conf['constraint']
ref = proto_ref(proto)
def check_item(item):
if isinstance(item, int):
return
elif item == '+':
return
elif item == 'odd':
return
else:
msg = 'constraint item of component "{}" in {} should be only digit or "+" or "odd"'
err('INVALID_COMPONENT_DEFINITION', msg.format(name, ref))
if not isinstance(const, list):
msg = 'constraint of component "{}" in {} should be array'
err('INVALID_COMPONENT_DEFINITION', msg.format(name, ref))
if len(const) > 2:
msg = 'constraint of component "{}" in {} should have only 1 or 2 elements'
err('INVALID_COMPONENT_DEFINITION', msg.format(name, ref))
check_item(const[0])
if len(const) > 1:
check_item(const[1])
def set_host_config(host_id, keys, value):
try:
host = Host.objects.get(id=host_id)
except Host.DoesNotExist:
msg = 'Host # {} does not exist'
err('HOST_NOT_FOUND', msg.format(host_id))
return set_object_config(host, keys, value)
def set_cluster_state(cluster_id, state):
try:
cluster = Cluster.objects.get(id=cluster_id)
except Cluster.DoesNotExist:
msg = 'Cluster # {} does not exist'
err('CLUSTER_NOT_FOUND', msg.format(cluster_id))
return push_obj(cluster, state)
def set_host_state(host_id, state):
try:
host = Host.objects.get(id=host_id)
except Host.DoesNotExist:
msg = 'Host # {} does not exist'
err('HOST_NOT_FOUND', msg.format(host_id))
return push_obj(host, state)
def log_check(job_id, group_data, check_data):
try:
job = JobLog.objects.get(id=job_id)
if job.status != config.Job.RUNNING:
err('JOB_NOT_FOUND', f'job #{job.id} has status "{job.status}", not "running"')
except JobLog.DoesNotExist:
err('JOB_NOT_FOUND', f'no job with id #{job_id}')
group_title = group_data.pop('title')
if group_title:
group, _ = GroupCheckLog.objects.get_or_create(job_id=job_id, title=group_title)
else:
group = None
check_data.update({'job_id': job_id, 'group': group})
cl = CheckLog.objects.create(**check_data)
if group is not None:
group_data.update({'group': group})
log_group_check(**group_data)
ls, _ = LogStorage.objects.get_or_create(job=job, name='ansible', type='check', format='json')
post_event('add_job_log', 'job', job_id, {
'id': ls.id, 'type': ls.type, 'name': ls.name, 'format': ls.format,
})
return cl
def add_host(proto, provider, fqdn, desc='', lock=False):
check_proto_type(proto, 'host')
check_license(proto.bundle)
if proto.bundle != provider.prototype.bundle:
msg = 'Host prototype bundle #{} does not match with host provider bundle #{}'
err('FOREIGN_HOST',
msg.format(proto.bundle.id, provider.prototype.bundle.id))
spec, _, conf, attr = get_prototype_config(proto)
event = Event()
with transaction.atomic():
obj_conf = init_object_config(spec, conf, attr)
host = Host(prototype=proto,
provider=provider,
fqdn=fqdn,
config=obj_conf,
description=desc)
host.save()
if lock:
host.stack = ['created']
set_object_state(host, config.Job.LOCKED, event)
process_file_type(host, spec, conf)
cm.issue.save_issue(host)
event.send_state()
cm.status_api.post_event('create', 'host', host.id, 'provider',
str(provider.id))
cm.status_api.load_service_map()
return host
def check_task(action, selector, conf):
obj, cluster, provider = get_action_context(action, selector)
check_action_state(action, obj)
iss = issue.get_issue(obj)
if not issue.issue_to_bool(iss):
err('TASK_ERROR', 'action has issues', iss)
return obj, cluster, provider
def start_task(action_id, selector, conf, attr, hc, hosts, verbose): # pylint: disable=too-many-locals
try:
action = Action.objects.get(id=action_id)
except Action.DoesNotExist:
err('ACTION_NOT_FOUND')
obj, cluster, provider = check_task(action, selector, conf)
act_conf, spec = check_action_config(action, obj, conf, attr)
host_map, delta = check_hostcomponentmap(cluster, action, hc)
check_action_hosts(action, cluster, provider, hosts)
old_hc = api.get_hc(cluster)
if action.type not in ['task', 'job']:
msg = f'unknown type "{action.type}" for action: {action}, {action.context}: {obj.name}'
err('WRONG_ACTION_TYPE', msg)
event = Event()
task = prepare_task(
action, obj, selector, act_conf, attr, spec, old_hc, delta, host_map, cluster, hosts,
event, verbose
)
event.send_state()
run_task(task, event)
event.send_state()
log_rotation()
return task
def get_action_context(action, selector):
cluster = None
provider = None
if action.prototype.type == 'service':
check_selector(selector, 'cluster')
obj = check_service_task(selector['cluster'], action)
cluster = obj.cluster
elif action.prototype.type == 'component':
check_selector(selector, 'cluster')
obj = check_component_task(selector['cluster'], action)
cluster = obj.cluster
elif action.prototype.type == 'host':
check_selector(selector, 'host')
obj = check_host(selector['host'], selector)
cluster = obj.cluster
elif action.prototype.type == 'cluster':
check_selector(selector, 'cluster')
obj = check_cluster(selector['cluster'])
cluster = obj
elif action.prototype.type == 'provider':
check_selector(selector, 'provider')
obj = check_provider(selector['provider'])
provider = obj
elif action.prototype.type == 'adcm':
check_selector(selector, 'adcm')
obj = check_adcm(selector['adcm'])
else:
err('WRONG_ACTION_CONTEXT', f'unknown action context "{action.prototype.type}"')
return obj, cluster, provider
def get_config_files(path, bundle_hash):
conf_list = []
conf_types = [
('config.yaml', 'yaml'),
('config.yml', 'yaml'),
('config.toml', 'toml'),
('config.json', 'json'),
]
if not os.path.isdir(path):
return err(
'STACK_LOAD_ERROR',
'no directory: {}'.format(path),
status.HTTP_404_NOT_FOUND
)
for root, _, files in os.walk(path):
for conf_file, conf_type in conf_types:
if conf_file in files:
dirs = root.split('/')
path = os.path.join('', *dirs[dirs.index(bundle_hash) + 1:])
conf_list.append((path, root + '/' + conf_file, conf_type))
break
if not conf_list:
msg = 'no config files in stack directory "{}"'
return err('STACK_LOAD_ERROR', msg.format(path))
return conf_list
def set_cluster_config(cluster_id, keys, value):
try:
cluster = Cluster.objects.get(id=cluster_id)
except Cluster.DoesNotExist:
msg = 'Cluster # {} does not exist'
err('CLUSTER_NOT_FOUND', msg.format(cluster_id))
return set_object_config(cluster, keys, value)
def read_definition(conf_file, conf_type):
parsers = {
'toml': toml.load,
'yaml': yaml.safe_load,
'json': json.load
}
fn = parsers[conf_type]
if os.path.isfile(conf_file):
with open(conf_file) as fd:
try:
conf = fn(fd)
except (toml.TomlDecodeError, IndexError) as e:
err('STACK_LOAD_ERROR', 'TOML decode "{}" error: {}'.format(conf_file, e))
except yaml.parser.ParserError as e:
err('STACK_LOAD_ERROR', 'YAML decode "{}" error: {}'.format(conf_file, e))
except yaml.composer.ComposerError as e:
err('STACK_LOAD_ERROR', 'YAML decode "{}" error: {}'.format(conf_file, e))
except yaml.constructor.ConstructorError as e:
err('STACK_LOAD_ERROR', 'YAML decode "{}" error: {}'.format(conf_file, e))
except yaml.scanner.ScannerError as e:
err('STACK_LOAD_ERROR', 'YAML decode "{}" error: {}'.format(conf_file, e))
log.info('Read config file: "%s"', conf_file)
return conf
log.warning('Can not open config file: "%s"', conf_file)
return {}
def process_limits(conf, name, subname):
opt = {}
if conf['type'] == 'option':
opt = {'option': conf['option']}
elif conf['type'] == 'variant':
opt['source'] = check_variant(conf, name, subname)
elif conf['type'] == 'integer' or conf['type'] == 'float':
if 'min' in conf:
opt['min'] = conf['min']
if 'max' in conf:
opt['max'] = conf['max']
elif conf['type'] == 'structure':
opt['yspec'] = get_yspec(proto, ref, bundle_hash, conf, name,
subname)
elif is_group(conf):
if 'activatable' in conf:
opt['activatable'] = conf['activatable']
opt['active'] = False
if 'active' in conf:
opt['active'] = conf['active']
if 'read_only' in conf and 'writable' in conf:
key_ref = f'(config key "{name}/{subname}" of {ref})'
msg = 'can not have "read_only" and "writable" simultaneously {}'
err('INVALID_CONFIG_DEFINITION', msg.format(key_ref))
for label in ('read_only', 'writable'):
if label in conf:
opt[label] = conf[label]
return opt
def check_action_hosts(action: Action, obj: ADCMEntity, cluster: Cluster,
hosts: List[Host]):
provider = None
if obj.prototype.type == 'provider':
provider = obj
if not hosts:
return
if not action.partial_execution:
err(
'TASK_ERROR',
'Only action with partial_execution permission can receive host list'
)
if not isinstance(hosts, list):
err('TASK_ERROR', 'Hosts should be array')
for host_id in hosts:
if not isinstance(host_id, int):
err('TASK_ERROR', f'host id should be integer ({host_id})')
host = Host.obj.get(id=host_id)
if cluster and host.cluster != cluster:
err('TASK_ERROR',
f'host #{host_id} does not belong to cluster #{cluster.pk}')
if provider and host.provider != provider:
err(
'TASK_ERROR',
f'host #{host_id} does not belong to host provider #{provider.pk}'
)
def add_delta(delta, action, key, fqdn, host):
service, comp = key.split('.')
if not check_action_hc(action_hc, service, comp, action):
msg = (f'no permission to "{action}" component "{comp}" of '
f'service "{service}" to/from hostcomponentmap')
err('WRONG_ACTION_HC', msg)
add_to_dict(delta[action], key, fqdn, host)
def copy_stage(bundle_hash, bundle_proto):
bundle = copy_obj(bundle_proto, Bundle,
('name', 'version', 'edition', 'license_path',
'license_hash', 'description'))
bundle.hash = bundle_hash
check_license(bundle)
if bundle.license_path:
bundle.license = 'unaccepted'
if check_license(bundle):
bundle.license = 'accepted'
try:
bundle.save()
except IntegrityError:
msg = 'Bundle "{}" {} already installed'
err('BUNDLE_ERROR', msg.format(bundle_proto.name,
bundle_proto.version))
stage_prototypes = StagePrototype.objects.exclude(type='component')
copy_stage_prototype(stage_prototypes, bundle)
for sp in stage_prototypes:
p = Prototype.objects.get(name=sp.name, type=sp.type, bundle=bundle)
copy_stage_actons(StageAction.objects.filter(prototype=sp), p)
copy_stage_config(StagePrototypeConfig.objects.filter(prototype=sp), p)
copy_stage_component(
StagePrototype.objects.filter(parent=sp, type='component'), sp, p,
bundle)
for se in StagePrototypeExport.objects.filter(prototype=sp):
pe = PrototypeExport(prototype=p, name=se.name)
pe.save()
copy_stage_import(StagePrototypeImport.objects.filter(prototype=sp), p)
copy_stage_sub_actons(bundle)
copy_stage_upgrade(StageUpgrade.objects.all(), bundle)
return bundle
def var_host_or(cluster, args):
if not isinstance(args, list):
err('CONFIG_VARIANT_ERROR',
'arguments of "or" predicate should be a list')
if not args:
return []
return sorted(list(set.union(*[set(a) for a in args])))
def var_host_get_component(cluster, args, service, func):
if 'component' not in args:
err('CONFIG_VARIANT_ERROR',
f'no "component" argument for predicate "{func}"')
return ServiceComponent.obj.get(cluster=cluster,
service=service,
prototype__name=args['component'])
def update_obj_config(obj_conf, conf, attr, desc=''):
if not isinstance(attr, dict):
err('INVALID_CONFIG_UPDATE', 'attr should be a map')
obj = obj_conf.object
if obj is None:
err('INVALID_CONFIG_UPDATE', f'unknown object type "{obj_conf}"')
group = None
if isinstance(obj, GroupConfig):
group = obj
obj = group.object
proto = obj.prototype
else:
proto = obj.prototype
old_conf = ConfigLog.objects.get(obj_ref=obj_conf, id=obj_conf.current)
if not attr:
if old_conf.attr:
attr = old_conf.attr
new_conf = check_json_config(proto, group or obj, conf, old_conf.config,
attr)
with transaction.atomic():
cl = save_obj_config(obj_conf, new_conf, attr, desc)
cm.issue.update_hierarchy_issues(obj)
if hasattr(obj_conf, 'adcm'):
prepare_social_auth(new_conf)
if group is not None:
cm.status_api.post_event('change_config', 'group-config', group.id,
'version', str(cl.id))
else:
cm.status_api.post_event('change_config', proto.type, obj.id,
'version', str(cl.id))
return cl
def check_multi_bind(actual_import,
cluster,
service,
export_cluster,
export_service,
cb_list=None):
if actual_import.multibind:
return
if cb_list is None:
cb_list = ClusterBind.objects.filter(cluster=cluster, service=service)
for cb in cb_list:
if cb.source_service:
source_proto = cb.source_service.prototype
else:
source_proto = cb.source_cluster.prototype
if export_service:
if source_proto == export_service.prototype:
msg = 'can not multi bind {} to {}'
err('BIND_ERROR',
msg.format(proto_ref(source_proto), obj_ref(cluster)))
else:
if source_proto == export_cluster.prototype:
msg = 'can not multi bind {} to {}'
err('BIND_ERROR',
msg.format(proto_ref(source_proto), obj_ref(cluster)))
def accept_license(bundle):
if not bundle.license_path:
err('LICENSE_ERROR', 'This bundle has no license')
if bundle.license == 'absent':
err('LICENSE_ERROR', 'This bundle has no license')
bundle.license = 'accepted'
bundle.save()
def add_service_to_cluster(cluster, proto):
check_proto_type(proto, 'service')
check_license(proto.bundle)
if not proto.shared:
if cluster.prototype.bundle != proto.bundle:
msg = '{} does not belong to bundle "{}" {}'
err(
'SERVICE_CONFLICT',
msg.format(proto_ref(proto), cluster.prototype.bundle.name,
cluster.prototype.version),
)
with transaction.atomic():
cs = ClusterObject.objects.create(cluster=cluster, prototype=proto)
obj_conf = init_object_config(proto, cs)
cs.config = obj_conf
cs.save()
add_components_to_service(cluster, cs)
cm.issue.update_hierarchy_issues(cs)
rbac.models.re_apply_object_policy(cluster)
cm.status_api.post_event('add', 'service', cs.id, 'cluster',
str(cluster.id))
load_service_map()
log.info(
f'service #{cs.id} {cs.prototype.name} is added to cluster #{cluster.id} {cluster.name}'
)
return cs
def check_extra_keys(conf, acceptable, ref):
if not isinstance(conf, dict):
return
for key in conf.keys():
if key not in acceptable:
msg = 'Not allowed key "{}" in {} definition'
err('INVALID_OBJECT_DEFINITION', msg.format(key, ref))
def save_upgrade(proto, conf):
ref = proto_ref(proto)
if not in_dict(conf, 'upgrade'):
return
if not isinstance(conf['upgrade'], list):
msg = 'Upgrade definition of {} should be an array'
err('INVALID_UPGRADE_DEFINITION', msg.format(ref))
for item in conf['upgrade']:
allow = ('versions', 'from_edition', 'states', 'name', 'description')
check_extra_keys(item, allow, 'upgrade of {}'.format(ref))
check_upgrade(proto, item)
upg = StageUpgrade(name=item['name'])
set_version(upg, item)
dict_to_obj(item, 'description', upg)
if 'states' in item:
check_upgrade_states(proto, item)
dict_to_obj(item['states'], 'available', upg)
if 'available' in item['states']:
upg.state_available = item['states']['available']
if 'on_success' in item['states']:
upg.state_on_success = item['states']['on_success']
check_upgrade_edition(proto, item)
if in_dict(item, 'from_edition'):
upg.from_edition = item['from_edition']
upg.save()
def check_upgrade_edition(proto, conf):
if 'from_edition' not in conf:
return
if isinstance(conf['from_edition'], str) and conf['from_edition'] == 'any':
return
if not isinstance(conf['from_edition'], list):
msg = 'from_edition upgrade filed of {} should be array, not string'
err('INVALID_UPGRADE_DEFINITION', msg.format(proto_ref(proto)))
