def make_template(num_masters, gen_arguments, varietal, bootstrap_variant_prefix):
'''
Return a tuple: the generated template for num_masters and the artifact dict.
@param num_masters: int, number of master nodes to embed in the generated template
@param gen_arguments: dict, args to pass to the gen library. These are user
input arguments which get filled in/prompted for.
@param varietal: string, indicate template varietal to build for either 'acs' or 'dcos'
'''
master_list_source = Source()
master_list_source.add_must('master_list', Late(master_list_arm_json(num_masters, varietal)))
master_list_source.add_must('num_masters', str(num_masters))
if varietal == 'dcos':
arm, results = gen_templates(
gen_arguments,
'azuredeploy',
extra_sources=[master_list_source, azure_dcos_source])
elif varietal == 'acs':
arm, results = gen_templates(
gen_arguments,
'acs',
extra_sources=[master_list_source, azure_acs_source])
else:
raise ValueError("Unknown Azure varietal specified")
yield {'packages': results.config_package_ids}
if results.late_package_id:
yield {'packages': [results.late_package_id]}
yield {
'channel_path': 'azure/{}{}-{}master.azuredeploy.json'.format(bootstrap_variant_prefix, varietal, num_masters),
'local_content': arm,
'content_type': 'application/json; charset=utf-8'
}
def make(num_masters, filename):
num_masters_source = Source()
num_masters_source.add_must('num_masters', str(num_masters))
yield from gen_simple_template(
variant_prefix,
filename,
variant_base_args,
num_masters_source)
def make(num_masters, filename):
num_masters_source = Source()
num_masters_source.add_must('num_masters', str(num_masters))
yield from gen_simple_template(
variant_prefix,
filename,
variant_base_args,
num_masters_source)
def test_resolve_simple():
test_user_source = Source(is_user=True)
test_user_source.add_must('c', 'c_str')
test_user_source.add_must('d_1_a', 'd_1_a_str')
resolver = gen.internals.resolve_configuration([test_source, test_user_source], [get_test_target()])
print(resolver)
assert resolver.status_dict == {'status': 'ok'}
# Make sure having a unset variable results in a non-ok status
test_partial_source = Source(is_user=True)
test_partial_source.add_must('d_1_a', 'd_1_a_str')
resolver = gen.internals.resolve_configuration([test_source, test_partial_source], [get_test_target()])
print(resolver)
assert resolver.status_dict == {'status': 'errors', 'errors': {}, 'unset': {'c'}}
def test_resolve_simple():
test_source = Source({
'validate': [validate_a],
'default': {
'a': 'a_str',
'd': 'd_1',
},
'must': {
'b': 'b_str'
},
'conditional': {
'd': {
'd_1': {
'must': {
'd_1_b': 'd_1_b_str'
}
},
'd_2': {
'must': {
'd_2_b': 'd_2_b_str'
}
}
}
}
})
def get_test_target():
return Target({'a', 'b', 'c'}, {
'd':
Scope(
'd', {
'd_1': Target({'d_1_a', 'd_1_b'}),
'd_2': Target({'d_2_a', 'd_2_b'})
})
})
resolver = gen.internals.resolve_configuration([test_source],
[get_test_target()], {
'c': 'c_str',
'd_1_a': 'd_1_a_str'
})
print(resolver)
assert resolver.status_dict == {'status': 'ok'}
# Make sure having a unset variable results in a non-ok status
resolver = gen.internals.resolve_configuration([test_source],
[get_test_target()],
{'d_1_a': 'd_1_a_str'})
print(resolver)
assert resolver.status_dict == {
'status': 'errors',
'errors': {},
'unset': {'c'}
}
def make_template(num_masters, gen_arguments, varietal,
bootstrap_variant_prefix):
'''
Return a tuple: the generated template for num_masters and the artifact dict.
@param num_masters: int, number of master nodes to embed in the generated template
@param gen_arguments: dict, args to pass to the gen library. These are user
input arguments which get filled in/prompted for.
@param varietal: string, indicate template varietal to build for either 'acs' or 'dcos'
'''
master_list_source = Source()
master_list_source.add_must('master_list',
master_list_arm_json(num_masters, varietal))
if varietal == 'dcos':
dcos_template = gen_templates(
gen_arguments,
'azuredeploy',
extra_sources=[master_list_source, azure_dcos_source])
elif varietal == 'acs':
dcos_template = gen_templates(
gen_arguments,
'acs',
extra_sources=[master_list_source, azure_acs_source])
else:
raise ValueError("Unknown Azure varietal specified")
yield {
'packages':
util.cluster_to_extra_packages(
dcos_template['results'].cluster_packages)
}
yield {
'channel_path':
'azure/{}{}-{}master.azuredeploy.json'.format(bootstrap_variant_prefix,
varietal, num_masters),
'local_content':
dcos_template['arm'],
'content_type':
'application/json; charset=utf-8'
}
def validate_config(user_config: dict) -> gen.internals.Resolver:
""" Converts the user config to to a source and evaluates it
versus the targets and source in this module. Also, catches
and reformats the validation exception before raising
"""
sources = [Source(entry), gen.user_arguments_to_source(user_config)]
try:
return gen.validate_and_raise(sources, [get_target()])
except gen.exceptions.ValidationError as ex:
raise launch.util.LauncherError(
'ValidationError', pretty_print_validate_error(ex.errors, ex.unset))
def test_resolve_simple():
test_source = Source({
'validate': [validate_a],
'default': {
'a': 'a_str',
'd': 'd_1',
},
'must': {
'b': 'b_str'
},
'conditional': {
'd': {
'd_1': {
'must': {
'd_1_b': 'd_1_b_str'
}
},
'd_2': {
'must': {
'd_2_b': 'd_2_b_str'
}
}
}
}
})
def get_test_target():
return Target(
{'a', 'b', 'c'},
{'d': Scope(
'd', {
'd_1': Target({'d_1_a', 'd_1_b'}),
'd_2': Target({'d_2_a', 'd_2_b'})
})})
test_user_source = Source(is_user=True)
test_user_source.add_must('c', 'c_str')
test_user_source.add_must('d_1_a', 'd_1_a_str')
resolver = gen.internals.resolve_configuration([test_source, test_user_source], [get_test_target()])
print(resolver)
assert resolver.status_dict == {'status': 'ok'}
# Make sure having a unset variable results in a non-ok status
test_partial_source = Source(is_user=True)
test_partial_source.add_must('d_1_a', 'd_1_a_str')
resolver = gen.internals.resolve_configuration([test_source, test_partial_source], [get_test_target()])
print(resolver)
assert resolver.status_dict == {'status': 'errors', 'errors': {}, 'unset': {'c'}}
def test_resolve_late():
test_late_source = Source()
test_late_source.add_must('c', gen.internals.Late('c_str'))
test_late_source.add_must('d_1_a', 'd_1_a_str')
resolver = gen.internals.resolve_configuration([test_source, test_late_source], [get_test_target()])
assert resolver.status_dict == {'status': 'ok'}
assert resolver.late == {'c'}
def test_resolve_simple():
test_source = Source({
'validate': [validate_a],
'default': {
'a': 'a_str',
'd': 'd_1',
},
'must': {
'b': 'b_str'
},
'conditional': {
'd': {
'd_1': {
'must': {
'd_1_b': 'd_1_b_str'
}
},
'd_2': {
'must': {
'd_2_b': 'd_2_b_str'
}
}
}
}
})
test_target = Target({'a', 'b', 'c'}, {
'd':
Scope('d', {
'd_1': Target({'d_1_a', 'd_1_b'}),
'd_2': Target({'d_2_a', 'd_2_b'})
})
})
resolver = gen.internals.resolve_configuration([test_source],
[test_target], {
'c': 'c_str',
'd_1_a': 'd_1_a_str'
})
print(resolver)
assert resolver.status_dict == {'status': 'ok'}
aws_base_source = Source(
entry={
'validate': [validate_provider],
'default': {
'platform': 'aws',
'resolvers': '["169.254.169.253"]',
'num_private_slaves': '5',
'num_public_slaves': '1',
'os_type': '',
'aws_masters_have_public_ip': 'true',
'enable_docker_gc': 'true'
},
'must': {
'aws_region':
Late('{ "Ref" : "AWS::Region" }'),
'aws_stack_id':
Late('{ "Ref" : "AWS::StackId" }'),
'aws_stack_name':
Late('{ "Ref" : "AWS::StackName" }'),
'ip_detect_contents':
get_ip_detect('aws'),
'ip_detect_public_contents':
calculate_ip_detect_public_contents,
'exhibitor_explicit_keys':
'false',
'cluster_name':
Late('{ "Ref" : "AWS::StackName" }'),
'master_discovery':
'master_http_loadbalancer',
# DRY the cluster packages list in CF templates.
# This late expression isn't a Late because cluster-packages.json must go into cloud config, not the late
# package. The variable referenced here is stored behind two unnecessary keys because of CF template syntax
# requirements. See
# https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/mappings-section-structure.html.
# TODO(branden): Make this unnecessary by turning cluster-packages.json into a build artifact. See
# https://mesosphere.atlassian.net/browse/DCOS-13824.
'cluster_packages_json':
'{ "Fn::FindInMap" : [ "ClusterPackagesJson", "default", "default" ] }',
'cluster_packages_json_var':
lambda cluster_packages: json.dumps(cluster_packages),
# The cloud_config template variables pertaining to "cloudformation.json"
'master_cloud_config':
'{{ master_cloud_config }}',
'agent_private_cloud_config':
'{{ slave_cloud_config }}',
'agent_public_cloud_config':
'{{ slave_public_cloud_config }}',
# template variable for the generating advanced template cloud configs
'cloud_config':
'{{ cloud_config }}',
'rexray_config_preset':
'aws'
},
'conditional': {
'oauth_available': {
'true': {
'must': {
'oauth_enabled':
Late('{ "Ref" : "OAuthEnabled" }'),
'adminrouter_auth_enabled':
Late('{ "Ref" : "OAuthEnabled" }'),
}
},
'false': {},
}
}
})
azure_base_source = Source(
entry={
'validate': [validate_provider],
'default': {
'platform': 'azure',
'enable_docker_gc': 'true'
},
'must': {
'resolvers':
'["168.63.129.16"]',
'ip_detect_contents':
yaml.dump(
pkg_resources.resource_string('gen',
'ip-detect/azure.sh').decode()),
'ip6_detect_contents':
yaml.dump(
pkg_resources.resource_string('gen',
'ip-detect/azure6.sh').decode()),
'master_discovery':
'static',
'exhibitor_storage_backend':
'azure',
'master_cloud_config':
'{{ master_cloud_config }}',
'slave_cloud_config':
'{{ slave_cloud_config }}',
'slave_public_cloud_config':
'{{ slave_public_cloud_config }}',
'fault_domain_detect_contents':
yaml.dump(
pkg_resources.resource_string(
'gen', 'fault-domain-detect/cloud.sh').decode())
},
'conditional': {
'oauth_available': {
'true': {
'must': {
'oauth_enabled':
Late("[[[variables('oauthEnabled')]]]"),
'adminrouter_auth_enabled':
Late("[[[variables('oauthEnabled')]]]"),
}
},
'false': {},
},
'licensing_enabled': {
'true': {
'must': {
'license_key_contents':
Late("[[[variables('licenseKey')]]]"),
},
'secret': [
'license_key_contents',
],
},
'false': {},
}
}
})
aws_base_source = Source(
entry={
'default': {
'resolvers': '["169.254.169.253"]',
'num_private_slaves': '5',
'num_public_slaves': '1',
'os_type': '',
'aws_masters_have_public_ip': 'true',
'enable_docker_gc': 'true'
},
'must': {
'aws_region': '{ "Ref" : "AWS::Region" }',
'ip_detect_contents': get_ip_detect('aws'),
'ip_detect_public_contents': calculate_ip_detect_public_contents,
'exhibitor_explicit_keys': 'false',
'cluster_name': '{ "Ref" : "AWS::StackName" }',
'master_discovery': 'master_http_loadbalancer',
# The cloud_config template variables pertaining to "cloudformation.json"
'master_cloud_config': '{{ master_cloud_config }}',
'agent_private_cloud_config': '{{ slave_cloud_config }}',
'agent_public_cloud_config': '{{ slave_public_cloud_config }}',
'master_external_loadbalancer':
'{ "Fn::GetAtt" : [ "ElasticLoadBalancer", "DNSName" ] }',
# template variable for the generating advanced template cloud configs
'cloud_config': '{{ cloud_config }}',
'oauth_available': 'true',
'oauth_enabled': '{ "Ref" : "OAuthEnabled" }',
'rexray_config_preset': 'aws'
}
})
def validate_a(a):
assert a == 'a_str'
test_source = Source({
'validate': [validate_a],
'default': {
'a': 'a_str',
'd': 'd_1',
},
'must': {
'b': 'b_str'
},
'conditional': {
'd': {
'd_1': {
'must': {
'd_1_b': 'd_1_b_str'
}
},
'd_2': {
'must': {
'd_2_b': 'd_2_b_str'
}
}
}
}
})
def get_test_target():
return Target(
def gen_advanced_template(arguments, variant_prefix, reproducible_artifact_path, os_type):
for node_type in ['master', 'priv-agent', 'pub-agent']:
# TODO(cmaloney): This forcibly overwriting arguments might overwrite a user set argument
# without noticing (such as exhibitor_storage_backend)
node_template_id, node_source = groups[node_type]
local_source = Source()
local_source.add_must('os_type', os_type)
local_source.add_must('region_to_ami_mapping', gen_ami_mapping({"coreos", "el7"}))
params = deepcopy(cf_instance_groups[node_template_id])
params['report_name'] = aws_advanced_report_names[node_type]
params['os_type'] = os_type
params['node_type'] = node_type
template_key = 'advanced-{}'.format(node_type)
template_name = template_key + '.json'
def _as_artifact(filename, bundle):
yield from _as_artifact_and_pkg(variant_prefix, filename, bundle)
if node_type == 'master':
for num_masters in [1, 3, 5, 7]:
master_tk = '{}-{}-{}'.format(os_type, template_key, num_masters)
print('Building {} {} for num_masters = {}'.format(os_type, node_type, num_masters))
num_masters_source = Source()
num_masters_source.add_must('num_masters', str(num_masters))
bundle = make_advanced_bundle(arguments,
[node_source, local_source, num_masters_source],
template_name,
params)
yield from _as_artifact('{}.json'.format(master_tk), bundle)
# Zen template corresponding to this number of masters
yield _as_cf_artifact(
'{}{}-zen-{}.json'.format(variant_prefix, os_type, num_masters),
render_cloudformation_transform(
resource_string("gen", "aws/templates/advanced/zen.json").decode(),
variant_prefix=variant_prefix,
reproducible_artifact_path=reproducible_artifact_path,
**bundle[1].arguments))
else:
local_source.add_must('num_masters', '1')
local_source.add_must('nat_ami_mapping', gen_ami_mapping({"natami"}))
bundle = make_advanced_bundle(arguments,
[node_source, local_source],
template_name,
params)
yield from _as_artifact('{}-{}'.format(os_type, template_name), bundle)
aws_base_source = Source(
entry={
'validate': [validate_provider],
'default': {
'platform': 'aws',
'resolvers': '["169.254.169.253"]',
'num_private_slaves': '5',
'num_public_slaves': '1',
'os_type': '',
'aws_masters_have_public_ip': 'true',
'enable_docker_gc': 'true'
},
'must': {
'aws_region':
Late('{ "Ref" : "AWS::Region" }'),
'aws_stack_id':
Late('{ "Ref" : "AWS::StackId" }'),
'aws_stack_name':
Late('{ "Ref" : "AWS::StackName" }'),
'ip_detect_contents':
get_ip_detect('aws'),
'ip_detect_public_contents':
calculate_ip_detect_public_contents,
'ip6_detect_contents':
get_ip_detect('aws6'),
'exhibitor_explicit_keys':
'false',
'cluster_name':
Late('{ "Ref" : "AWS::StackName" }'),
'master_discovery':
'master_http_loadbalancer',
# The cloud_config template variables pertaining to "cloudformation.json"
'master_cloud_config':
'{{ master_cloud_config }}',
'agent_private_cloud_config':
'{{ slave_cloud_config }}',
'agent_public_cloud_config':
'{{ slave_public_cloud_config }}',
# template variable for the generating advanced template cloud configs
'cloud_config':
'{{ cloud_config }}',
'rexray_config_preset':
'aws',
'fault_domain_detect_contents':
yaml.dump(
pkg_resources.resource_string(
'gen', 'fault-domain-detect/cloud.sh').decode()),
},
'conditional': {
'oauth_available': {
'true': {
'must': {
'oauth_enabled':
Late('{ "Ref" : "OAuthEnabled" }'),
'adminrouter_auth_enabled':
Late('{ "Ref" : "OAuthEnabled" }'),
}
},
'false': {}
},
'licensing_enabled': {
'true': {
'must': {
'license_key_contents': Late(
'{ "Ref" : "LicenseKey" }'),
},
'secret': [
'license_key_contents',
],
},
'false': {},
}
}
})
}
}
azure_base_source = Source(
entry={
'default': {
'enable_docker_gc': 'true'
},
'must': {
'resolvers':
'["168.63.129.16"]',
'ip_detect_contents':
yaml.dump(
pkg_resources.resource_string('gen',
'ip-detect/azure.sh').decode()),
'master_discovery':
'static',
'exhibitor_storage_backend':
'azure',
'master_cloud_config':
'{{ master_cloud_config }}',
'slave_cloud_config':
'{{ slave_cloud_config }}',
'slave_public_cloud_config':
'{{ slave_public_cloud_config }}',
'oauth_enabled':
"[[[variables('oauthEnabled')]]]",
}
})
def validate_cloud_config(cc_string):
import gen.build_deploy.util as util
import gen.template
import pkgpanda.util
from gen.calc import calculate_environment_variable
from gen.internals import Source
from pkgpanda.build.src_fetchers import GitLocalSrcFetcher
from pkgpanda.util import logger
onprem_source = Source(
entry={
'default': {
'platform': 'onprem',
'resolvers': '["8.8.8.8", "8.8.4.4"]',
'ip_detect_filename': 'genconf/ip-detect',
'bootstrap_id':
lambda: calculate_environment_variable('BOOTSTRAP_ID'),
'enable_docker_gc': 'false',
},
'must': {
'provider': 'onprem'
}
})
file_template = """mkdir -p `dirname {filename}`
cat <<'EOF' > "{filename}"
{content}
EOF
chmod {mode} {filename}
"""
source = Source({
'validate': [
lambda agent_list: gen.calc.validate_ip_list(agent_list),
lambda public_agent_list: gen.calc.validate_ip_list(public_agent_list),
lambda master_list: gen.calc.validate_ip_list(master_list),
# master list shouldn't contain anything in either agent lists
lambda master_list, agent_list: compare_lists(master_list, agent_list),
lambda master_list, public_agent_list: compare_lists(
master_list, public_agent_list),
# the agent lists shouldn't contain any common items
lambda agent_list, public_agent_list: compare_lists(
agent_list, public_agent_list),
validate_ssh_key_path,
lambda ssh_port: gen.calc.validate_int_in_range(ssh_port, 1, 32000),
lambda ssh_parallelism: gen.calc.validate_int_in_range(
ssh_parallelism, 1, 100)
],
'default': {
'ssh_key_path': 'genconf/ssh_key',
'agent_list': '[]',
'public_agent_list': '[]',
'ssh_user': '******',
'ssh_port': '22',
'process_timeout': '120',
'ssh_parallelism': '20'
}
})
import gen.template
import pkgpanda
import pkgpanda.util
from gen.calc import calculate_environment_variable
from gen.internals import Source
from pkgpanda.util import logger
onprem_source = Source(entry={
'default': {
'platform': 'onprem',
'resolvers': '["8.8.8.8", "8.8.4.4"]',
'ip_detect_filename': 'genconf/ip-detect',
'bootstrap_id': lambda: calculate_environment_variable('BOOTSTRAP_ID'),
'enable_docker_gc': 'false',
},
'must': {
'provider': 'onprem',
'package_ids': lambda bootstrap_variant: json.dumps(
dcos_installer.config_util.installer_latest_complete_artifact(bootstrap_variant)['packages']
),
}
})
file_template = """mkdir -p `dirname {filename}`
cat <<'EOF' > "{filename}"
{content}
EOF
chmod {mode} {filename}
def gen_advanced_template(arguments, variant_prefix,
reproducible_artifact_path, os_type):
for node_type in ['master', 'priv-agent', 'pub-agent']:
# TODO(cmaloney): This forcibly overwriting arguments might overwrite a user set argument
# without noticing (such as exhibitor_storage_backend)
node_template_id, node_source = groups[node_type]
local_source = Source()
local_source.add_must('os_type', os_type)
local_source.add_must('region_to_ami_mapping',
gen_ami_mapping({"coreos", "el7", "el7prereq"}))
params = cf_instance_groups[node_template_id]
params['report_name'] = aws_advanced_report_names[node_type]
params['os_type'] = os_type
params['node_type'] = node_type
template_key = 'advanced-{}'.format(node_type)
template_name = template_key + '.json'
def _as_artifact(filename, bundle):
yield from _as_artifact_and_pkg(variant_prefix, filename, bundle)
if node_type == 'master':
for num_masters in [1, 3, 5, 7]:
master_tk = '{}-{}-{}'.format(os_type, template_key,
num_masters)
print('Building {} {} for num_masters = {}'.format(
os_type, node_type, num_masters))
num_masters_source = Source()
num_masters_source.add_must('num_masters', str(num_masters))
bundle = make_advanced_bundle(
arguments, [node_source, local_source, num_masters_source],
template_name, deepcopy(params))
yield from _as_artifact('{}.json'.format(master_tk), bundle)
# Zen template corresponding to this number of masters
yield _as_cf_artifact(
'{}{}-zen-{}.json'.format(variant_prefix, os_type,
num_masters),
render_cloudformation_transform(
resource_string(
"gen", "aws/templates/advanced/zen.json").decode(),
variant_prefix=variant_prefix,
reproducible_artifact_path=reproducible_artifact_path,
**bundle[1].arguments))
else:
local_source.add_must('num_masters', '1')
local_source.add_must('nat_ami_mapping',
gen_ami_mapping({"natami"}))
bundle = make_advanced_bundle(arguments,
[node_source, local_source],
template_name, deepcopy(params))
yield from _as_artifact('{}-{}'.format(os_type, template_name),
bundle)
onprem_source = Source(
entry={
'validate': [
validate_custom_check_bins_dir,
lambda custom_check_bins_provided: validate_true_false(
custom_check_bins_provided),
],
'default': {
'platform': 'onprem',
'resolvers': '["8.8.8.8", "8.8.4.4"]',
'ip_detect_filename': 'genconf/ip-detect',
'ip6_detect_filename': '',
'bootstrap_id':
lambda: calculate_environment_variable('BOOTSTRAP_ID'),
'enable_docker_gc': 'false'
},
'must': {
'provider': 'onprem',
'package_ids': calculate_package_ids,
'custom_check_bins_dir': 'genconf/check_bins/',
'custom_check_bins_package_name': 'custom-check-bins',
'custom_check_bins_provided': calculate_custom_check_bins_provided,
'custom_check_bins_hash': calculate_custom_check_bins_hash,
'custom_check_bins_package_id':
calculate_custom_check_bins_package_id,
'check_search_path': calculate_check_search_path,
},
})
file_template = """mkdir -p `dirname {filename}`
