def make_template(num_masters, gen_arguments, varietal, bootstrap_variant_prefix): ''' Return a tuple: the generated template for num_masters and the artifact dict. @param num_masters: int, number of master nodes to embed in the generated template @param gen_arguments: dict, args to pass to the gen library. These are user input arguments which get filled in/prompted for. @param varietal: string, indicate template varietal to build for either 'acs' or 'dcos' ''' master_list_source = Source() master_list_source.add_must('master_list', Late(master_list_arm_json(num_masters, varietal))) master_list_source.add_must('num_masters', str(num_masters)) if varietal == 'dcos': arm, results = gen_templates( gen_arguments, 'azuredeploy', extra_sources=[master_list_source, azure_dcos_source]) elif varietal == 'acs': arm, results = gen_templates( gen_arguments, 'acs', extra_sources=[master_list_source, azure_acs_source]) else: raise ValueError("Unknown Azure varietal specified") yield {'packages': results.config_package_ids} if results.late_package_id: yield {'packages': [results.late_package_id]} yield { 'channel_path': 'azure/{}{}-{}master.azuredeploy.json'.format(bootstrap_variant_prefix, varietal, num_masters), 'local_content': arm, 'content_type': 'application/json; charset=utf-8' }
def make(num_masters, filename): num_masters_source = Source() num_masters_source.add_must('num_masters', str(num_masters)) yield from gen_simple_template( variant_prefix, filename, variant_base_args, num_masters_source)
def test_resolve_simple(): test_user_source = Source(is_user=True) test_user_source.add_must('c', 'c_str') test_user_source.add_must('d_1_a', 'd_1_a_str') resolver = gen.internals.resolve_configuration([test_source, test_user_source], [get_test_target()]) print(resolver) assert resolver.status_dict == {'status': 'ok'} # Make sure having a unset variable results in a non-ok status test_partial_source = Source(is_user=True) test_partial_source.add_must('d_1_a', 'd_1_a_str') resolver = gen.internals.resolve_configuration([test_source, test_partial_source], [get_test_target()]) print(resolver) assert resolver.status_dict == {'status': 'errors', 'errors': {}, 'unset': {'c'}}
def test_resolve_simple(): test_source = Source({ 'validate': [validate_a], 'default': { 'a': 'a_str', 'd': 'd_1', }, 'must': { 'b': 'b_str' }, 'conditional': { 'd': { 'd_1': { 'must': { 'd_1_b': 'd_1_b_str' } }, 'd_2': { 'must': { 'd_2_b': 'd_2_b_str' } } } } }) def get_test_target(): return Target({'a', 'b', 'c'}, { 'd': Scope( 'd', { 'd_1': Target({'d_1_a', 'd_1_b'}), 'd_2': Target({'d_2_a', 'd_2_b'}) }) }) resolver = gen.internals.resolve_configuration([test_source], [get_test_target()], { 'c': 'c_str', 'd_1_a': 'd_1_a_str' }) print(resolver) assert resolver.status_dict == {'status': 'ok'} # Make sure having a unset variable results in a non-ok status resolver = gen.internals.resolve_configuration([test_source], [get_test_target()], {'d_1_a': 'd_1_a_str'}) print(resolver) assert resolver.status_dict == { 'status': 'errors', 'errors': {}, 'unset': {'c'} }
def make_template(num_masters, gen_arguments, varietal, bootstrap_variant_prefix): ''' Return a tuple: the generated template for num_masters and the artifact dict. @param num_masters: int, number of master nodes to embed in the generated template @param gen_arguments: dict, args to pass to the gen library. These are user input arguments which get filled in/prompted for. @param varietal: string, indicate template varietal to build for either 'acs' or 'dcos' ''' master_list_source = Source() master_list_source.add_must('master_list', master_list_arm_json(num_masters, varietal)) if varietal == 'dcos': dcos_template = gen_templates( gen_arguments, 'azuredeploy', extra_sources=[master_list_source, azure_dcos_source]) elif varietal == 'acs': dcos_template = gen_templates( gen_arguments, 'acs', extra_sources=[master_list_source, azure_acs_source]) else: raise ValueError("Unknown Azure varietal specified") yield { 'packages': util.cluster_to_extra_packages( dcos_template['results'].cluster_packages) } yield { 'channel_path': 'azure/{}{}-{}master.azuredeploy.json'.format(bootstrap_variant_prefix, varietal, num_masters), 'local_content': dcos_template['arm'], 'content_type': 'application/json; charset=utf-8' }
def validate_config(user_config: dict) -> gen.internals.Resolver: """ Converts the user config to to a source and evaluates it versus the targets and source in this module. Also, catches and reformats the validation exception before raising """ sources = [Source(entry), gen.user_arguments_to_source(user_config)] try: return gen.validate_and_raise(sources, [get_target()]) except gen.exceptions.ValidationError as ex: raise launch.util.LauncherError( 'ValidationError', pretty_print_validate_error(ex.errors, ex.unset))
def test_resolve_simple(): test_source = Source({ 'validate': [validate_a], 'default': { 'a': 'a_str', 'd': 'd_1', }, 'must': { 'b': 'b_str' }, 'conditional': { 'd': { 'd_1': { 'must': { 'd_1_b': 'd_1_b_str' } }, 'd_2': { 'must': { 'd_2_b': 'd_2_b_str' } } } } }) def get_test_target(): return Target( {'a', 'b', 'c'}, {'d': Scope( 'd', { 'd_1': Target({'d_1_a', 'd_1_b'}), 'd_2': Target({'d_2_a', 'd_2_b'}) })}) test_user_source = Source(is_user=True) test_user_source.add_must('c', 'c_str') test_user_source.add_must('d_1_a', 'd_1_a_str') resolver = gen.internals.resolve_configuration([test_source, test_user_source], [get_test_target()]) print(resolver) assert resolver.status_dict == {'status': 'ok'} # Make sure having a unset variable results in a non-ok status test_partial_source = Source(is_user=True) test_partial_source.add_must('d_1_a', 'd_1_a_str') resolver = gen.internals.resolve_configuration([test_source, test_partial_source], [get_test_target()]) print(resolver) assert resolver.status_dict == {'status': 'errors', 'errors': {}, 'unset': {'c'}}
def test_resolve_late(): test_late_source = Source() test_late_source.add_must('c', gen.internals.Late('c_str')) test_late_source.add_must('d_1_a', 'd_1_a_str') resolver = gen.internals.resolve_configuration([test_source, test_late_source], [get_test_target()]) assert resolver.status_dict == {'status': 'ok'} assert resolver.late == {'c'}
def test_resolve_simple(): test_source = Source({ 'validate': [validate_a], 'default': { 'a': 'a_str', 'd': 'd_1', }, 'must': { 'b': 'b_str' }, 'conditional': { 'd': { 'd_1': { 'must': { 'd_1_b': 'd_1_b_str' } }, 'd_2': { 'must': { 'd_2_b': 'd_2_b_str' } } } } }) test_target = Target({'a', 'b', 'c'}, { 'd': Scope('d', { 'd_1': Target({'d_1_a', 'd_1_b'}), 'd_2': Target({'d_2_a', 'd_2_b'}) }) }) resolver = gen.internals.resolve_configuration([test_source], [test_target], { 'c': 'c_str', 'd_1_a': 'd_1_a_str' }) print(resolver) assert resolver.status_dict == {'status': 'ok'}
aws_base_source = Source( entry={ 'validate': [validate_provider], 'default': { 'platform': 'aws', 'resolvers': '["169.254.169.253"]', 'num_private_slaves': '5', 'num_public_slaves': '1', 'os_type': '', 'aws_masters_have_public_ip': 'true', 'enable_docker_gc': 'true' }, 'must': { 'aws_region': Late('{ "Ref" : "AWS::Region" }'), 'aws_stack_id': Late('{ "Ref" : "AWS::StackId" }'), 'aws_stack_name': Late('{ "Ref" : "AWS::StackName" }'), 'ip_detect_contents': get_ip_detect('aws'), 'ip_detect_public_contents': calculate_ip_detect_public_contents, 'exhibitor_explicit_keys': 'false', 'cluster_name': Late('{ "Ref" : "AWS::StackName" }'), 'master_discovery': 'master_http_loadbalancer', # DRY the cluster packages list in CF templates. # This late expression isn't a Late because cluster-packages.json must go into cloud config, not the late # package. The variable referenced here is stored behind two unnecessary keys because of CF template syntax # requirements. See # https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/mappings-section-structure.html. # TODO(branden): Make this unnecessary by turning cluster-packages.json into a build artifact. See # https://mesosphere.atlassian.net/browse/DCOS-13824. 'cluster_packages_json': '{ "Fn::FindInMap" : [ "ClusterPackagesJson", "default", "default" ] }', 'cluster_packages_json_var': lambda cluster_packages: json.dumps(cluster_packages), # The cloud_config template variables pertaining to "cloudformation.json" 'master_cloud_config': '{{ master_cloud_config }}', 'agent_private_cloud_config': '{{ slave_cloud_config }}', 'agent_public_cloud_config': '{{ slave_public_cloud_config }}', # template variable for the generating advanced template cloud configs 'cloud_config': '{{ cloud_config }}', 'rexray_config_preset': 'aws' }, 'conditional': { 'oauth_available': { 'true': { 'must': { 'oauth_enabled': Late('{ "Ref" : "OAuthEnabled" }'), 'adminrouter_auth_enabled': Late('{ "Ref" : "OAuthEnabled" }'), } }, 'false': {}, } } })
azure_base_source = Source( entry={ 'validate': [validate_provider], 'default': { 'platform': 'azure', 'enable_docker_gc': 'true' }, 'must': { 'resolvers': '["168.63.129.16"]', 'ip_detect_contents': yaml.dump( pkg_resources.resource_string('gen', 'ip-detect/azure.sh').decode()), 'ip6_detect_contents': yaml.dump( pkg_resources.resource_string('gen', 'ip-detect/azure6.sh').decode()), 'master_discovery': 'static', 'exhibitor_storage_backend': 'azure', 'master_cloud_config': '{{ master_cloud_config }}', 'slave_cloud_config': '{{ slave_cloud_config }}', 'slave_public_cloud_config': '{{ slave_public_cloud_config }}', 'fault_domain_detect_contents': yaml.dump( pkg_resources.resource_string( 'gen', 'fault-domain-detect/cloud.sh').decode()) }, 'conditional': { 'oauth_available': { 'true': { 'must': { 'oauth_enabled': Late("[[[variables('oauthEnabled')]]]"), 'adminrouter_auth_enabled': Late("[[[variables('oauthEnabled')]]]"), } }, 'false': {}, }, 'licensing_enabled': { 'true': { 'must': { 'license_key_contents': Late("[[[variables('licenseKey')]]]"), }, 'secret': [ 'license_key_contents', ], }, 'false': {}, } } })
aws_base_source = Source( entry={ 'default': { 'resolvers': '["169.254.169.253"]', 'num_private_slaves': '5', 'num_public_slaves': '1', 'os_type': '', 'aws_masters_have_public_ip': 'true', 'enable_docker_gc': 'true' }, 'must': { 'aws_region': '{ "Ref" : "AWS::Region" }', 'ip_detect_contents': get_ip_detect('aws'), 'ip_detect_public_contents': calculate_ip_detect_public_contents, 'exhibitor_explicit_keys': 'false', 'cluster_name': '{ "Ref" : "AWS::StackName" }', 'master_discovery': 'master_http_loadbalancer', # The cloud_config template variables pertaining to "cloudformation.json" 'master_cloud_config': '{{ master_cloud_config }}', 'agent_private_cloud_config': '{{ slave_cloud_config }}', 'agent_public_cloud_config': '{{ slave_public_cloud_config }}', 'master_external_loadbalancer': '{ "Fn::GetAtt" : [ "ElasticLoadBalancer", "DNSName" ] }', # template variable for the generating advanced template cloud configs 'cloud_config': '{{ cloud_config }}', 'oauth_available': 'true', 'oauth_enabled': '{ "Ref" : "OAuthEnabled" }', 'rexray_config_preset': 'aws' } })
def validate_a(a): assert a == 'a_str' test_source = Source({ 'validate': [validate_a], 'default': { 'a': 'a_str', 'd': 'd_1', }, 'must': { 'b': 'b_str' }, 'conditional': { 'd': { 'd_1': { 'must': { 'd_1_b': 'd_1_b_str' } }, 'd_2': { 'must': { 'd_2_b': 'd_2_b_str' } } } } }) def get_test_target(): return Target(
def gen_advanced_template(arguments, variant_prefix, reproducible_artifact_path, os_type): for node_type in ['master', 'priv-agent', 'pub-agent']: # TODO(cmaloney): This forcibly overwriting arguments might overwrite a user set argument # without noticing (such as exhibitor_storage_backend) node_template_id, node_source = groups[node_type] local_source = Source() local_source.add_must('os_type', os_type) local_source.add_must('region_to_ami_mapping', gen_ami_mapping({"coreos", "el7"})) params = deepcopy(cf_instance_groups[node_template_id]) params['report_name'] = aws_advanced_report_names[node_type] params['os_type'] = os_type params['node_type'] = node_type template_key = 'advanced-{}'.format(node_type) template_name = template_key + '.json' def _as_artifact(filename, bundle): yield from _as_artifact_and_pkg(variant_prefix, filename, bundle) if node_type == 'master': for num_masters in [1, 3, 5, 7]: master_tk = '{}-{}-{}'.format(os_type, template_key, num_masters) print('Building {} {} for num_masters = {}'.format(os_type, node_type, num_masters)) num_masters_source = Source() num_masters_source.add_must('num_masters', str(num_masters)) bundle = make_advanced_bundle(arguments, [node_source, local_source, num_masters_source], template_name, params) yield from _as_artifact('{}.json'.format(master_tk), bundle) # Zen template corresponding to this number of masters yield _as_cf_artifact( '{}{}-zen-{}.json'.format(variant_prefix, os_type, num_masters), render_cloudformation_transform( resource_string("gen", "aws/templates/advanced/zen.json").decode(), variant_prefix=variant_prefix, reproducible_artifact_path=reproducible_artifact_path, **bundle[1].arguments)) else: local_source.add_must('num_masters', '1') local_source.add_must('nat_ami_mapping', gen_ami_mapping({"natami"})) bundle = make_advanced_bundle(arguments, [node_source, local_source], template_name, params) yield from _as_artifact('{}-{}'.format(os_type, template_name), bundle)
aws_base_source = Source( entry={ 'validate': [validate_provider], 'default': { 'platform': 'aws', 'resolvers': '["169.254.169.253"]', 'num_private_slaves': '5', 'num_public_slaves': '1', 'os_type': '', 'aws_masters_have_public_ip': 'true', 'enable_docker_gc': 'true' }, 'must': { 'aws_region': Late('{ "Ref" : "AWS::Region" }'), 'aws_stack_id': Late('{ "Ref" : "AWS::StackId" }'), 'aws_stack_name': Late('{ "Ref" : "AWS::StackName" }'), 'ip_detect_contents': get_ip_detect('aws'), 'ip_detect_public_contents': calculate_ip_detect_public_contents, 'ip6_detect_contents': get_ip_detect('aws6'), 'exhibitor_explicit_keys': 'false', 'cluster_name': Late('{ "Ref" : "AWS::StackName" }'), 'master_discovery': 'master_http_loadbalancer', # The cloud_config template variables pertaining to "cloudformation.json" 'master_cloud_config': '{{ master_cloud_config }}', 'agent_private_cloud_config': '{{ slave_cloud_config }}', 'agent_public_cloud_config': '{{ slave_public_cloud_config }}', # template variable for the generating advanced template cloud configs 'cloud_config': '{{ cloud_config }}', 'rexray_config_preset': 'aws', 'fault_domain_detect_contents': yaml.dump( pkg_resources.resource_string( 'gen', 'fault-domain-detect/cloud.sh').decode()), }, 'conditional': { 'oauth_available': { 'true': { 'must': { 'oauth_enabled': Late('{ "Ref" : "OAuthEnabled" }'), 'adminrouter_auth_enabled': Late('{ "Ref" : "OAuthEnabled" }'), } }, 'false': {} }, 'licensing_enabled': { 'true': { 'must': { 'license_key_contents': Late( '{ "Ref" : "LicenseKey" }'), }, 'secret': [ 'license_key_contents', ], }, 'false': {}, } } })
} } azure_base_source = Source( entry={ 'default': { 'enable_docker_gc': 'true' }, 'must': { 'resolvers': '["168.63.129.16"]', 'ip_detect_contents': yaml.dump( pkg_resources.resource_string('gen', 'ip-detect/azure.sh').decode()), 'master_discovery': 'static', 'exhibitor_storage_backend': 'azure', 'master_cloud_config': '{{ master_cloud_config }}', 'slave_cloud_config': '{{ slave_cloud_config }}', 'slave_public_cloud_config': '{{ slave_public_cloud_config }}', 'oauth_enabled': "[[[variables('oauthEnabled')]]]", } }) def validate_cloud_config(cc_string):
import gen.build_deploy.util as util import gen.template import pkgpanda.util from gen.calc import calculate_environment_variable from gen.internals import Source from pkgpanda.build.src_fetchers import GitLocalSrcFetcher from pkgpanda.util import logger onprem_source = Source( entry={ 'default': { 'platform': 'onprem', 'resolvers': '["8.8.8.8", "8.8.4.4"]', 'ip_detect_filename': 'genconf/ip-detect', 'bootstrap_id': lambda: calculate_environment_variable('BOOTSTRAP_ID'), 'enable_docker_gc': 'false', }, 'must': { 'provider': 'onprem' } }) file_template = """mkdir -p `dirname {filename}` cat <<'EOF' > "{filename}" {content} EOF chmod {mode} {filename} """
source = Source({ 'validate': [ lambda agent_list: gen.calc.validate_ip_list(agent_list), lambda public_agent_list: gen.calc.validate_ip_list(public_agent_list), lambda master_list: gen.calc.validate_ip_list(master_list), # master list shouldn't contain anything in either agent lists lambda master_list, agent_list: compare_lists(master_list, agent_list), lambda master_list, public_agent_list: compare_lists( master_list, public_agent_list), # the agent lists shouldn't contain any common items lambda agent_list, public_agent_list: compare_lists( agent_list, public_agent_list), validate_ssh_key_path, lambda ssh_port: gen.calc.validate_int_in_range(ssh_port, 1, 32000), lambda ssh_parallelism: gen.calc.validate_int_in_range( ssh_parallelism, 1, 100) ], 'default': { 'ssh_key_path': 'genconf/ssh_key', 'agent_list': '[]', 'public_agent_list': '[]', 'ssh_user': '******', 'ssh_port': '22', 'process_timeout': '120', 'ssh_parallelism': '20' } })
import gen.template import pkgpanda import pkgpanda.util from gen.calc import calculate_environment_variable from gen.internals import Source from pkgpanda.util import logger onprem_source = Source(entry={ 'default': { 'platform': 'onprem', 'resolvers': '["8.8.8.8", "8.8.4.4"]', 'ip_detect_filename': 'genconf/ip-detect', 'bootstrap_id': lambda: calculate_environment_variable('BOOTSTRAP_ID'), 'enable_docker_gc': 'false', }, 'must': { 'provider': 'onprem', 'package_ids': lambda bootstrap_variant: json.dumps( dcos_installer.config_util.installer_latest_complete_artifact(bootstrap_variant)['packages'] ), } }) file_template = """mkdir -p `dirname {filename}` cat <<'EOF' > "{filename}" {content} EOF chmod {mode} {filename}
def gen_advanced_template(arguments, variant_prefix, reproducible_artifact_path, os_type): for node_type in ['master', 'priv-agent', 'pub-agent']: # TODO(cmaloney): This forcibly overwriting arguments might overwrite a user set argument # without noticing (such as exhibitor_storage_backend) node_template_id, node_source = groups[node_type] local_source = Source() local_source.add_must('os_type', os_type) local_source.add_must('region_to_ami_mapping', gen_ami_mapping({"coreos", "el7", "el7prereq"})) params = cf_instance_groups[node_template_id] params['report_name'] = aws_advanced_report_names[node_type] params['os_type'] = os_type params['node_type'] = node_type template_key = 'advanced-{}'.format(node_type) template_name = template_key + '.json' def _as_artifact(filename, bundle): yield from _as_artifact_and_pkg(variant_prefix, filename, bundle) if node_type == 'master': for num_masters in [1, 3, 5, 7]: master_tk = '{}-{}-{}'.format(os_type, template_key, num_masters) print('Building {} {} for num_masters = {}'.format( os_type, node_type, num_masters)) num_masters_source = Source() num_masters_source.add_must('num_masters', str(num_masters)) bundle = make_advanced_bundle( arguments, [node_source, local_source, num_masters_source], template_name, deepcopy(params)) yield from _as_artifact('{}.json'.format(master_tk), bundle) # Zen template corresponding to this number of masters yield _as_cf_artifact( '{}{}-zen-{}.json'.format(variant_prefix, os_type, num_masters), render_cloudformation_transform( resource_string( "gen", "aws/templates/advanced/zen.json").decode(), variant_prefix=variant_prefix, reproducible_artifact_path=reproducible_artifact_path, **bundle[1].arguments)) else: local_source.add_must('num_masters', '1') local_source.add_must('nat_ami_mapping', gen_ami_mapping({"natami"})) bundle = make_advanced_bundle(arguments, [node_source, local_source], template_name, deepcopy(params)) yield from _as_artifact('{}-{}'.format(os_type, template_name), bundle)
onprem_source = Source( entry={ 'validate': [ validate_custom_check_bins_dir, lambda custom_check_bins_provided: validate_true_false( custom_check_bins_provided), ], 'default': { 'platform': 'onprem', 'resolvers': '["8.8.8.8", "8.8.4.4"]', 'ip_detect_filename': 'genconf/ip-detect', 'ip6_detect_filename': '', 'bootstrap_id': lambda: calculate_environment_variable('BOOTSTRAP_ID'), 'enable_docker_gc': 'false' }, 'must': { 'provider': 'onprem', 'package_ids': calculate_package_ids, 'custom_check_bins_dir': 'genconf/check_bins/', 'custom_check_bins_package_name': 'custom-check-bins', 'custom_check_bins_provided': calculate_custom_check_bins_provided, 'custom_check_bins_hash': calculate_custom_check_bins_hash, 'custom_check_bins_package_id': calculate_custom_check_bins_package_id, 'check_search_path': calculate_check_search_path, }, }) file_template = """mkdir -p `dirname {filename}`