def simu(options):
if (options.generate):
try:
for _conf in ('Basic10', 'Basic100', 'Basic1000'):
conf = getattr(cliquid, _conf)
gen.generate(conf)
except ImportError, e:
print "this configuration is not defined"
return
def test_exhibitor_tls_initialize_prints_errors(capsys):
gen.generate(arguments=make_arguments({
'platform': 'onprem',
'exhibitor_tls_enabled': 'true',
}))
expected_message = (
'[gen.exhibitor_tls_bootstrap] not bootstrapping '
'exhibitor CA: Exhibitor security is an enterprise feature')
assert expected_message in capsys.readouterr().out
def simu(options):
if options.generate:
try:
for _conf in ("Basic10", "Basic100", "Basic1000"):
conf = getattr(cliquid, _conf)
gen.generate(conf)
except ImportError, e:
print "this configuration is not defined"
return
def test_master(self, adminrouter_tls_1_0_enabled, tls_versions):
"""
Test that Master Admin Router config file has the correct content.
"""
config_path = '/etc/adminrouter-tls-master.conf'
arguments = make_arguments({
'adminrouter_tls_1_0_enabled':
adminrouter_tls_1_0_enabled,
})
generated = gen.generate(arguments=arguments)
package = generated.templates['dcos-config.yaml']['package']
[config] = [item for item in package if item['path'] == config_path]
expected_configuration = dedent("""\
# Ref: https://github.com/cloudflare/sslconfig/blob/master/conf
# Modulo ChaCha20 cipher.
ssl_ciphers EECDH+AES128:RSA+AES128:EECDH+AES256:RSA+AES256:EECDH+3DES:RSA+3DES:!MD5;
ssl_prefer_server_ciphers on;
# To manually test which TLS versions are enabled on a node, use
# `openssl` commands.
#
# See comments on https://jira.mesosphere.com/browse/DCOS-13437 for more
# details.
ssl_protocols {tls_versions};
""".format(tls_versions=tls_versions))
assert config['content'] == expected_configuration
def supported_tls_protocols_ar_master(
self, new_config_arguments: Dict[str, str]) -> List[str]:
"""
This finds a line which looks like the following:
ssl_protocols TLSv1, TLSv1.1;
in the Admin Router TLS configuration.
It then returns the listed protocols.
Args:
new_config_arguments: Arguments which are added to the 'standard'
set of arguments before generating configuration files.
Returns:
A list of supported TLS protocols.
"""
arguments = make_arguments(new_arguments=new_config_arguments)
generated = gen.generate(arguments=arguments)
package = generated.templates['dcos-config.yaml']['package']
config_path = '/etc/adminrouter-tls-master.conf'
[config] = [item for item in package if item['path'] == config_path]
[ssl_protocols_line] = [
line for line in config['content'].split('\n') if
# We strip whitespace from the beginning of the line as NGINX
# configuration lines can start with whitespace.
line.lstrip().startswith('ssl_protocols ')
]
ssl_protocols_line = ssl_protocols_line.strip(';')
protocols = ssl_protocols_line.split()[1:]
return protocols
def test_master(self, adminrouter_tls_1_0_enabled, tls_versions):
"""
Test that Master Admin Router config file has the correct content.
"""
config_path = '/etc/adminrouter-tls-master.conf'
arguments = make_arguments({
'adminrouter_tls_1_0_enabled': adminrouter_tls_1_0_enabled,
})
generated = gen.generate(arguments=arguments)
package = generated.templates['dcos-config.yaml']['package']
[config] = [item for item in package if item['path'] == config_path]
expected_configuration = dedent(
"""\
# Ref: https://github.com/cloudflare/sslconfig/blob/master/conf
# Modulo ChaCha20 cipher.
ssl_ciphers EECDH+AES128:RSA+AES128:EECDH+AES256:RSA+AES256:EECDH+3DES:RSA+3DES:!MD5;
ssl_prefer_server_ciphers on;
# To manually test which TLS versions are enabled on a node, use
# `openssl` commands.
#
# See comments on https://jira.mesosphere.com/browse/DCOS-13437 for more
# details.
ssl_protocols {tls_versions};
""".format(tls_versions=tls_versions)
)
assert config['content'] == expected_configuration
def supported_ssl_protocols(self, new_config_arguments) -> List[str]:
"""
This finds a line which looks like the following:
ssl protocols TLSv1, TLSv1.1;
in the Admin Router TLS configuration.
It then returns the listed protocols.
Args:
new_config_arguments: Arguments which are added to the 'standard'
set of arguments before generating configuration files.
Returns:
A ``list`` of supported SSL protocols.
"""
arguments = make_arguments(new_arguments=new_config_arguments)
generated = gen.generate(arguments=arguments)
package = generated.templates['dcos-config.yaml']['package']
config_path = '/etc/adminrouter-tls.conf'
[config] = [item for item in package if item['path'] == config_path]
[ssl_protocols_line] = [
line for line in config['content'].split('\n') if
# We strip whitespace from the beginning of the line as NGINX
# configuration lines can start with whitespace.
line.lstrip().startswith('ssl_protocols ')
]
ssl_protocols_line = ssl_protocols_line.strip(';')
protocols = ssl_protocols_line.split()[1:]
return protocols
def api_getChart():
submitted_name = request.args.get('recipe')
multiplier = float(request.args.get('multiplier'))
recipe_data = get_recipe_data(submitted_name, recipes)
recipe_data = adjust_recipe_qty(recipe_data, multiplier)
chart = generate(recipe_data)
return chart
def supported_ssl_ciphers(
self,
new_config_arguments: Dict[str, str],
config_path: str) -> List[str]:
"""
Finds the line that looks like:
ssl_ciphers EECDH+AES256:RSA+AES256:EECDH+AES128:RSA+AES128:EECDH+3DES:RSA+3DES:!MD5;
and returns the list of ciphers.
Args:
new_config_arguments: Arguments which are added to the 'standard'
set of arguments before generating configuration files.
config_path: A path to configuration file which should be examined
for ssl_ciphers configuration.
"""
arguments = make_arguments(new_arguments=new_config_arguments)
generated = gen.generate(arguments=arguments)
package = generated.templates['dcos-config.yaml']['package']
[config] = [item for item in package if item['path'] == config_path]
[ssl_ciphers_line] = [
line for line in config['content'].split('\n') if
# We strip whitespace from the beginning of the line as NGINX
# configuration lines can start with whitespace.
line.lstrip().startswith('ssl_ciphers ')
]
ssl_ciphers_line = ssl_ciphers_line.strip(';')
ciphers = ssl_ciphers_line.split()[1:]
return ciphers
def test_master_default(self):
"""
Test that Master Admin Router config file has the correct default
`ssl_ciphers` and `ssl_protocols` values. Defaults are present in
`dcos-config.yaml` file and in `calc.py`.
"""
config_path = '/etc/adminrouter-tls-master.conf'
arguments = make_arguments(new_arguments={})
generated = gen.generate(arguments=arguments)
package = generated.templates['dcos-config.yaml']['package']
[config] = [item for item in package if item['path'] == config_path]
expected_configuration = dedent(
"""\
# Ref: https://github.com/cloudflare/sslconfig/blob/master/conf
# Modulo ChaCha20 cipher and 3DES bulk encryption algorithm.
# For 3DES see https://jira.mesosphere.com/browse/DCOS-21958
ssl_ciphers EECDH+AES128:RSA+AES128:EECDH+AES256:RSA+AES256:!MD5:!3DES;
ssl_prefer_server_ciphers on;
# To manually test which TLS versions are enabled on a node, use
# `openssl` commands.
#
# See comments on https://jira.mesosphere.com/browse/DCOS-13437 for more
# details.
ssl_protocols TLSv1.2;
"""
)
assert config['content'] == expected_configuration
def supported_ssl_ciphers(self, new_config_arguments: Dict[str, str],
config_path: str) -> List[str]:
"""
Finds the line that looks like:
ssl_ciphers EECDH+AES256:RSA+AES256:EECDH+AES128:RSA+AES128:!MD5:!3DES;
and returns the list of ciphers.
Args:
new_config_arguments: Arguments which are added to the 'standard'
set of arguments before generating configuration files.
config_path: A path to configuration file which should be examined
for ssl_ciphers configuration.
"""
arguments = make_arguments(new_arguments=new_config_arguments)
generated = gen.generate(arguments=arguments)
package = generated.templates['dcos-config.yaml']['package']
[config] = [item for item in package if item['path'] == config_path]
[ssl_ciphers_line] = [
line for line in config['content'].split('\n') if
# We strip whitespace from the beginning of the line as NGINX
# configuration lines can start with whitespace.
line.lstrip().startswith('ssl_ciphers ')
]
ssl_ciphers_line = ssl_ciphers_line.strip(';')
ciphers = ssl_ciphers_line.split()[1:]
return ciphers
def test_default(self):
"""
By default, the configuration specifies certain TLS settings.
This test is a sanity check for the configuration template logic
rather than a particularly useful feature test.
"""
config_path = '/etc/adminrouter-tls.conf'
arguments = make_arguments(new_arguments={})
generated = gen.generate(arguments=arguments)
package = generated.templates['dcos-config.yaml']['package']
[config] = [item for item in package if item['path'] == config_path]
expected_configuration = dedent(
"""\
# Ref: https://github.com/cloudflare/sslconfig/blob/master/conf
# Modulo ChaCha20 cipher.
ssl_ciphers EECDH+AES128:RSA+AES128:EECDH+AES256:RSA+AES256:EECDH+3DES:RSA+3DES:!MD5;
ssl_prefer_server_ciphers on;
# To manually test which TLS versions are enabled on a node, use
# `openssl` commands.
#
# See comments on https://jira.mesosphere.com/browse/DCOS-13437 for more
# details.
ssl_protocols TLSv1.1 TLSv1.2;
"""
)
assert config['content'] == expected_configuration
def test_master_default(self):
"""
Test that Master Admin Router config file has the correct default
`ssl_ciphers` and `ssl_protocols` values. Defaults are present in
`dcos-config.yaml` file and in `calc.py`.
"""
config_path = '/etc/adminrouter-tls-master.conf'
arguments = make_arguments(new_arguments={})
generated = gen.generate(arguments=arguments)
package = generated.templates['dcos-config.yaml']['package']
[config] = [item for item in package if item['path'] == config_path]
expected_configuration = dedent("""\
# Ref: https://github.com/cloudflare/sslconfig/blob/master/conf
# Modulo ChaCha20 cipher and 3DES bulk encryption algorithm.
# For 3DES see https://jira.mesosphere.com/browse/DCOS-21958
ssl_ciphers EECDH+AES128:RSA+AES128:EECDH+AES256:RSA+AES256:!MD5:!3DES;
ssl_prefer_server_ciphers on;
# To manually test which TLS versions are enabled on a node, use
# `openssl` commands.
#
# See comments on https://jira.mesosphere.com/browse/DCOS-13437 for more
# details.
ssl_protocols TLSv1.2;
""")
assert config['content'] == expected_configuration
def chart(i):
print("Running Live Chart")
gen.generate(ticker)
data = pd.read_csv('stock.csv')
x = data['x']
y = data['y']
#clear current axis
plt.cla()
plt.plot(x, y, label=stock.info['symbol'])
plt.legend(loc='upper left')
plt.tight_layout()
def test_random(self):
for i in range(0, 1000):
n = randint(1, 40)
min_d = randint(2 * n, 100 * n)
a, l = gen.generate(n, min_d)
md = sol.compute_min_distance(n, a, l)
self.assertEqual(md, min_d)
def render_recipe_page(name=None):
render_recipe = None
for recipe in data['recipes']:
if recipe['safe_name'] == name:
with open('data/' + name + '.json') as f1:
data1 = json.load(f1)
chart = generate(data1)
return render_template('chart.html', recipe=data1, chart=chart)
def test_random(self): for i in range(0, 1000): n = randint(1, 40) min_d = randint(2 * n, 100 * n) a, l = gen.generate(n, min_d) md = sol.compute_min_distance(n, a, l) self.assertEqual(md, min_d)
def make_advanced_bunch(variant_args, template_name, cc_params):
extra_templates = [
'aws/dcos-config.yaml',
'aws/templates/advanced/{}'.format(template_name)
]
if cc_params['os_type'] == 'coreos':
extra_templates += ['coreos-aws/cloud-config.yaml', 'coreos/cloud-config.yaml']
cloud_init_implementation = 'coreos'
elif cc_params['os_type'] == 'el7':
cloud_init_implementation = 'canonical'
else:
raise RuntimeError('Unsupported os_type: {}'.format(cc_params['os_type']))
cc_package_files = [
'/etc/cfn_signal_metadata',
'/etc/adminrouter.env',
'/etc/dns_config',
'/etc/exhibitor',
'/etc/mesos-master-provider']
if cc_params['node_type'] == 'master':
cc_package_files.append('/etc/aws_dnsnames')
results = gen.generate(
arguments=variant_args,
extra_templates=extra_templates,
cc_package_files=cc_package_files)
cloud_config = results.templates['cloud-config.yaml']
# Add general services
cloud_config = results.utils.add_services(cloud_config, cloud_init_implementation)
cc_variant = deepcopy(cloud_config)
cc_variant = results.utils.add_units(
cc_variant,
yaml.load(gen.template.parse_str(late_services).render(cc_params)),
cloud_init_implementation)
# Add roles
cc_variant = results.utils.add_roles(cc_variant, cc_params['roles'] + ['aws'])
# NOTE: If this gets printed in string stylerather than '|' the AWS
# parameters which need to be split out for the cloudformation to
# interpret end up all escaped and undoing it would be hard.
variant_cloudconfig = results.utils.render_cloudconfig(cc_variant)
# Render the cloudformation
cloudformation = render_cloudformation(
results.templates[template_name],
cloud_config=variant_cloudconfig)
print("Validating CloudFormation: {}".format(template_name))
validate_cf(cloudformation)
return gen.Bunch({
'cloudformation': cloudformation,
'results': results
})
def make_advanced_bunch(variant_args, template_name, cc_params):
extra_templates = [
'aws/dcos-config.yaml',
'aws/templates/advanced/{}'.format(template_name)
]
if cc_params['os_type'] == 'coreos':
extra_templates += ['coreos-aws/cloud-config.yaml', 'coreos/cloud-config.yaml']
cloud_init_implementation = 'coreos'
elif cc_params['os_type'] == 'el7':
cloud_init_implementation = 'canonical'
else:
raise RuntimeError('Unsupported os_type: {}'.format(cc_params['os_type']))
cc_package_files = [
'/etc/cfn_signal_metadata',
'/etc/adminrouter.env',
'/etc/dns_config',
'/etc/exhibitor',
'/etc/mesos-master-provider']
if cc_params['node_type'] == 'master':
cc_package_files.append('/etc/aws_dnsnames')
results = gen.generate(
arguments=variant_args,
extra_templates=extra_templates,
cc_package_files=cc_package_files)
cloud_config = results.templates['cloud-config.yaml']
# Add general services
cloud_config = results.utils.add_services(cloud_config, cloud_init_implementation)
cc_variant = deepcopy(cloud_config)
cc_variant = results.utils.add_units(
cc_variant,
yaml.load(gen.template.parse_str(late_services).render(cc_params)),
cloud_init_implementation)
# Add roles
cc_variant = results.utils.add_roles(cc_variant, cc_params['roles'] + ['aws'])
# NOTE: If this gets printed in string stylerather than '|' the AWS
# parameters which need to be split out for the cloudformation to
# interpret end up all escaped and undoing it would be hard.
variant_cloudconfig = results.utils.render_cloudconfig(cc_variant)
# Render the cloudformation
cloudformation = render_cloudformation(
results.templates[template_name],
cloud_config=variant_cloudconfig)
print("Validating CloudFormation: {}".format(template_name))
validate_cf(cloudformation)
return gen.Bunch({
'cloudformation': cloudformation,
'results': results
})
def test_fault_domain_disabled():
arguments = make_arguments(new_arguments={
'fault_domain_detect_filename': pkg_resources.resource_filename('gen', 'fault-domain-detect/aws.sh')
})
generated = gen.generate(arguments=arguments)
assert generated.arguments['fault_domain_enabled'] == 'false'
assert 'fault_domain_detect_contents' not in generated.arguments
def test_fault_domain_disabled():
arguments = make_arguments(new_arguments={
'fault_domain_detect_filename': pkg_resources.resource_filename('gen', 'fault-domain-detect/aws.sh')
})
generated = gen.generate(arguments=arguments)
assert generated.arguments['fault_domain_enabled'] == 'false'
assert 'fault_domain_detect_contents' not in generated.arguments
def make_advanced_bundle(variant_args, extra_sources, template_name,
cc_params):
extra_templates = [
'aws/dcos-config.yaml',
'aws/templates/advanced/{}'.format(template_name)
]
supported_os = ('coreos', 'el7')
if cc_params['os_type'] not in supported_os:
raise RuntimeError('Unsupported os_type: {}'.format(
cc_params['os_type']))
elif cc_params['os_type'] == 'coreos':
extra_templates += [
'coreos-aws/cloud-config.yaml', 'coreos/cloud-config.yaml'
]
cloud_init_implementation = 'coreos'
elif cc_params['os_type'] == 'el7':
cloud_init_implementation = 'canonical'
cc_params['os_type'] = 'el7prereq'
results = gen.generate(
arguments=variant_args,
extra_templates=extra_templates,
extra_sources=extra_sources + [aws_base_source],
# TODO(cmaloney): Merge this with dcos_installer/backend.py::get_aws_advanced_target()
extra_targets=[
gen.internals.Target(variables={'cloudformation_s3_url_full'})
])
cloud_config = results.templates['cloud-config.yaml']
# Add general services
cloud_config = results.utils.add_services(cloud_config,
cloud_init_implementation)
cc_variant = deepcopy(cloud_config)
cc_variant = results.utils.add_units(
cc_variant,
yaml.safe_load(
gen.template.parse_str(late_services).render(cc_params)),
cloud_init_implementation)
# Add roles
cc_variant = results.utils.add_roles(cc_variant,
cc_params['roles'] + ['aws'])
# NOTE: If this gets printed in string stylerather than '|' the AWS
# parameters which need to be split out for the cloudformation to
# interpret end up all escaped and undoing it would be hard.
variant_cloudconfig = results.utils.render_cloudconfig(cc_variant)
# Render the cloudformation
cloudformation = render_cloudformation(results.templates[template_name],
cloud_config=variant_cloudconfig)
print("Validating CloudFormation: {}".format(template_name))
validate_cf(cloudformation)
return (cloudformation, results)
def gen_templates(user_args, arm_template):
'''
Render the cloud_config template given a particular set of options
@param user_args: dict, args to pass to the gen library. These are user
input arguments which get filled in/prompted for.
@param arm_template: string, path to the source arm template for rendering
by the gen library (e.g. 'azure/templates/azuredeploy.json')
'''
results = gen.generate(
arguments=user_args,
extra_templates=['azure/cloud-config.yaml', 'azure/templates/' + arm_template + '.json'],
cc_package_files=[
'/etc/exhibitor',
'/etc/exhibitor.properties',
'/etc/adminrouter.env',
'/etc/ui-config.json',
'/etc/mesos-master-provider',
'/etc/master_list']
)
cloud_config = results.templates['cloud-config.yaml']
# Add general services
cloud_config = results.utils.add_services(cloud_config, 'canonical')
# Specialize for master, slave, slave_public
variant_cloudconfig = {}
for variant, params in INSTANCE_GROUPS.items():
cc_variant = deepcopy(cloud_config)
# TODO(cmaloney): Add the dcos-arm-signal service here
# cc_variant = results.utils.add_units(
# cc_variant,
# yaml.load(gen.template.parse_str(late_services).render(params)))
# Add roles
cc_variant = results.utils.add_roles(cc_variant, params['roles'] + ['azure'])
# NOTE: If this gets printed in string stylerather than '|' the Azure
# parameters which need to be split out for the arm to
# interpret end up all escaped and undoing it would be hard.
variant_cloudconfig[variant] = results.utils.render_cloudconfig(cc_variant)
# Render the arm
arm = render_arm(
results.templates[arm_template + '.json'],
variant_cloudconfig['master'],
variant_cloudconfig['slave'],
variant_cloudconfig['slave_public']
)
return gen.Bunch({
'arm': arm,
'results': results
})
def gen_templates(arguments):
results = gen.generate(
arguments=arguments,
extra_templates=[
'aws/templates/cloudformation.json',
'aws/dcos-config.yaml',
'coreos-aws/cloud-config.yaml',
'coreos/cloud-config.yaml'],
cc_package_files=[
'/etc/cfn_signal_metadata',
'/etc/adminrouter.env',
'/etc/ui-config.json',
'/etc/dns_config',
'/etc/exhibitor',
'/etc/mesos-master-provider',
'/etc/aws_dnsnames'])
cloud_config = results.templates['cloud-config.yaml']
# Add general services
cloud_config = results.utils.add_services(cloud_config, 'coreos')
# Specialize for master, slave, slave_public
variant_cloudconfig = {}
for variant, params in cf_instance_groups.items():
cc_variant = deepcopy(cloud_config)
# Specialize the dcos-cfn-signal service
cc_variant = results.utils.add_units(
cc_variant,
yaml.load(gen.template.parse_str(late_services).render(params)))
# Add roles
cc_variant = results.utils.add_roles(cc_variant, params['roles'] + ['aws'])
# NOTE: If this gets printed in string stylerather than '|' the AWS
# parameters which need to be split out for the cloudformation to
# interpret end up all escaped and undoing it would be hard.
variant_cloudconfig[variant] = results.utils.render_cloudconfig(cc_variant)
# Render the cloudformation
cloudformation = render_cloudformation(
results.templates['cloudformation.json'],
master_cloud_config=variant_cloudconfig['master'],
slave_cloud_config=variant_cloudconfig['slave'],
slave_public_cloud_config=variant_cloudconfig['slave_public']
)
print("Validating CloudFormation")
validate_cf(cloudformation)
return gen.Bunch({
'cloudformation': cloudformation,
'results': results
})
def test_edited_ip_detect_script_yields_new_packages():
with tempfile.NamedTemporaryFile() as f:
arguments = make_arguments(new_arguments={'ip_detect_filename': f.name})
f.write('initial script contents\n'.encode('utf-8'))
f.flush()
initial_cluster_packages = gen.generate(arguments).cluster_packages
# Running genconf with the same config yields the same set of packages.
initial_cluster_packages_rerun = gen.generate(arguments).cluster_packages
assert initial_cluster_packages == initial_cluster_packages_rerun
f.seek(0)
f.truncate()
f.write('edited script contents\n'.encode('utf-8'))
f.flush()
edited_cluster_packages = gen.generate(arguments).cluster_packages
# Running genconf with an edited IP detect script yields a new set of packages.
assert initial_cluster_packages != edited_cluster_packages
def do_configure(config):
gen_out = gen.generate(config.as_gen_format(), extra_sources=[gen.build_deploy.bash.onprem_source])
subprocess.check_call(['mkdir', '-p', SERVE_DIR])
gen.build_deploy.bash.generate(gen_out, SERVE_DIR)
# Get bootstrap from artifacts
# TODO(cmaloney): Switch to use a local storage provider like do_aws_configure does.
fetch_bootstrap(gen_out.arguments['bootstrap_id'])
# Write some package metadata
pkgpanda.util.write_json(CLUSTER_PACKAGES_PATH, gen_out.cluster_packages)
def test_edited_ip_detect_script_yields_new_packages():
with tempfile.NamedTemporaryFile() as f:
arguments = make_arguments(new_arguments={'ip_detect_filename': f.name})
f.write('initial script contents\n'.encode('utf-8'))
f.flush()
initial_cluster_packages = gen.generate(arguments).cluster_packages
# Running genconf with the same config yields the same set of packages.
initial_cluster_packages_rerun = gen.generate(arguments).cluster_packages
assert initial_cluster_packages == initial_cluster_packages_rerun
f.seek(0)
f.truncate()
f.write('edited script contents\n'.encode('utf-8'))
f.flush()
edited_cluster_packages = gen.generate(arguments).cluster_packages
# Running genconf with an edited IP detect script yields a new set of packages.
assert initial_cluster_packages != edited_cluster_packages
def gen_templates(user_args, arm_template):
'''
Render the cloud_config template given a particular set of options
@param user_args: dict, args to pass to the gen library. These are user
input arguments which get filled in/prompted for.
@param arm_template: string, path to the source arm template for rendering
by the gen library (e.g. 'azure/templates/azuredeploy.json')
'''
results = gen.generate(
arguments=user_args,
extra_templates=['azure/cloud-config.yaml', 'azure/templates/' + arm_template + '.json'],
cc_package_files=[
'/etc/exhibitor',
'/etc/exhibitor.properties',
'/etc/adminrouter.env',
'/etc/ui-config.json',
'/etc/mesos-master-provider',
'/etc/master_list'])
cloud_config = results.templates['cloud-config.yaml']
# Add general services
cloud_config = results.utils.add_services(cloud_config, 'canonical')
# Specialize for master, slave, slave_public
variant_cloudconfig = {}
for variant, params in INSTANCE_GROUPS.items():
cc_variant = deepcopy(cloud_config)
# TODO(cmaloney): Add the dcos-arm-signal service here
# cc_variant = results.utils.add_units(
# cc_variant,
# yaml.load(gen.template.parse_str(late_services).render(params)))
# Add roles
cc_variant = results.utils.add_roles(cc_variant, params['roles'] + ['azure'])
# NOTE: If this gets printed in string stylerather than '|' the Azure
# parameters which need to be split out for the arm to
# interpret end up all escaped and undoing it would be hard.
variant_cloudconfig[variant] = results.utils.render_cloudconfig(cc_variant)
# Render the arm
arm = render_arm(
results.templates[arm_template + '.json'],
variant_cloudconfig['master'],
variant_cloudconfig['slave'],
variant_cloudconfig['slave_public'])
return gen.Bunch({
'arm': arm,
'results': results
})
def gen_templates(arguments):
results = gen.generate(
arguments=arguments,
extra_templates=[
'aws/templates/cloudformation.json',
'aws/dcos-config.yaml',
'coreos-aws/cloud-config.yaml',
'coreos/cloud-config.yaml'],
cc_package_files=[
'/etc/cfn_signal_metadata',
'/etc/adminrouter.env',
'/etc/ui-config.json',
'/etc/dns_config',
'/etc/exhibitor',
'/etc/mesos-master-provider'])
cloud_config = results.templates['cloud-config.yaml']
# Add general services
cloud_config = results.utils.add_services(cloud_config, 'coreos')
# Specialize for master, slave, slave_public
variant_cloudconfig = {}
for variant, params in cf_instance_groups.items():
cc_variant = deepcopy(cloud_config)
# Specialize the dcos-cfn-signal service
cc_variant = results.utils.add_units(
cc_variant,
yaml.load(gen.template.parse_str(late_services).render(params)))
# Add roles
cc_variant = results.utils.add_roles(cc_variant, params['roles'] + ['aws'])
# NOTE: If this gets printed in string stylerather than '|' the AWS
# parameters which need to be split out for the cloudformation to
# interpret end up all escaped and undoing it would be hard.
variant_cloudconfig[variant] = results.utils.render_cloudconfig(cc_variant)
# Render the cloudformation
cloudformation = render_cloudformation(
results.templates['cloudformation.json'],
master_cloud_config=variant_cloudconfig['master'],
slave_cloud_config=variant_cloudconfig['slave'],
slave_public_cloud_config=variant_cloudconfig['slave_public']
)
print("Validating CloudFormation")
validate_cf(cloudformation)
return gen.Bunch({
'cloudformation': cloudformation,
'results': results
})
def do_configure(config):
gen_out = gen.generate(config.as_gen_format(),
extra_sources=[gen.build_deploy.bash.onprem_source])
subprocess.check_call(['mkdir', '-p', SERVE_DIR])
gen.build_deploy.bash.generate(gen_out, SERVE_DIR)
# Get bootstrap from artifacts
# TODO(cmaloney): Switch to use a local storage provider like do_aws_configure does.
fetch_bootstrap(gen_out.arguments['bootstrap_id'])
# Write some package metadata
pkgpanda.util.write_json(CLUSTER_PACKAGES_PATH, gen_out.cluster_packages)
def do_configure(gen_config):
gen_config.update(get_gen_extra_args())
gen_out = gen.generate(arguments=gen_config)
subprocess.check_call(['mkdir', '-p', SERVE_DIR])
gen.installer.bash.generate(gen_out, SERVE_DIR)
# Get bootstrap from artifacts
# TODO(cmaloney): Switch to use a local storage provider like do_aws_configure does.
fetch_bootstrap(gen_out.arguments['bootstrap_id'])
# Write some package metadata
pkgpanda.util.write_json('genconf/cluster_packages.json', gen_out.cluster_packages)
def gen_simple_template(variant_prefix, filename, arguments, extra_source):
results = gen.generate(
arguments=arguments,
extra_templates=[
'aws/templates/cloudformation.json', 'aws/dcos-config.yaml',
'coreos-aws/cloud-config.yaml', 'coreos/cloud-config.yaml'
],
cc_package_files=[
'/etc/cfn_signal_metadata', '/etc/adminrouter.env',
'/etc/ui-config.json', '/etc/dns_config', '/etc/exhibitor',
'/etc/mesos-master-provider', '/etc/extra_master_addresses'
],
extra_sources=[aws_base_source, aws_simple_source, extra_source])
cloud_config = results.templates['cloud-config.yaml']
# Add general services
cloud_config = results.utils.add_services(cloud_config, 'coreos')
# Specialize for master, slave, slave_public
variant_cloudconfig = {}
for variant, params in cf_instance_groups.items():
cc_variant = deepcopy(cloud_config)
# Specialize the dcos-cfn-signal service
cc_variant = results.utils.add_units(
cc_variant,
yaml.safe_load(
gen.template.parse_str(late_services).render(params)))
# Add roles
cc_variant = results.utils.add_roles(cc_variant,
params['roles'] + ['aws'])
# NOTE: If this gets printed in string stylerather than '|' the AWS
# parameters which need to be split out for the cloudformation to
# interpret end up all escaped and undoing it would be hard.
variant_cloudconfig[variant] = results.utils.render_cloudconfig(
cc_variant)
# Render the cloudformation
cloudformation = render_cloudformation(
results.templates['cloudformation.json'],
master_cloud_config=variant_cloudconfig['master'],
slave_cloud_config=variant_cloudconfig['slave'],
slave_public_cloud_config=variant_cloudconfig['slave_public'])
with logger.scope("Validating CloudFormation"):
validate_cf(cloudformation)
yield from _as_artifact_and_pkg(variant_prefix, filename,
ResultTuple(cloudformation, results))
def test_exhibitor_admin_password_obscured():
var_name = 'exhibitor_admin_password'
var_value = 'secret'
generated = gen.generate(make_arguments(new_arguments={var_name: var_value}))
assert var_name not in json.loads(generated.arguments['expanded_config'])
assert json.loads(generated.arguments['expanded_config_full'])[var_name] == var_value
assert json.loads(generated.arguments['user_arguments'])[var_name] == '**HIDDEN**'
assert json.loads(generated.arguments['user_arguments_full'])[var_name] == var_value
assert yaml.load(generated.arguments['config_yaml'])[var_name] == '**HIDDEN**'
assert yaml.load(generated.arguments['config_yaml_full'])[var_name] == var_value
def test_exhibitor_admin_password_obscured():
var_name = 'exhibitor_admin_password'
var_value = 'secret'
generated = gen.generate(make_arguments(new_arguments={var_name: var_value}))
assert var_name not in json.loads(generated.arguments['expanded_config'])
assert json.loads(generated.arguments['expanded_config_full'])[var_name] == var_value
assert json.loads(generated.arguments['user_arguments'])[var_name] == '**HIDDEN**'
assert json.loads(generated.arguments['user_arguments_full'])[var_name] == var_value
assert yaml.load(generated.arguments['config_yaml'])[var_name] == '**HIDDEN**'
assert yaml.load(generated.arguments['config_yaml_full'])[var_name] == var_value
def do_configure(gen_config):
gen_config.update(get_gen_extra_args())
subprocess.check_output(['mkdir', '-p', SERVE_DIR])
do_validate_gen_config(gen_config)
gen_out = gen.generate(arguments=gen_config)
gen.installer.bash.generate(gen_out, SERVE_DIR)
# Get bootstrap from artifacts
fetch_bootstrap(gen_out.arguments['bootstrap_id'])
# Write some package metadata
pkgpanda.util.write_json('/genconf/cluster_packages.json', gen_out.cluster_packages)
def make_advanced_bundle(variant_args, extra_sources, template_name, cc_params):
extra_templates = [
'aws/dcos-config.yaml',
'aws/templates/advanced/{}'.format(template_name)
]
supported_os = ('coreos', 'el7')
if cc_params['os_type'] not in supported_os:
raise RuntimeError('Unsupported os_type: {}'.format(cc_params['os_type']))
elif cc_params['os_type'] == 'coreos':
extra_templates += ['coreos-aws/cloud-config.yaml', 'coreos/cloud-config.yaml']
cloud_init_implementation = 'coreos'
elif cc_params['os_type'] == 'el7':
cloud_init_implementation = 'canonical'
cc_params['os_type'] = 'el7prereq'
results = gen.generate(
arguments=variant_args,
extra_templates=extra_templates,
extra_sources=extra_sources + [aws_base_source],
# TODO(cmaloney): Merge this with dcos_installer/backend.py::get_aws_advanced_target()
extra_targets=[gen.internals.Target(variables={'cloudformation_s3_url_full'})])
cloud_config = results.templates['cloud-config.yaml']
# Add general services
cloud_config = results.utils.add_services(cloud_config, cloud_init_implementation)
cc_variant = deepcopy(cloud_config)
cc_variant = results.utils.add_units(
cc_variant,
yaml.safe_load(gen.template.parse_str(late_services).render(cc_params)),
cloud_init_implementation)
# Add roles
cc_variant = results.utils.add_roles(cc_variant, cc_params['roles'] + ['aws'])
# NOTE: If this gets printed in string stylerather than '|' the AWS
# parameters which need to be split out for the cloudformation to
# interpret end up all escaped and undoing it would be hard.
variant_cloudconfig = results.utils.render_cloudconfig(cc_variant)
# Render the cloudformation
cloudformation = render_cloudformation(
results.templates[template_name],
cloud_config=variant_cloudconfig)
print("Validating CloudFormation: {}".format(template_name))
validate_cf(cloudformation)
return (cloudformation, results)
def test_adminrouter_ui_x_frame_options_default():
"""
Test that Master Admin Router config file has the correct default
`X-Frame-Options` value. Defaults are present in `calc.py`.
"""
config_path = '/etc_master/adminrouter-ui-security.conf'
arguments = make_arguments(new_arguments={})
generated = gen.generate(arguments=arguments)
package = generated.templates['dcos-config.yaml']['package']
[config] = [item for item in package if item['path'] == config_path]
expected_configuration = dedent("""\
# Browser security settings for the DC/OS UI
add_header X-Frame-Options "DENY";
""")
assert config['content'] == expected_configuration
def test_adminrouter_ui_x_frame_options_default():
"""
Test that Master Admin Router config file has the correct default
`X-Frame-Options` value. Defaults are present in `calc.py`.
"""
config_path = '/etc_master/adminrouter-ui-security.conf'
arguments = make_arguments(new_arguments={})
generated = gen.generate(arguments=arguments)
package = generated.templates['dcos-config.yaml']['package']
[config] = [item for item in package if item['path'] == config_path]
expected_configuration = dedent(
"""\
# Browser security settings for the DC/OS UI
add_header X-Frame-Options "DENY";
"""
)
assert config['content'] == expected_configuration
def test_adminrouter_ui_x_frame_options_custom(value):
"""
Test for all 3 allowed values
See: https://tools.ietf.org/html/rfc7034#section-2.1
"""
config_path = '/etc_master/adminrouter-ui-security.conf'
arguments = make_arguments(new_arguments={
'adminrouter_x_frame_options': value,
})
generated = gen.generate(arguments=arguments)
package = generated.templates['dcos-config.yaml']['package']
[config] = [item for item in package if item['path'] == config_path]
expected_configuration = dedent("""\
# Browser security settings for the DC/OS UI
add_header X-Frame-Options "{value}";
""".format(value=value))
assert config['content'] == expected_configuration
def gen_simple_template(variant_prefix, filename, arguments, extra_source):
results = gen.generate(
arguments=arguments,
extra_templates=[
'aws/templates/cloudformation.json',
'aws/dcos-config.yaml',
'coreos-aws/cloud-config.yaml',
'coreos/cloud-config.yaml'],
extra_sources=[aws_base_source, aws_simple_source, extra_source])
cloud_config = results.templates['cloud-config.yaml']
# Add general services
cloud_config = results.utils.add_services(cloud_config, 'coreos')
# Specialize for master, slave, slave_public
variant_cloudconfig = {}
for variant, params in cf_instance_groups.items():
cc_variant = deepcopy(cloud_config)
# Specialize the dcos-cfn-signal service
cc_variant = results.utils.add_units(
cc_variant,
yaml.safe_load(gen.template.parse_str(late_services).render(params)))
# Add roles
cc_variant = results.utils.add_roles(cc_variant, params['roles'] + ['aws'])
# NOTE: If this gets printed in string stylerather than '|' the AWS
# parameters which need to be split out for the cloudformation to
# interpret end up all escaped and undoing it would be hard.
variant_cloudconfig[variant] = results.utils.render_cloudconfig(cc_variant)
# Render the cloudformation
cloudformation = render_cloudformation(
results.templates['cloudformation.json'],
master_cloud_config=variant_cloudconfig['master'],
slave_cloud_config=variant_cloudconfig['slave'],
slave_public_cloud_config=variant_cloudconfig['slave_public'])
with logger.scope("Validating CloudFormation"):
validate_cf(cloudformation)
yield from _as_artifact_and_pkg(variant_prefix, filename, (cloudformation, results))
def gen_templates(gen_arguments, arm_template, extra_sources):
'''
Render the cloud_config template given a particular set of options
@param user_args: dict, args to pass to the gen library. These are user
input arguments which get filled in/prompted for.
@param arm_template: string, path to the source arm template for rendering
by the gen library (e.g. 'azure/templates/azuredeploy.json')
'''
results = gen.generate(arguments=gen_arguments,
extra_templates=[
'azure/' + cloud_config_yaml,
'azure/templates/' + arm_template + '.json'
],
extra_sources=[azure_base_source] + extra_sources)
cloud_config = results.templates[cloud_config_yaml]
# Add general services
cloud_config = results.utils.add_services(cloud_config, 'canonical')
# Specialize for master, slave, slave_public
variant_cloudconfig = {}
for variant, params in INSTANCE_GROUPS.items():
cc_variant = deepcopy(cloud_config)
# Add roles
cc_variant = results.utils.add_roles(cc_variant,
params['roles'] + ['azure'])
# NOTE: If this gets printed in string stylerather than '|' the Azure
# parameters which need to be split out for the arm to
# interpret end up all escaped and undoing it would be hard.
variant_cloudconfig[variant] = results.utils.render_cloudconfig(
cc_variant)
# Render the arm
arm = render_arm(results.templates[arm_template + '.json'],
variant_cloudconfig['master'],
variant_cloudconfig['slave'],
variant_cloudconfig['slave_public'])
return (arm, results)
def test_adminrouter_ui_x_frame_options_custom(value):
"""
Test for all 3 allowed values
See: https://tools.ietf.org/html/rfc7034#section-2.1
"""
config_path = '/etc_master/adminrouter-ui-security.conf'
arguments = make_arguments(new_arguments={
'adminrouter_x_frame_options': value,
})
generated = gen.generate(arguments=arguments)
package = generated.templates['dcos-config.yaml']['package']
[config] = [item for item in package if item['path'] == config_path]
expected_configuration = dedent(
"""\
# Browser security settings for the DC/OS UI
add_header X-Frame-Options "{value}";
""".format(value=value)
)
assert config['content'] == expected_configuration
def test_agent_default(self):
"""
Test that Agent Admin Router config file has the correct `ssl_ciphers`
and `ssl_protocols` values. It is not possible to override these with
any configuration parameters.
"""
config_path = '/etc/adminrouter-tls-agent.conf'
arguments = make_arguments(new_arguments={})
generated = gen.generate(arguments=arguments)
package = generated.templates['dcos-config.yaml']['package']
[config] = [item for item in package if item['path'] == config_path]
expected_configuration = dedent("""\
# Note that Agent Admin Router only serves cluster-internal clients. Hence,
# browser compatibility is not a criterion for the TLS cipher suite selection.
ssl_ciphers EECDH+AES128:RSA+AES128:EECDH+AES256:RSA+AES256:!MD5;
ssl_prefer_server_ciphers on;
ssl_protocols TLSv1.2;
""")
assert config['content'] == expected_configuration
def make_advanced_bunch(variant_args, template_name, cc_params):
results = gen.generate(
arguments=variant_args,
extra_templates=extra_templates + ['aws/templates/advanced/' + template_name],
cc_package_files=[
'/etc/cfn_signal_metadata',
'/etc/dns_config',
'/etc/exhibitor',
'/etc/mesos-master-provider'])
cloud_config = results.templates['cloud-config.yaml']
# Add general services
cloud_config = results.utils.add_services(cloud_config)
cc_variant = deepcopy(cloud_config)
cc_variant = results.utils.add_units(
cc_variant,
yaml.load(gen.template.parse_str(late_services).render(cc_params)))
# Add roles
cc_variant = results.utils.add_roles(cc_variant, cc_params['roles'] + ['aws'])
# NOTE: If this gets printed in string stylerather than '|' the AWS
# parameters which need to be split out for the cloudformation to
# interpret end up all escaped and undoing it would be hard.
variant_cloudconfig = results.utils.render_cloudconfig(cc_variant)
# Render the cloudformation
cloudformation = render_cloudformation(
results.templates[template_name],
cloud_config=variant_cloudconfig,
)
print("Validating CloudFormation: {}".format(template_name))
validate_cf(cloudformation)
return gen.Bunch({
'cloudformation': cloudformation,
'results': results
})
def gen_templates(gen_arguments, arm_template, extra_sources):
'''
Render the cloud_config template given a particular set of options
@param user_args: dict, args to pass to the gen library. These are user
input arguments which get filled in/prompted for.
@param arm_template: string, path to the source arm template for rendering
by the gen library (e.g. 'azure/templates/azuredeploy.json')
'''
results = gen.generate(
arguments=gen_arguments,
extra_templates=['azure/cloud-config.yaml', 'azure/templates/' + arm_template + '.json'],
extra_sources=[azure_base_source] + extra_sources)
cloud_config = results.templates['cloud-config.yaml']
# Add general services
cloud_config = results.utils.add_services(cloud_config, 'canonical')
# Specialize for master, slave, slave_public
variant_cloudconfig = {}
for variant, params in INSTANCE_GROUPS.items():
cc_variant = deepcopy(cloud_config)
# Add roles
cc_variant = results.utils.add_roles(cc_variant, params['roles'] + ['azure'])
# NOTE: If this gets printed in string stylerather than '|' the Azure
# parameters which need to be split out for the arm to
# interpret end up all escaped and undoing it would be hard.
variant_cloudconfig[variant] = results.utils.render_cloudconfig(cc_variant)
# Render the arm
arm = render_arm(
results.templates[arm_template + '.json'],
variant_cloudconfig['master'],
variant_cloudconfig['slave'],
variant_cloudconfig['slave_public'])
return (arm, results)
def test_agent(self, adminrouter_tls_1_0_enabled):
"""
Test that Agent Admin Router config file has the correct content.
"""
config_path = '/etc/adminrouter-tls-agent.conf'
arguments = make_arguments(new_arguments={
'adminrouter_tls_1_0_enabled': adminrouter_tls_1_0_enabled,
})
generated = gen.generate(arguments=arguments)
package = generated.templates['dcos-config.yaml']['package']
[config] = [item for item in package if item['path'] == config_path]
expected_configuration = dedent(
"""\
# Note that Agent Admin Router only serves cluster-internal clients. Hence,
# browser compatibility is not a criterion for the TLS cipher suite selection.
ssl_ciphers EECDH+AES128:RSA+AES128:EECDH+AES256:RSA+AES256:!MD5;
ssl_prefer_server_ciphers on;
ssl_protocols TLSv1.2;
"""
)
assert config['content'] == expected_configuration
def test_agent_cannot_be_configured(self, tls_versions, ciphers):
"""
Agent Admin Router configuration is not affected by changing Master
Admin Router TLS version or TLS cipher suites configuration.
"""
config_path = '/etc/adminrouter-tls-agent.conf'
new_arguments = {
'adminrouter_tls_1_0_enabled': tls_versions[0],
'adminrouter_tls_1_1_enabled': tls_versions[1],
'adminrouter_tls_1_2_enabled': tls_versions[2],
'adminrouter_tls_cipher_suite': ciphers
}
arguments = make_arguments(new_arguments=new_arguments)
generated = gen.generate(arguments=arguments)
package = generated.templates['dcos-config.yaml']['package']
[config] = [item for item in package if item['path'] == config_path]
expected_configuration = dedent("""\
# Note that Agent Admin Router only serves cluster-internal clients. Hence,
# browser compatibility is not a criterion for the TLS cipher suite selection.
ssl_ciphers EECDH+AES128:RSA+AES128:EECDH+AES256:RSA+AES256:!MD5;
ssl_prefer_server_ciphers on;
ssl_protocols TLSv1.2;
""")
assert config['content'] == expected_configuration
def test_agent_cannot_be_configured(self, tls_versions, ciphers):
"""
Agent Admin Router configuration is not affected by changing Master
Admin Router TLS version or TLS cipher suites configuration.
"""
config_path = '/etc/adminrouter-tls-agent.conf'
new_arguments = {'adminrouter_tls_1_0_enabled': tls_versions[0],
'adminrouter_tls_1_1_enabled': tls_versions[1],
'adminrouter_tls_1_2_enabled': tls_versions[2],
'adminrouter_tls_cipher_suite': ciphers}
arguments = make_arguments(new_arguments=new_arguments)
generated = gen.generate(arguments=arguments)
package = generated.templates['dcos-config.yaml']['package']
[config] = [item for item in package if item['path'] == config_path]
expected_configuration = dedent(
"""\
# Note that Agent Admin Router only serves cluster-internal clients. Hence,
# browser compatibility is not a criterion for the TLS cipher suite selection.
ssl_ciphers EECDH+AES128:RSA+AES128:EECDH+AES256:RSA+AES256:!MD5;
ssl_prefer_server_ciphers on;
ssl_protocols TLSv1.2;
"""
)
assert config['content'] == expected_configuration
def test_exhibitor_tls_initialize_fail():
with pytest.raises(ExhibitorTLSBootstrapError) as exc:
gen.generate(arguments=make_arguments({
'platform': 'onprem',
'exhibitor_tls_enabled': 'false',
'exhibitor_tls_required': 'true',
}))
print(exc.value.errors)
assert exc.value.errors == [
'Exhibitor security is disabled',
'Exhibitor security is an enterprise feature',
'CA init in gen is only supported when using a remote bootstrap node',
]
with pytest.raises(ExhibitorTLSBootstrapError) as exc:
gen.generate(arguments=make_arguments({
'platform': 'onprem',
'exhibitor_tls_enabled': 'true',
'exhibitor_tls_required': 'true',
}))
assert exc.value.errors == [
'Exhibitor security is an enterprise feature',
'CA init in gen is only supported when using a remote bootstrap node',
]
with pytest.raises(ExhibitorTLSBootstrapError) as exc:
gen.generate(arguments=make_arguments({
'platform': 'onprem',
'exhibitor_tls_enabled': 'true',
'exhibitor_tls_required': 'true',
'master_discovery': 'master_http_loadbalancer',
'exhibitor_address': 'http://foobar',
'num_masters': '5',
}))
assert exc.value.errors == [
'Only static master discovery is supported',
'Exhibitor security is an enterprise feature',
'CA init in gen is only supported when using a remote bootstrap node',
]
def add_pw(pwmap, modify):
wnd = Tk()
wnd.title("Add Password")
frame = Frame(wnd)
newkey = StringVar()
newpw = StringVar()
keylbl = Label(frame, text="Name: ")
pwlbl = Label(frame, text="Password: "******"Password Size: ")
pattern = Combobox(frame)
pattern['values'] = ["Alphanumeric", "Restricted Special", "All Characters"]
pattern.set("Alphanumeric")
sizeBox = Combobox(frame)
sizeBox['values'] = list(range(8, 26))
sizeBox.set(15)
editkey = Entry(frame)
editpw = Entry(frame)
gen = Button(frame, text="Generate", command=lambda: also(lambda: editpw.delete(0,END), lambda: editpw.insert(index=END, string=generate(int(sizeBox.get()),pattern.get(), 0))))
done = Button(frame, text="Confirm", command=lambda w=wnd, k=editkey, p=editpw: quit_and(w, lambda: pwmap.update({k.get(): p.get()}), lambda: modify(pwmap)))
frame.grid(column=0, row=0, padx=6, pady=6)
keylbl.grid(column=0, row=0, padx=6, pady=6)
pwlbl.grid(column=0, row=1, padx=6, pady=6)
editkey.grid(column=1, row=0, padx=6, pady=6)
editpw.grid(column=1, row=1, padx=6, pady=6)
pattern.grid(column=2, row=1, padx=6, pady=6)
gen.grid(column=2, row=2, columnspan=2, padx=6, pady=6)
done.grid(column=0, row=2, columnspan=2, padx=6, pady=6)
sizelbl.grid(column=4, row=0, padx=6, pady=6)
sizeBox.grid(column=4,row=1, padx=6, pady=6)
wnd.mainloop()
for i,k in enumerate(elem):
vec[k][0] = vec[k][0] + zv[i][0]
return mat,vec
# print(gen_points())
# elems, lim_points, points = gen_elems(), gen_lim_points(), gen_points()
# mat, vec = asamble(elems, lim_points, points)
elems, lim_points, points = generate()
#points = [(0,0), (0,2), (1,1), (2,1)]
# points = [(-1,-1), (0,0), (1,0), (0,1)]
#elems = [(0,1,2), (1,2,3), (2,3,0)]
#lim_points = [(0,1), (1,3), (3,0)]
# print('old', gen_lim_points())
print('new', lim_points)
print('elems', elems)
from pprint import pprint
pprint(list(enumerate(points)))
def onprem_generate(config):
return gen.generate(config.as_gen_format(), extra_sources=[gen.build_deploy.bash.onprem_source])
import pandas as pd
import numpy as np
import random
import xgboost as xgb
import gen
import model
treeDepths = [1, 2, 3]
learnRates = [0.05, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7]
nRoundList = [100]
#random.seed(0)
train = gen.generate(1, 10, 1000)
test = gen.generate(1, 10, 1000)
for treeDepth in treeDepths:
print("TREEDEPTH: " + str(treeDepth))
mins = []
for learnRate in learnRates:
opList = []
for nRound in nRoundList:
mae, rmse = model.model_and_get_stats(
train, test, {
'max_depth': treeDepth,
'learning_rate': learnRate,
"n_rounds": nRound
})
opList.append(round(mae, 4))
print(opList)
def generate(args):
import gen
if not check_project_folder():
return
if args.election or get_project_option('election', bool):
args.answer_template = 'templates/election_template.tex'
args.sort_questions = True
# args.dont_shuffle_options = True
args.dont_generate_text = True
args.dont_generate_master = True
if args.questionnaire or get_project_option('questionnaire', bool):
args.answer_template = 'templates/questionnaire_template.tex'
args.sort_questions = True
args.dont_shuffle_options = True
args.dont_generate_text = True
args.dont_generate_master = True
name = get_project_option('name')
test_id = int(get_project_option('next_version'))
print("Generating project `{0}`".format(name))
gen.test_id = test_id
set_project_option('next_version', test_id + 1)
os.mkdir(os.path.join('generated', 'v{0}'.format(test_id)))
gen.parser(args.master)
gen.generate(args.tests_count, args)
print("Compiling LaTeX source files")
gen_folder = os.path.join('generated', 'v' + str(test_id))
def dst(path):
return os.path.join(gen_folder, path)
os.chdir(gen_folder)
for f in os.listdir('.'):
if f.endswith('.tex'):
os.system("pdflatex %s -interaction=nonstopmode > compile_latex.log" % f)
os.mkdir('pdf')
os.mkdir('src')
os.mkdir('log')
for f in os.listdir('.'):
if f.endswith('.pdf'):
shutil.move(f, 'pdf')
elif f.endswith('.log'):
shutil.move(f, 'log')
elif f.endswith('.tex') or f.endswith('.png'):
shutil.move(f, 'src')
elif f.endswith('.aux'):
os.remove(f)
os.chdir('..')
if os.path.exists('last'):
os.remove('last')
os.symlink('v' + str(test_id), 'last')
print("Test generated successfully")
os.chdir('..')
dst_image_dir = os.path.join('generated', 'last', 'images')
if not os.path.exists(dst_image_dir):
os.mkdir(dst_image_dir)
print 'Created images directory'
file_list = glob.glob('generated/last/pdf/Answer*')
for i, filename in enumerate(file_list):
os.system('pdftocairo -jpeg {filename} {dst_image_dir}/{i}-scan'
.format(filename=filename, dst_image_dir=dst_image_dir, i=i))
print("Added empty pictures into images folder for debugging.")
from code.display import *
from lib.get_program import get_program
from parser.yaccFile import *
from gen.generate import *
s = get_program()
yacc.yacc()
t = yacc.parse(s)
run_tree(t)
# display(t)
print EXCEPTIONS
if not EXCEPTIONS.list:
# print 'generation'
generate(t)
# print 'number of func/proc:', len(stack.proc_func)
# for i in stack.proc_func: print i.name,i,i.r_type
def do_gen_configure(self):
return gen.generate(self.as_gen_format())