def maybe_copy_master_yml(options):
"""Copy the specified master spinnaker-local.yml, and credentials.
This will look for paths to credentials within the spinnaker-local.yml, and
copy those as well. The paths to the credentials (and the reference
in the config file) will be changed to reflect the filesystem on the
new instance, which may be different than on this instance.
Args:
options [Namespace]: The parser namespace options contain information
about the instance we're going to copy to, as well as the source
of the master spinnaker-local.yml file.
"""
if not options.master_yml:
maybe_inform('custom spinnaker-local.yml',
'.spinnaker/spinnaker-local.yml', '--copy_master_yml')
return
bindings = YamlBindings()
bindings.import_path(options.master_yml)
try:
json_credential_path = bindings.get(
'providers.google.primaryCredentials.jsonPath')
except KeyError:
json_credential_path = None
gcp_home = os.path.join('/home', os.environ['LOGNAME'], '.spinnaker')
# If there are credentials, write them to this path
gcp_credential_path = os.path.join(gcp_home, 'google-credentials.json')
with open(options.master_yml, 'r') as f:
content = f.read()
# Replace all the occurrences of the original credentials path with the
# path that we are going to place the file in on the new instance.
if json_credential_path:
if not os.path.exists(json_credential_path):
raise ValueError('{0} specifies google credentials in {1},'
' which does not exist.'.format(
options.master_yml, json_credential_path))
content = content.replace(json_credential_path, gcp_credential_path)
fd, temp_path = tempfile.mkstemp()
os.fchmod(fd, os.stat(options.master_yml).st_mode) # Copy original mode
os.write(fd, content)
os.close(fd)
actual_path = temp_path
# Copy the credentials here. The cfg file will be copied after.
copy_custom_file(options, actual_path, '.spinnaker/spinnaker-local.yml')
if json_credential_path:
copy_custom_file(options, json_credential_path,
'.spinnaker/google-credentials.json')
if temp_path:
os.remove(temp_path)
def test_load_path(self):
yaml = """
a: A
b: 0
c:
- A
- B
d:
child:
grandchild: x
e:
"""
expect = {
'a': 'A',
'b': 0,
'c': ['A', 'B'],
'd': {
'child': {
'grandchild': 'x'
}
},
'e': None
}
fd, temp_path = tempfile.mkstemp()
os.write(fd, yaml)
os.close(fd)
bindings = YamlBindings()
bindings.import_path(temp_path)
self.assertEqual(expect, bindings.map)
def test_verify_user_access_only_bad(self):
bindings = YamlBindings()
validator = ValidateConfig(configurator=Configurator(
bindings=bindings))
fd, temp = tempfile.mkstemp()
os.close(fd)
try:
os.chmod(temp, 0410)
self.assertFalse(validator.verify_user_access_only(temp))
self.assertEqual(
'"{temp}" should not have non-owner access. Mode is 410.'.
format(temp=temp), validator.errors[0])
os.chmod(temp, 0420)
self.assertFalse(validator.verify_user_access_only(temp))
os.chmod(temp, 0440)
self.assertFalse(validator.verify_user_access_only(temp))
os.chmod(temp, 0401)
self.assertFalse(validator.verify_user_access_only(temp))
os.chmod(temp, 0402)
self.assertFalse(validator.verify_user_access_only(temp))
os.chmod(temp, 0404)
self.assertFalse(validator.verify_user_access_only(temp))
finally:
os.remove(temp)
def test_load_string(self):
yaml = """
a: A
b: 0
c:
- A
- B
d:
child:
grandchild: x
e:
"""
expect = {
'a': 'A',
'b': 0,
'c': ['A', 'B'],
'd': {
'child': {
'grandchild': 'x'
}
},
'e': None
}
bindings = YamlBindings()
bindings.import_string(yaml)
self.assertEqual(expect, bindings.map)
def test_true_false_not_resolved(self):
bindings = YamlBindings()
bindings.import_dict({'indirect': '${t}'})
validator = ValidateConfig(configurator=Configurator(
bindings=bindings))
self.assertFalse(validator.verify_true_false('indirect'))
self.assertEqual('Missing "indirect".', validator.errors[0])
def populate_google_yml(content):
credentials = {'project': '', 'jsonPath': ''}
google_dict = {
'enabled': False,
'defaultRegion': 'us-central1',
'defaultZone': 'us-central1-f',
}
google_dict['primaryCredentials'] = credentials
if is_google_instance():
zone = os.path.basename(
check_fetch(GOOGLE_INSTANCE_METADATA_URL + '/zone',
google=True).content)
google_dict['enabled'] = 'true'
google_dict['defaultRegion'] = zone[:-2]
google_dict['defaultZone'] = zone
credentials['project'] = check_fetch(GOOGLE_METADATA_URL +
'/project/project-id',
google=True).content
bindings = YamlBindings()
bindings.import_dict({'providers': {'google': google_dict}})
content = bindings.transform_yaml_source(content,
'providers.google.enabled')
content = bindings.transform_yaml_source(content,
'providers.google.defaultRegion')
content = bindings.transform_yaml_source(content,
'providers.google.defaultZone')
content = bindings.transform_yaml_source(
content, 'providers.google.primaryCredentials.project')
content = bindings.transform_yaml_source(
content, 'providers.google.primaryCredentials.jsonPath')
return content
def test_load_transitive_indirect(self):
bindings = YamlBindings()
bindings.import_dict({'field': '${injected.value}', 'found': 'FOUND'})
bindings.import_dict({'injected': {'value': '${found}'}})
self.assertEqual('FOUND', bindings.get('field'))
self.assertEqual('FOUND', bindings['field'])
self.assertEqual('FOUND', bindings.get('field', None))
def test_transform_ok(self):
bindings = YamlBindings()
bindings.import_dict({
'a': {
'b': {
'space': 'WithSpace',
'empty': 'Empty'
}
},
'x': {
'unique': True
}
})
template = """
a:
b:
space: {space}
empty:{empty}
unique:
b:
space: A
empty:
"""
source = template.format(space='SPACE', empty='')
expect = template.format(space='WithSpace', empty=' Empty')
got = source
for key in ['a.b.space', 'a.b.empty']:
got = bindings.transform_yaml_source(got, key)
self.assertEqual(expect,
bindings.transform_yaml_source(expect, 'bogus'))
self.assertEqual(expect, got)
def test_list(self):
bindings = YamlBindings()
bindings.import_string(
"root:\n - elem: 'first'\n - elem: 2\n - elem: true\ncopy: ${root}")
self.assertEqual([{'elem': 'first'}, {'elem': 2}, {'elem': True}],
bindings.get('root'))
self.assertEqual(bindings.get('root'), bindings.get('copy'))
def test_bool(self):
bindings = YamlBindings()
bindings.import_string(
"root:\n - elem: true\n - elem: True\n - elem: false\n - elem: False\ncopy: ${root}")
self.assertEqual([{'elem': True}, {'elem': True}, {'elem': False}, {'elem': False}],
bindings.get('root'))
self.assertEqual(bindings.get('root'), bindings.get('copy'))
def test_load_key_not_found(self):
bindings = YamlBindings()
bindings.import_dict({'field': '${injected.value}', 'injected': {}})
with self.assertRaises(KeyError):
bindings['unknown']
self.assertEqual(None, bindings.get('unknown', None))
def test_update_field_union_child(self):
bindings = YamlBindings()
bindings.import_dict({'parent1': {'a': 'A'}, 'parent2': {'x': 'X'}})
bindings.import_dict({'parent1': {'b': 'B'}})
self.assertEqual({'parent1': {'a': 'A', 'b': 'B'},
'parent2': {'x': 'X'}},
bindings.map)
def test_update_yml_source(self):
yaml = """
a: A
b: 0
c:
- A
- B
d:
child:
grandchild: x
e:
"""
fd, temp_path = tempfile.mkstemp()
os.write(fd, yaml)
os.close(fd)
update_dict = {
'b': 'Z',
'd': {
'child': {
'grandchild': 'xy',
'new_grandchild': {
'new_node': 'inserted'
}
}
},
'e': 'AA'
}
expect = {
'a': 'A',
'b': 'Z',
'c': ['A', 'B'],
'd': {
'child': {
'grandchild': 'xy',
'new_grandchild': {
'new_node': 'inserted'
}
}
},
'e': 'AA'
}
with self.assertRaises(KeyError):
YamlBindings.update_yml_source(temp_path,
update_dict,
add_new_nodes=False)
# Reset the file
with open(temp_path, 'w') as fd:
fd.write(yaml)
YamlBindings.update_yml_source(temp_path, update_dict)
comparison_bindings = YamlBindings()
comparison_bindings.import_path(temp_path)
self.assertEqual(expect, comparison_bindings.map)
os.remove(temp_path)
def test_number(self):
bindings = YamlBindings()
bindings.import_string(
"scalar: 123\nneg: -321\ndef: ${unkown:234}\nindirect: ${scalar}")
self.assertEqual(123, bindings.get('scalar'))
self.assertEqual(-321, bindings.get('neg'))
self.assertEqual(234, bindings.get('def'))
self.assertEqual(123, bindings.get('indirect'))
def test_boolean(self):
bindings = YamlBindings()
bindings.import_string(
"t: true\nf: false\ndef: ${unkown:true}\nindirect: ${f}")
self.assertEqual(True, bindings.get('t'))
self.assertEqual(False, bindings.get('f'))
self.assertEqual(True, bindings.get('def'))
self.assertEqual(False, bindings.get('indirect'))
def test_is_reference_bad(self):
bindings = YamlBindings()
validator = ValidateConfig(configurator=Configurator(
bindings=bindings))
self.assertFalse(validator.is_reference('str'))
self.assertFalse(validator.is_reference('true'))
self.assertFalse(validator.is_reference('0'))
self.assertFalse(validator.is_reference('not ${a}'))
def copy_master_yml(options):
"""Copy the specified master spinnaker-local.yml, and credentials.
This will look for paths to credentials within the spinnaker-local.yml, and
copy those as well. The paths to the credentials (and the reference
in the config file) will be changed to reflect the filesystem on the
new instance, which may be different than on this instance.
Args:
options [Namespace]: The parser namespace options contain information
about the instance we're going to copy to, as well as the source
of the master spinnaker-local.yml file.
"""
print 'Creating .spinnaker directory...'
check_run_quick('gcloud compute ssh --command "mkdir -p .spinnaker"'
' --project={project} --zone={zone} {instance}'
.format(project=get_project(options),
zone=options.zone,
instance=options.instance),
echo=False)
bindings = YamlBindings()
bindings.import_path(options.master_yml)
try:
json_credential_path = bindings.get(
'providers.google.primaryCredentials.jsonPath')
except KeyError:
json_credential_path = None
gcp_home = os.path.join('/home', os.environ['LOGNAME'], '.spinnaker')
# If there are credentials, write them to this path
gcp_credential_path = os.path.join(gcp_home, 'google-credentials.json')
with open(options.master_yml, 'r') as f:
content = f.read()
# Replace all the occurances of the original credentials path with the
# path that we are going to place the file in on the new instance.
if json_credential_path:
content = content.replace(json_credential_path, gcp_credential_path)
fd, temp_path = tempfile.mkstemp()
os.write(fd, content)
os.close(fd)
actual_path = temp_path
# Copy the credentials here. The cfg file will be copied after.
copy_file(options, actual_path, '.spinnaker/spinnaker-local.yml')
if json_credential_path:
copy_file(options, json_credential_path,
'.spinnaker/google-credentials.json')
if temp_path:
os.remove(temp_path)
def test_load_dict(self):
expect = {'a': 'A',
'b': 0,
'c': ['A','B'],
'd': {'child': {'grandchild': 'x'}},
'e': None}
bindings = YamlBindings()
bindings.import_dict(expect)
self.assertEqual(expect, bindings.map)
def test_cyclic_reference(self):
bindings = YamlBindings()
bindings.import_dict({
'field': '${injected.value}',
'injected': {
'value': '${field}'
}
})
with self.assertRaises(ValueError):
bindings.get('field')
def test_load_default_bool(self):
bindings = YamlBindings()
bindings.import_dict({'field': '${undefined:True}'})
self.assertEqual(True, bindings.get('field'))
bindings.import_dict({'field': '${undefined:true}'})
self.assertEqual(True, bindings.get('field'))
bindings.import_dict({'field': '${undefined:false}'})
self.assertEqual(False, bindings.get('field'))
bindings.import_dict({'field': '${undefined:False}'})
self.assertEqual(False, bindings.get('field'))
def test_true_false_good(self):
bindings = YamlBindings()
bindings.import_dict(
{'t': True, 'f':False, 'indirect':'${t}', 'default': '${x:true}'})
validator = ValidateConfig(
configurator=Configurator(bindings=bindings))
self.assertTrue(validator.verify_true_false('t'))
self.assertTrue(validator.verify_true_false('f'))
self.assertTrue(validator.verify_true_false('indirect'))
self.assertTrue(validator.verify_true_false('default'))
def test_update_field_replace_child(self):
bindings = YamlBindings()
bindings.import_dict({'parent': {'a': 'A', 'b': 'B', 'c': 'C'}})
bindings.import_dict({'parent': {'a': 'X', 'b': 'Y', 'z': 'Z'}})
self.assertEqual({'parent': {
'a': 'X',
'b': 'Y',
'z': 'Z',
'c': 'C'
}}, bindings.map)
def test_transform_fail(self):
bindings = YamlBindings()
bindings.import_dict({'a': {'b': { 'child': 'Hello, World!'}},
'x' : {'unique': True}})
yaml = """
a:
b:
child: Hello
"""
with self.assertRaises(ValueError):
bindings.transform_yaml_source(yaml, 'x.unique')
def test_update_deck_settings(self):
temp_sourcedir = tempfile.mkdtemp()
temp_targetdir = tempfile.mkdtemp()
template = """
preamble
// BEGIN reconfigure_spinnaker
// var gateUrl = ${{services.gate.baseUrl}};
{gate_url_value}
// var bakeryBaseUrl = ${{services.bakery.baseUrl}};
{bakery_url_value}
// END reconfigure_spinnaker
// var gateUrl = ${{services.gate.baseUrl}};
stuff here is left along.
"""
# This was originally just a comment, which was preserved.
bakery_url_assignment = ("var bakeryBaseUrl = 'BAKERY_BASE_URL';"
"\n# comment")
# This was originally a different let statement that was removed.
gate_url_assignment = "var gateUrl = 'GATE_BASE_URL';"
bindings = YamlBindings()
bindings.import_dict({
'services': {
'gate': {
'baseUrl': 'GATE_BASE_URL'
},
'bakery': {
'baseUrl': 'BAKERY_BASE_URL'
},
}
})
installation = InstallationParameters
installation.INSTALLED_CONFIG_DIR = temp_sourcedir
installation.DECK_INSTALL_DIR = temp_targetdir
configurator = Configurator(installation_parameters=installation,
bindings=bindings)
try:
source_settings_path = os.path.join(temp_sourcedir, 'settings.js')
target_settings_path = os.path.join(temp_targetdir, 'settings.js')
with open(source_settings_path, 'w') as f:
f.write(
template.format(gate_url_value="var gateUrl='old';",
bakery_url_value='# comment'))
configurator.update_deck_settings()
with open(target_settings_path, 'r') as f:
got = f.read()
expect = template.format(gate_url_value=gate_url_assignment,
bakery_url_value=bakery_url_assignment)
self.assertEqual(expect, got)
finally:
shutil.rmtree(temp_sourcedir)
shutil.rmtree(temp_targetdir)
def host_test_helper(self, tests, valid, required=False):
bindings = YamlBindings()
bindings.import_dict(tests)
validator = ValidateConfig(configurator=Configurator(
bindings=bindings))
for key, value in tests.items():
msg = '"{key}" was {valid}'.format(
key=key, valid='invalid' if valid else 'valid')
self.assertEqual(valid, validator.verify_host(key, required), msg)
return validator
def test_create_yml_source(self):
expect = {'first': {'child': 'FirstValue'}, 'second': {'child': True}}
fd, temp_path = tempfile.mkstemp()
os.write(fd, "")
os.close(fd)
YamlBindings.update_yml_source(temp_path, expect)
comparison_bindings = YamlBindings()
comparison_bindings.import_path(temp_path)
self.assertEqual(expect, comparison_bindings.map)
os.remove(temp_path)
def test_verify_at_least_one_provider_enabled_good(self):
bindings = YamlBindings()
bindings.import_dict({
'providers': {
'aws': { 'enabled': False },
'google': {'enabled': False },
'another': {'enabled': True }
},
})
validator = ValidateConfig(
configurator=Configurator(bindings=bindings))
self.assertTrue(validator.verify_at_least_one_provider_enabled())
def test_verify_at_least_one_provider_enabled_bad(self):
bindings = YamlBindings()
bindings.import_dict({
'providers': {
'aws': { 'enabled': False },
'google': {'enabled': False }
},
'services': {'test': { 'enabled': True }}
})
validator = ValidateConfig(
configurator=Configurator(bindings=bindings))
self.assertFalse(validator.verify_at_least_one_provider_enabled())
self.assertEqual('None of the providers are enabled.',
validator.errors[0])
def baseUrl_test_helper(self, tests, valid, scheme_optional):
bindings = YamlBindings()
bindings.import_dict(tests)
validator = ValidateConfig(configurator=Configurator(
bindings=bindings))
for key, value in tests.items():
msg = '"{key}" was {valid}'.format(
key=key, valid='invalid' if valid else 'valid')
self.assertEqual(
valid,
validator.verify_baseUrl(key,
True,
scheme_optional=scheme_optional), msg)
def test_verify_user_access_only_good(self):
bindings = YamlBindings()
validator = ValidateConfig(configurator=Configurator(
bindings=bindings))
fd, temp = tempfile.mkstemp()
os.close(fd)
try:
os.chmod(temp, 0400)
self.assertTrue(validator.verify_user_access_only(temp))
os.chmod(temp, 0600)
self.assertTrue(validator.verify_user_access_only(temp))
finally:
os.remove(temp)