def normalise_filled(self, meta, val):
typ = formatted(overridden("{_key_name_1}"), formatter=MergedOptionStringFormatter).normalise(meta, val)
name = formatted(overridden("{_key_name_0}"), formatter=MergedOptionStringFormatter).normalise(meta, val)
special = {}
kls = special.get(typ, GenericNetscalerConfig)
formatted_string = formatted(string_spec(), formatter=MergedOptionStringFormatter)
formatted_options = dictof(string_spec(), match_spec((six.string_types, formatted_string), fallback=any_spec()))
options = dict(
typ=overridden(typ)
, name=overridden(name)
, bindings=dictof(string_spec()
, netscaler_binding_spec())
, tags=listof(string_spec())
, options=formatted_options
, overrides=formatted_options
, binding_options=formatted_options
, environments=optional_spec(listof(valid_environment_spec()))
)
if typ == "sslcertkey":
options["link"] = listof(string_spec())
as_dict = set_options(**options).normalise(meta, val)
return kls(**dict((name, as_dict[name]) for name in options))
def normalise(self, meta, val):
pairs = []
has_self = False
for account_id in self.accounts(meta):
users = sb.listof(sb.string_spec()).normalise(meta.at("users"), self.resource.get('users', NotSpecified))
for index, name in enumerate(sb.listof(sb.any_spec()).normalise(meta, val)):
if name == "__self__":
if self.self_type != 'role':
raise BadPolicy("No __self__ iam role for this policy", meta=meta)
else:
has_self = True
else:
if isinstance(name, six.string_types):
name = sb.formatted(sb.string_spec(), formatter=MergedOptionStringFormatter).normalise(meta.indexed_at(index), name)
pairs.append((name, account_id))
if has_self:
pairs.append(("role/{0}".format(self.self_name), self.default_account_id(meta)))
for name, account_id in pairs:
service = "sts" if name.startswith("assumed-role") else "iam"
arn = "arn:aws:{0}::{1}:{2}".format(service, account_id, name)
if not users:
yield arn
else:
for user in users:
yield "{0}/{1}".format(arn, user)
def bespin_spec(self):
"""Spec for bespin options"""
formatted_string = formatted(string_spec(),
MergedOptionStringFormatter,
expected_type=six.string_types)
formatted_boolean = formatted(boolean(),
MergedOptionStringFormatter,
expected_type=bool)
return create_spec(Bespin,
validators.deprecated_key(
"region",
"Please use ``environments.<env>.region``"),
config=file_spec(),
configuration=any_spec(),
assume_role=optional_spec(string_spec()),
extra=defaulted(string_spec(), ""),
dry_run=defaulted(boolean(), False),
flat=defaulted(boolean(), False),
environment=optional_spec(string_spec()),
no_assume_role=defaulted(formatted_boolean, False),
chosen_task=defaulted(formatted_string,
"list_tasks"),
chosen_stack=defaulted(formatted_string, ""),
chosen_artifact=defaulted(formatted_string, ""),
extra_imports=listof(imports.import_spec()))
def normalise(self, meta, val):
pairs = []
has_self = False
for account_id in self.accounts(meta):
users = sb.listof(sb.string_spec()).normalise(
meta.at("users"), self.resource.get('users', NotSpecified))
for index, name in enumerate(
sb.listof(sb.any_spec()).normalise(meta, val)):
if name == "__self__":
if self.self_type != 'role':
raise BadPolicy("No __self__ iam role for this policy",
meta=meta)
else:
has_self = True
else:
if isinstance(name, six.string_types):
name = sb.formatted(
sb.string_spec(),
formatter=MergedOptionStringFormatter).normalise(
meta.indexed_at(index), name)
pairs.append((name, account_id))
if has_self:
pairs.append(("role/{0}".format(self.self_name),
self.default_account_id(meta)))
for name, account_id in pairs:
service = "sts" if name.startswith("assumed-role") else "iam"
arn = "arn:aws:{0}::{1}:{2}".format(service, account_id, name)
if not users:
yield arn
else:
for user in users:
yield "{0}/{1}".format(arn, user)
def normalise(self, meta, val):
iam_spec = iam_specs(val, self.self_type, self.self_name)
result = sb.set_options(Service=sb.listof(sb.string_spec()),
Federated=sb.listof(sb.string_spec()),
AWS=sb.listof(sb.string_spec())).normalise(
meta, val)
special = sb.set_options(service=sb.listof(principal_service_spec()),
federated=resource_spec(
self.self_type, self.self_name),
iam=iam_spec).normalise(meta, val)
for arg, lst in special.items():
capitalized = arg.capitalize()
if arg == 'iam':
capitalized = "AWS"
result[capitalized].extend(lst)
for key, val in list(result.items()):
if not val:
del result[key]
continue
# Amazon gets rid of the lists if only one item
# And this mucks around with the diffing....
if len(val) is 1:
result[key] = val[0]
else:
result[key] = sorted(val)
return result
class Result(dictobj.Spec):
specs = dictobj.Field(sb.dictof(spec_key_spec(), sb.has("normalise")))
extra = dictobj.Field(
no_such_key_spec("Use extras instead (notice the s!)"))
extras = dictobj.Field(
sb.listof(sb.tuple_spec(sb.string_spec(),
sb.tupleof(sb.string_spec()))))
def bespin_spec(self):
"""Spec for bespin options"""
formatted_string = formatted(string_spec(), MergedOptionStringFormatter, expected_type=six.string_types)
formatted_boolean = formatted(boolean(), MergedOptionStringFormatter, expected_type=bool)
return create_spec(Bespin
, validators.deprecated_key("region", "Please use ``environments.<env>.region``")
, config = file_spec()
, configuration = any_spec()
, assume_role = optional_spec(string_spec())
, dry_run = defaulted(boolean(), False)
, flat = defaulted(boolean(), False)
, environment = optional_spec(string_spec())
, no_assume_role = defaulted(formatted_boolean, False)
, chosen_task = defaulted(formatted_string, "list_tasks")
, chosen_stack = defaulted(formatted_string, "")
, chosen_artifact = defaulted(formatted_string, "")
, extra_imports = listof(imports.import_spec())
)
def url_checker_spec(self):
return create_spec(deployment_check.UrlChecker
, check_url = required(formatted(string_spec(), formatter=MergedOptionStringFormatter))
, endpoint = required(delayed(stack_specs.var_spec()))
, expect = required(formatted(string_spec(), formatter=MergedOptionStringFormatter))
, timeout_after = defaulted(integer_spec(), 600)
)
def normalise(self, meta, val):
if "content" in val:
spec = sb.set_options(dest=sb.required(
sb.formatted(sb.string_spec(),
formatter=MergedOptionStringFormatter)),
content=sb.string_spec())
result = spec.normalise(meta, val)
context_name = "{0}-{1}".format(
hashlib.md5(result['content'].encode('utf-8')).hexdigest(),
result["dest"].replace("/", "-").replace(" ", "--"))
return Command(
("ADD", "{0} {1}".format(context_name, result["dest"])),
(result["content"], context_name))
else:
spec = sb.set_options(get=sb.required(
sb.listof(
sb.formatted(sb.string_spec(),
formatter=MergedOptionStringFormatter))),
prefix=sb.defaulted(sb.string_spec(), ""))
result = spec.normalise(meta, val)
final = []
for val in result["get"]:
final.append(
Command(
("ADD", "{0} {1}/{2}".format(val, result["prefix"],
val))))
return final
def normalise(self, meta, val):
iam_spec = iam_specs(val, self.self_type, self.self_name)
result = sb.set_options(
Service = sb.listof(sb.string_spec())
, Federated = sb.listof(sb.string_spec())
, AWS = sb.listof(sb.string_spec())
).normalise(meta, val)
special = sb.set_options(
service = sb.listof(principal_service_spec())
, federated = resource_spec(self.self_type, self.self_name)
, iam = iam_spec
).normalise(meta, val)
for arg, lst in special.items():
capitalized = arg.capitalize()
if arg == 'iam':
capitalized = "AWS"
result[capitalized].extend(lst)
for key, val in list(result.items()):
if not val:
del result[key]
continue
# Amazon gets rid of the lists if only one item
# And this mucks around with the diffing....
if len(val) is 1:
result[key] = val[0]
else:
result[key] = sorted(val)
return result
class array_command_spec(many_item_formatted_spec):
value_name = "Command"
specs = [
# First item is just a string
sb.string_spec()
# Second item is a required list of either dicts or strings
,
sb.required(
sb.listof(
sb.match_spec(
(dict, complex_ADD_spec()),
(six.string_types + (list, ),
sb.formatted(sb.string_spec(),
formatter=MergedOptionStringFormatter)))))
]
def create_result(self, action, command, meta, val, dividers):
if callable(command) or isinstance(command, six.string_types):
command = [command]
result = []
for cmd in command:
if not isinstance(cmd, list):
cmd = [cmd]
for c in cmd:
if isinstance(c, Command):
result.append(c)
else:
result.append(Command((action, c)))
return result
def url_checker_spec(self):
return create_spec(deployment_check.UrlChecker
, check_url = required(formatted(string_spec(), formatter=MergedOptionStringFormatter))
, endpoint = required(delayed(stack_specs.var_spec()))
, expect = required(formatted(string_spec(), formatter=MergedOptionStringFormatter))
, timeout_after = defaulted(integer_spec(), 600)
)
def normalise(self, meta, val):
if 'use' in val:
template = val['use']
if template not in meta.everything['templates']:
available = list(meta.everything['templates'].keys())
raise BadTemplate("Template doesn't exist!",
wanted=template,
available=available,
meta=meta)
val = MergedOptions.using(meta.everything['templates'][template],
val)
gateway_name = meta.key_names()['_key_name_0']
gateway_location = formatted_string().normalise(
meta.at('location'), val.get('location', ''))
return sb.create_spec(
Gateway,
name=sb.overridden(gateway_name),
location=sb.required(formatted_string()),
stages=sb.listof(formatted_string()),
api_keys=sb.listof(api_key_spec()),
domain_names=sb.dictof(sb.string_spec(),
custom_domain_name_spec(gateway_location)),
resources=sb.dictof(sb.string_spec(),
gateway_resource_spec())).normalise(meta, val)
def normalise(self, meta, val):
from harpoon.option_spec.harpoon_specs import HarpoonSpec
if "content" in val or "context" in val:
spec = sb.set_options(mtime=sb.optional_spec(sb.integer_spec()), dest=sb.required(sb.formatted(sb.string_spec(), formatter=MergedOptionStringFormatter)), content=sb.string_spec(), context=sb.optional_spec(HarpoonSpec().context_spec))
result = spec.normalise(meta, val)
if result["content"] != "" and result["context"] is not NotSpecified:
raise BadOption("Please don't specify both context and content")
mtime = result["mtime"]
if mtime is NotSpecified:
ctxt = type("Context", (object, ), {"use_git": True})()
mtime = meta.everything["mtime"](ctxt)
context_name = "{0}-{1}-mtime({2})".format(hashlib.md5(result['content'].encode('utf-8')).hexdigest(), result["dest"].replace("/", "-").replace(" ", "--"), mtime)
extra_context = (result["content"], context_name)
if result["context"] is not NotSpecified:
context_name = "{0}.tar".format(context_name)
extra_context = ({"context": result["context"]}, context_name)
return Command(("ADD", "{0} {1}".format(context_name, result["dest"])), extra_context)
else:
spec = sb.set_options(
get=sb.required(sb.listof(sb.formatted(sb.string_spec(), formatter=MergedOptionStringFormatter)))
, prefix = sb.defaulted(sb.string_spec(), "")
)
result = spec.normalise(meta, val)
final = []
for val in result["get"]:
final.append(Command(("ADD", "{0} {1}/{2}".format(val, result["prefix"], val))))
return final
def setup_addon_register(self, photons_app, __main__):
"""Setup our addon register"""
# Create the addon getter and register the crosshair namespace
self.addon_getter = AddonGetter()
self.addon_getter.add_namespace("lifx.photons", Result.FieldSpec(),
Addon.FieldSpec())
# Initiate the addons from our configuration
register = Register(self.addon_getter, self)
if "addons" in photons_app:
addons = photons_app["addons"]
if type(addons) in (MergedOptions, dict) or getattr(
addons, "is_dict", False):
spec = sb.dictof(sb.string_spec(), sb.listof(sb.string_spec()))
meta = Meta(photons_app, []).at("addons")
for namespace, adns in spec.normalise(meta, addons).items():
register.add_pairs(*[(namespace, adn) for adn in adns])
elif photons_app.get("default_activate_all_modules"):
register.add_pairs(("lifx.photons", "__all__"))
if __main__ is not None:
register.add_pairs(("lifx.photons", "__main__"))
# Import our addons
register.recursive_import_known()
# Resolve our addons
register.recursive_resolve_imported()
return register
def environment_spec(self):
"""Spec for each environment"""
return create_spec(Environment
, account_id = required(or_spec(string_spec(), valid_string_spec(validators.regexed("\d+"))))
, region = defaulted(string_spec(), "ap-southeast-2")
, vars = dictionary_spec()
)
def __init__(self, extras=sb.NotSpecified, post_register=False):
self.post_register = post_register
if post_register and extras not in (None, {}, sb.NotSpecified):
msg = "Sorry, can't specify ``extras`` and ``post_register`` at the same time"
raise ProgrammerError(msg)
spec = sb.listof(
sb.tuple_spec(sb.string_spec(), sb.listof(sb.string_spec())))
self.extras = spec.normalise(Meta({}, []), extras)
class artifact_path_spec(many_item_formatted_spec):
value_name = "Artifact Path"
specs = [sb.string_spec(), sb.string_spec()]
creates = ArtifactPath
formatter = MergedOptionStringFormatter
def create_result(self, host_path, artifact_path, meta, val, dividers):
return ArtifactPath(host_path, artifact_path)
class import_spec(many_item_formatted_spec):
value_name = "Import specification"
specs = [sb.string_spec(), sb.string_spec()]
formatter = MergedOptionStringFormatter
def create_result(self, directory, import_name, meta, val, dividers):
directory = sb.directory_spec().normalise(meta, directory)
return Import(directory, import_name)
class logging_statement_spec(statement_spec):
args = lambda s, self_type, self_name: {
"prefix":
sb.formatted(sb.string_spec(), formatter=MergedOptionStringFormatter),
"destination":
sb.formatted(sb.string_spec(), formatter=MergedOptionStringFormatter)
}
required = ["prefix", "destination"]
final_kls = lambda s, *args, **kwargs: LoggingConfig(*args, **kwargs)
class Addon(dictobj.Spec):
name = dictobj.Field(sb.string_spec)
extras = dictobj.Field(
sb.listof(sb.tuple_spec(sb.string_spec(), sb.string_spec())))
resolver = dictobj.Field(sb.any_spec)
namespace = dictobj.Field(sb.string_spec)
class BadHook(DelfickError):
desc = "Bad Hook"
@property
def resolved(self):
errors = []
if getattr(self, "_resolved", None) is None:
try:
self._resolved = list(self.resolver())
except Exception as error:
errors.append(
self.BadHook("Failed to resolve a hook",
name=self.name,
namespace=self.namespace,
error=str(error)))
if errors:
raise self.BadHook(_errors=errors)
return self._resolved
def process(self, collector):
for result in self.resolved:
if collector is not None:
collector.register_converters(result.get("specs", {}), Meta,
collector.configuration,
sb.NotSpecified)
def post_register(self, **kwargs):
list(self.resolver(post_register=True, **kwargs))
def unresolved_dependencies(self):
for namespace, name in self.extras:
yield (namespace, name)
def resolved_dependencies(self):
for result in self.resolved:
for namespace, names in result.get("extras", []):
if not isinstance(names, (tuple, list)):
names = (names, )
for name in names:
yield (namespace, name)
def dependencies(self, all_deps):
for dep in self.unresolved_dependencies():
yield dep
if hasattr(self, "_resolved"):
for dep in self.resolved_dependencies():
yield dep
def normalise(self, meta, val):
val = sb.dictionary_spec().normalise(meta, val)
if "plain" in val:
return val["plain"]
elif "kms" in val:
val = sb.set_options(
kms = sb.required(sb.string_spec())
, location = sb.required(sb.string_spec())
).normalise(meta, val)
return lambda: __import__("boto3").client("kms", val['location']).decrypt(CiphertextBlob=base64.b64decode(val['kms']))['Plaintext'].decode('utf-8')
def normalise(self, meta, val):
shell = defaulted(string_spec(), "/bin/bash").normalise(
meta, meta.everything[[
"images", meta.key_names()["_key_name_2"]
]].get("shell", NotSpecified))
shell = defaulted(
formatted(string_spec(),
formatter=MergedOptionStringFormatter),
shell).normalise(meta, val)
return shell
class mount_spec(many_item_formatted_spec):
value_name = "Volume mounting"
specs = [sb.string_spec(), sb.string_spec()]
optional_specs = [sb.string_spec()]
formatter = MergedOptionStringFormatter
def create_result(self, local_path, container_path, permissions, meta, val,
dividers):
"""Default permissions to rw"""
if permissions is NotSpecified:
permissions = 'rw'
return Mount(local_path, container_path, permissions)
def normalise(self, meta, val):
if 'use' in val:
template = val['use']
if template not in meta.everything['templates']:
available = list(meta.everything['templates'].keys())
raise BadTemplate("Template doesn't exist!",
wanted=template,
available=available,
meta=meta)
val = MergedOptions.using(meta.everything['templates'][template],
val)
formatted_string = sb.formatted(sb.string_or_int_as_string_spec(),
MergedOptionStringFormatter,
expected_type=six.string_types)
function_name = meta.key_names()['_key_name_0']
val = sb.create_spec(Lambda,
name=sb.overridden(function_name),
role=sb.required(
only_one_spec(
resource_spec("lambda",
function_name,
only=["iam"]))),
code=sb.required(function_code_spec()),
handler=function_handler_spec(),
timeout=sb.integer_spec(),
runtime=sb.required(formatted_string),
location=sb.required(formatted_string),
description=formatted_string,
sample_event=sb.defaulted(
sb.or_spec(formatted_dictionary(),
sb.string_spec()), ""),
desired_output_for_test=sb.defaulted(
sb.or_spec(formatted_dictionary(),
sb.string_spec()), ""),
memory_size=sb.defaulted(divisible_by_spec(64),
128)).normalise(
meta, val)
# Hack to make sample_event and desired_output_for_test not appear as a MergedOptions
for key in ('sample_event', 'desired_output_for_test'):
if isinstance(val[key], MergedOptions):
v = val[key].as_dict()
class Arbritrary(dictobj):
fields = list(v.keys())
val[key] = Arbritrary(**v)
return val
def context_spec(self):
"""Spec for specifying context options"""
from harpoon.option_spec import image_objs
return dict_from_bool_spec(lambda meta, val: {"enabled": val}
, create_spec(image_objs.Context
, include = listof(string_spec())
, exclude = listof(string_spec())
, enabled = defaulted(boolean(), True)
, parent_dir = directory_spec(formatted(defaulted(string_spec(), "{config_root}"), formatter=MergedOptionStringFormatter))
, use_gitignore = defaulted(boolean(), False)
, use_git_timestamps = defaulted(or_spec(boolean(), listof(string_spec())), False)
)
)
def setup(self, **kwargs):
account_spec = sb.set_options(
account_id = sb.required(sb.string_spec())
, role_to_assume = sb.required(sb.string_spec())
)
kwargs = sb.set_options(
accounts = sb.required(sb.dictof(sb.string_spec(), account_spec))
, ordered_accounts = sb.required(sb.listof(sb.string_spec()))
, cloudability_auth_token = sb.required(sb.any_spec())
).normalise(Meta({}, []), kwargs)
for key, val in kwargs.items():
setattr(self, key, val)
def normalise(self, meta, val):
if 'use' in val:
template = val['use']
if template not in meta.everything['templates']:
available = list(meta.everything['templates'].keys())
raise BadTemplate("Template doesn't exist!", wanted=template, available=available, meta=meta)
val = MergedOptions.using(meta.everything['templates'][template], val)
formatted_string = sb.formatted(sb.string_or_int_as_string_spec(), MergedOptionStringFormatter, expected_type=six.string_types)
function_name = meta.key_names()['_key_name_0']
val = sb.create_spec(Lambda
, name = sb.overridden(function_name)
, role = sb.required(only_one_spec(resource_spec("lambda", function_name, only=["iam"])))
, code = sb.required(function_code_spec())
, handler = function_handler_spec()
, timeout = sb.integer_spec()
, runtime = sb.required(formatted_string)
, location = sb.required(formatted_string)
, description = formatted_string
, sample_event = sb.defaulted(sb.or_spec(formatted_dictionary(), sb.string_spec()), "")
, desired_output_for_test = sb.defaulted(sb.or_spec(formatted_dictionary(), sb.string_spec()), "")
, memory_size = sb.defaulted(divisible_by_spec(64), 128)
).normalise(meta, val)
# Hack to make sample_event and desired_output_for_test not appear as a MergedOptions
for key in ('sample_event', 'desired_output_for_test'):
if isinstance(val[key], MergedOptions):
v = val[key].as_dict()
class Arbritrary(dictobj):
fields = list(v.keys())
val[key] = Arbritrary(**v)
return val
def normalise_filled(self, meta, val):
"""
Convert to and from a string into an integer
"""
if self.unpacking:
if type(val) is str:
if not regexes["version_number"].match(val):
raise BadSpecValue("Expected string to match \d+.\d+",
got=val,
meta=meta)
return val
val = sb.integer_spec().normalise(meta, val)
major = val >> 0x10
minor = val & 0xFF
return f"{major}.{minor}"
else:
if type(val) is int:
return val
val = sb.string_spec().normalise(meta, val)
m = regexes["version_number"].match(val)
if not m:
raise BadSpecValue(
"Expected version string to match (\d+.\d+)",
wanted=val,
meta=meta)
groups = m.groupdict()
major = int(groups["major"])
minor = int(groups["minor"])
return (major << 0x10) + minor
def ultradns_provider_spec(self):
return sb.create_spec(UltraDNSProvider
, name = sb.formatted(sb.overridden("{_key_name_1}"), formatter=MergedOptionStringFormatter)
, provider_type = sb.required(sb.string_spec())
, username = sb.required(formatted_string)
, password = sb.required(formatted_string)
)
def tags_spec(self):
# http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/Using_Tags.html#tag-restrictions
# Keys = 127 UTF-8 '^aws:' reserved. Values = 255 UTF-8
return dictof(
valid_string_spec(validators.regexed("^.{0,127}$"))
, formatted(string_spec(), after_format=valid_string_spec(validators.regexed("^(?!aws:).{0,255}$")), formatter=MergedOptionStringFormatter)
)
def alerting_system_spec(self):
return create_spec(stack_objs.AlertingSystem
, name = formatted(overridden("{_key_name_1}"), formatter=MergedOptionStringFormatter)
, type = string_choice_spec(["nagios"])
, endpoint = required(formatted(string_spec(), formatter=MergedOptionStringFormatter))
, verify_ssl = defaulted(boolean(), True)
)
def alerting_system_spec(self):
return create_spec(stack_objs.AlertingSystem
, name = formatted(overridden("{_key_name_1}"), formatter=MergedOptionStringFormatter)
, type = string_choice_spec(["nagios"])
, endpoint = required(formatted(string_spec(), formatter=MergedOptionStringFormatter))
, verify_ssl = defaulted(boolean(), True)
)
def normalise(self, meta, val):
if 'use' in val:
template = val['use']
if template not in meta.everything['templates']:
available = list(meta.everything['templates'].keys())
raise BadTemplate("Template doesn't exist!", wanted=template, available=available, meta=meta)
val = MergedOptions.using(meta.everything['templates'][template], val)
formatted_string = sb.formatted(sb.string_spec(), MergedOptionStringFormatter, expected_type=six.string_types)
role_name = meta.key_names()['_key_name_0']
original_permission = sb.listof(permission_dict()).normalise(meta.at("permission"), NotSpecified if "permission" not in val else val["permission"])
deny_permission = sb.listof(permission_dict(effect='Deny')).normalise(meta.at("deny_permission"), NotSpecified if "deny_permission" not in val else val["deny_permission"])
allow_permission = sb.listof(permission_dict(effect='Allow')).normalise(meta.at("allow_permission"), NotSpecified if "allow_permission" not in val else val["allow_permission"])
allow_to_assume_me = sb.listof(trust_dict("principal")).normalise(meta.at("allow_to_assume_me"), val.get("allow_to_assume_me", NotSpecified))
disallow_to_assume_me = sb.listof(trust_dict("notprincipal")).normalise(meta.at("disallow_to_assume_me"), val.get("disallow_to_assume_me", NotSpecified))
val = val.wrapped()
val['trust'] = allow_to_assume_me + disallow_to_assume_me
val['permission'] = original_permission + deny_permission + allow_permission
return sb.create_spec(Role
, name = sb.overridden(role_name)
, description = formatted_string
, trust = sb.container_spec(Document, sb.listof(trust_statement_spec('role', role_name)))
, permission = sb.container_spec(Document, sb.listof(permission_statement_spec('role', role_name)))
, make_instance_profile = sb.defaulted(sb.boolean(), False)
).normalise(meta, val)
def normalise(self, meta, val):
if 'use' in val:
template = val['use']
if template not in meta.everything['templates']:
available = list(meta.everything['templates'].keys())
raise BadTemplate("Template doesn't exist!", wanted=template, available=available, meta=meta)
val = MergedOptions.using(meta.everything['templates'][template], val)
formatted_string = sb.formatted(sb.string_or_int_as_string_spec(), MergedOptionStringFormatter, expected_type=six.string_types)
bucket_name = meta.key_names()['_key_name_0']
original_permission = sb.listof(resource_policy_dict()).normalise(meta.at("permission"), NotSpecified if "permission" not in val else val["permission"])
deny_permission = sb.listof(resource_policy_dict(effect='Deny')).normalise(meta.at("deny_permission"), NotSpecified if "deny_permission" not in val else val["deny_permission"])
allow_permission = sb.listof(resource_policy_dict(effect='Allow')).normalise(meta.at("allow_permission"), NotSpecified if "allow_permission" not in val else val["allow_permission"])
# require_mfa_to_delete is an alias for this permission
if val.get("require_mfa_to_delete") is True:
delete_policy = {"action": "s3:DeleteBucket", "resource": { "s3": "__self__" }, "Condition": { "Bool": { "aws:MultiFactorAuthPresent": True } } }
normalised_delete_policy = resource_policy_dict(effect='Allow').normalise(meta.at("require_mfa_to_delete"), delete_policy)
allow_permission.append(normalised_delete_policy)
val = val.wrapped()
val['permission'] = original_permission + deny_permission + allow_permission
return sb.create_spec(Bucket
, acl = sb.defaulted(sb.match_spec((six.string_types, canned_acl_spec()), (dict, acl_statement_spec('acl', 'acl'))), None)
, name = sb.overridden(bucket_name)
, location = sb.defaulted(formatted_string, None)
, permission = sb.container_spec(Document, sb.listof(resource_policy_statement_spec('bucket', bucket_name)))
, tags = sb.dictof(sb.string_spec(), formatted_string)
, website = sb.defaulted(website_statement_spec("website", "website"), None)
, logging = sb.defaulted(logging_statement_spec("logging", "logging"), None)
, lifecycle = sb.defaulted(sb.listof(lifecycle_statement_spec("lifecycle", "lifecycle")), None)
).normalise(meta, val)
def harpoon_spec(self):
"""Spec for harpoon options"""
formatted_string = formatted(string_spec(),
MergedOptionStringFormatter,
expected_type=six.string_types)
formatted_boolean = formatted(boolean(),
MergedOptionStringFormatter,
expected_type=bool)
return create_spec(
Harpoon,
config=file_spec(),
extra=defaulted(formatted_string, ""),
debug=defaulted(boolean(), False),
chosen_task=defaulted(formatted_string, "list_tasks"),
chosen_image=defaulted(formatted_string, ""),
flat=defaulted(formatted_boolean, False),
no_cleanup=defaulted(formatted_boolean, False),
interactive=defaulted(formatted_boolean, True),
silent_build=defaulted(formatted_boolean, False),
keep_replaced=defaulted(formatted_boolean, False),
ignore_missing=defaulted(formatted_boolean, False),
no_intervention=defaulted(formatted_boolean, False),
intervene_afterwards=defaulted(formatted_boolean, False),
do_push=defaulted(formatted_boolean, False),
only_pushable=defaulted(formatted_boolean, False),
docker_context=any_spec(),
docker_context_maker=any_spec(),
stdout=defaulted(any_spec(), sys.stdout),
tty_stdout=defaulted(any_spec(), lambda: sys.stdout),
tty_stderr=defaulted(any_spec(), lambda: sys.stderr))
class s3_address(many_item_formatted_spec):
value_name = "s3 address"
specs = [sb.string_spec()]
optional_specs = [sb.integer_spec()]
creates = S3Address
seperators = None
formatter = MergedOptionStringFormatter
def create_result(self, address, timeout, meta, val, dividers):
if timeout is NotSpecified:
timeout = 600
options = urlparse(address)
if options.scheme != "s3":
raise BadSpecValue("Not a valid s3 address", meta=meta, got=val)
if not options.netloc:
path = ''
domain = options.path
else:
path = options.path
domain = options.netloc
if not path.startswith('/'):
path = '/'
return S3Address(domain, path, timeout)
def normalise(self, meta, val):
if 'use' in val:
template = val['use']
if template not in meta.everything['templates']:
available = list(meta.everything['templates'].keys())
raise BadTemplate("Template doesn't exist!", wanted=template, available=available, meta=meta)
val = MergedOptions.using(meta.everything['templates'][template], val)
formatted_string = sb.formatted(sb.string_spec(), MergedOptionStringFormatter)
route_name = meta.key_names()['_key_name_0']
val = sb.create_spec(DNSRoute
, name = sb.overridden(route_name)
, zone = formatted_string
, record_type = sb.string_choice_spec(["CNAME"])
, record_target = formatted_string
).normalise(meta, val)
if not val.zone.endswith("."):
val.zone = "{0}.".format(val.zone)
if not isinstance(val.record_target, six.string_types):
if not hasattr(val.record_target, "cname"):
raise BadSpecValue("record_target must point at an object with a cname property", got=type(val.record_target), meta=meta)
val.record_target = val.record_target.cname
return val
def context_spec(self):
"""Spec for specifying context options"""
from harpoon.option_spec import image_objs
return dict_from_bool_spec(
lambda meta, val: {"enabled": val},
create_spec(image_objs.Context,
include=listof(string_spec()),
exclude=listof(string_spec()),
enabled=defaulted(boolean(), True),
parent_dir=directory_spec(
formatted(defaulted(string_spec(),
"{config_root}"),
formatter=MergedOptionStringFormatter)),
use_gitignore=defaulted(boolean(), False),
use_git_timestamps=defaulted(
or_spec(boolean(), listof(string_spec())), False)))
def harpoon_spec(self):
"""Spec for harpoon options"""
formatted_string = formatted(string_spec(), MergedOptionStringFormatter, expected_type=six.string_types)
formatted_boolean = formatted(boolean(), MergedOptionStringFormatter, expected_type=bool)
return create_spec(
Harpoon,
config=file_spec(),
extra=defaulted(formatted_string, ""),
debug=defaulted(boolean(), False),
chosen_task=defaulted(formatted_string, "list_tasks"),
chosen_image=defaulted(formatted_string, ""),
flat=defaulted(formatted_boolean, False),
no_cleanup=defaulted(formatted_boolean, False),
interactive=defaulted(formatted_boolean, True),
silent_build=defaulted(formatted_boolean, False),
keep_replaced=defaulted(formatted_boolean, False),
ignore_missing=defaulted(formatted_boolean, False),
no_intervention=defaulted(formatted_boolean, False),
intervene_afterwards=defaulted(formatted_boolean, False),
do_push=defaulted(formatted_boolean, False),
only_pushable=defaulted(formatted_boolean, False),
docker_context=any_spec(),
docker_context_maker=any_spec(),
stdout=defaulted(any_spec(), sys.stdout),
tty_stdout=defaulted(any_spec(), lambda: sys.stdout),
tty_stderr=defaulted(any_spec(), lambda: sys.stderr),
)
def normalise(self, meta, val):
accounts = list(self.accounts(meta))
if not accounts:
accounts = [self.default_account_id(meta)]
for account_id in accounts:
string_or_dict = sb.or_spec(sb.string_spec(), sb.dictof(sb.string_choice_spec(["key_id", "alias"]), sb.string_spec()))
for key_id in sb.listof(string_or_dict).normalise(meta, val):
alias = None
if key_id == "__self__" or (isinstance(key_id, dict) and (key_id.get("alias") == "__self__" or key_id.get("key_id") == "__self__")):
if self.self_type != "key":
raise BadPolicy("No __self__ key for this policy", meta=meta)
else:
alias = self.self_name
location = self.default_location(meta)
else:
location = self.location(meta)
if not alias:
if isinstance(key_id, six.string_types):
alias = key_id
else:
alias = key_id.get("alias")
key_id = key_id.get("key_id")
if alias:
yield "arn:aws:kms:{0}:{1}:alias/{2}".format(location, account_id, alias)
else:
yield "arn:aws:kms:{0}:{1}:key/{2}".format(location, account_id, key_id)
def normalise_filled(self, meta, val):
"""Only care about valid image names"""
available = list(meta.everything["images"].keys())
val = sb.formatted(sb.string_spec(), formatter=MergedOptionStringFormatter).normalise(meta, val)
if val not in available:
raise BadConfiguration("Specified image doesn't exist", specified=val, available=available)
return val
def normalise(self, meta, val):
result = sb.create_spec(
LambdaMethod,
http_method=sb.overridden(self.method),
resource_name=sb.overridden(self.resource_name),
function=formatted_string(),
location=formatted_string(),
account=sb.optional_spec(formatted_string()),
require_api_key=sb.defaulted(sb.boolean(), False),
request_mapping=sb.defaulted(mapping_spec(),
Mapping("application/json", "")),
mapping=sb.defaulted(
mapping_spec(), Mapping("application/json",
"$input.json('$')")),
sample_event=sb.or_spec(formatted_dictionary(), sb.string_spec()),
desired_output_for_test=sb.or_spec(formatted_dictionary(),
sb.string_spec())).normalise(
meta, val)
for key in ('sample_event', 'desired_output_for_test'):
if isinstance(result[key], six.string_types):
v = result[key]
if v.startswith("{") and v.endswith("}"):
v = sb.formatted(
sb.string_spec(),
formatter=MergedOptionStringFormatter).normalise(
meta.at(key), v)
result[key] = v
function = result.function
location = None
if result.location is not NotSpecified and location is not None:
raise BadSpecValue(
"Please don't specify a defined lambda function and location at the same time",
meta=meta)
if not isinstance(function, six.string_types):
location = function.location
function = function.name
if location is None and result.location is NotSpecified:
raise BadSpecValue("Location is a required key!", meta=meta)
result.function = function
result.location = location
return result
def add_configuration(self, configuration, collect_another_source, done,
result, src):
"""Used to add a file to the configuration, result here is the yaml.load of the src"""
def make_mtime_func(source):
"""Lazily calculate the mtime to avoid wasted computation"""
return lambda context: self.get_committime_or_mtime(
context, source)
if "harpoon" in result:
if "extra_files" in result["harpoon"]:
spec = sb.listof(
sb.formatted(sb.string_spec(),
formatter=MergedOptionStringFormatter))
meta = Meta(MergedOptions.using(result),
[]).at("harpoon").at("extra_files")
for extra in spec.normalise(meta,
result["harpoon"]["extra_files"]):
if os.path.abspath(extra) not in done:
if not os.path.exists(extra):
raise BadConfiguration(
"Specified extra file doesn't exist",
extra=extra,
source=src)
collect_another_source(extra)
if "images" in result and "__images_from__" in result["images"]:
images_from_path = result["images"]["__images_from__"]
if isinstance(images_from_path, six.string_types):
images_from_path = [images_from_path]
for ifp in images_from_path:
if not ifp.startswith("/"):
ifp = os.path.join(os.path.dirname(src), ifp)
if not os.path.exists(ifp) or not os.path.isdir(ifp):
raise self.BadConfigurationErrorKls(
"Specified folder for other configuration files points to a folder that doesn't exist",
path="images.__images_from__",
value=ifp)
for root, dirs, files in os.walk(ifp):
for fle in files:
location = os.path.join(root, fle)
if fle.endswith(".yml") or fle.endswith(".yaml"):
collect_another_source(
location,
prefix=[
"images",
os.path.splitext(os.path.basename(fle))[0]
],
extra={"mtime": make_mtime_func(location)})
del result["images"]["__images_from__"]
if "mtime" not in result:
result["mtime"] = make_mtime_func(src)
configuration.update(result, source=src)
def ultradns_provider_spec(self):
return sb.create_spec(UltraDNSProvider,
name=sb.formatted(
sb.overridden("{_key_name_1}"),
formatter=MergedOptionStringFormatter),
provider_type=sb.required(sb.string_spec()),
username=sb.required(formatted_string),
password=sb.required(formatted_string))
def confirm_deployment_spec(self):
return create_spec(deployment_check.ConfirmDeployment
, deploys_s3_path = optional_spec(listof(stack_specs.s3_address()))
, zero_instances_is_ok = defaulted(boolean(), False)
, auto_scaling_group_name = optional_spec(formatted(string_spec(), formatter=MergedOptionStringFormatter))
, url_checker = optional_spec(self.url_checker_spec)
, sns_confirmation = optional_spec(create_spec(deployment_check.SNSConfirmation
, validators.deprecated_key("auto_scaling_group_id", "Use ``confirm_deployment.auto_scaling_group_name``")
, validators.deprecated_key("env", "Use ``stack.<stack>.env`` instead``")
, timeout = defaulted(integer_spec(), 300)
, version_message = required(formatted(string_spec(), formatter=MergedOptionStringFormatter))
, deployment_queue = required(formatted(string_spec(), formatter=MergedOptionStringFormatter))
))
)
def context_spec(self):
"""Spec for specifying context options"""
from harpoon.option_spec import image_objs
return dict_from_bool_spec(lambda meta, val: {"enabled": val}
, create_spec(image_objs.Context
, validators.deprecated_key("use_git_timestamps", "Since docker 1.8, timestamps no longer invalidate the docker layer cache")
, include = listof(string_spec())
, exclude = listof(string_spec())
, enabled = defaulted(boolean(), True)
, find_options = string_spec()
, parent_dir = directory_spec(formatted(defaulted(string_spec(), "{config_root}"), formatter=MergedOptionStringFormatter))
, use_gitignore = defaulted(boolean(), False)
, ignore_find_errors = defaulted(boolean(), False)
)
)
def extra_prepare(self, configuration, args_dict):
"""Called before the configuration.converters are activated"""
harpoon = MergedOptions.using(
configuration.get('harpoon', MergedOptions()).as_dict(),
dict(args_dict.get("harpoon", MergedOptions()).items())).as_dict()
# Args_dict may itself be a MergedOptions
while "harpoon" in args_dict:
del args_dict["harpoon"]
# Create the addon getter and register the crosshair namespace
self.addon_getter = AddonGetter()
self.addon_getter.add_namespace("harpoon.crosshairs",
Result.FieldSpec(), Addon.FieldSpec())
# Initiate the addons from our configuration
self.register = Register(self.addon_getter, self)
if ("addons" in harpoon) and (
type(harpoon["addons"]) in (MergedOptions, dict)
or getattr(harpoon["addons"], "is_dict", False)):
for namespace, adns in sb.dictof(
sb.string_spec(), sb.listof(sb.string_spec())).normalise(
Meta(harpoon, []).at("addons"),
harpoon["addons"]).items():
self.register.add_pairs(*[(namespace, adn) for adn in adns])
# Import our addons
self.register.recursive_import_known()
# Resolve our addons
self.register.recursive_resolve_imported()
# Make sure images is started
if "images" not in self.configuration:
self.configuration["images"] = {}
# Add our special stuff to the configuration
self.configuration.update(
{
"$@": harpoon.get("extra", ""),
"bash": args_dict["bash"] or NotSpecified,
"harpoon": harpoon,
"command": args_dict['command'] or NotSpecified,
"assume_role": args_dict["assume_role"] or NotSpecified
},
source="<args_dict>")
def environment_spec(self):
"""Spec for each environment"""
return create_spec(stack_objs.Environment
, account_id = required(or_spec(valid_string_spec(validators.regexed("\d+")), integer_spec()))
, region = defaulted(string_spec(), "ap-southeast-2")
, vars = dictionary_spec()
, tags = self.tags_spec
)
def confirm_deployment_spec(self):
return create_spec(deployment_check.ConfirmDeployment
, deploys_s3_path = optional_spec(listof(stack_specs.s3_address()))
, zero_instances_is_ok = defaulted(boolean(), False)
, auto_scaling_group_name = optional_spec(formatted(string_spec(), formatter=MergedOptionStringFormatter))
, url_checker = optional_spec(self.url_checker_spec)
, sns_confirmation = optional_spec(create_spec(deployment_check.SNSConfirmation
, validators.deprecated_key("auto_scaling_group_id", "Use ``confirm_deployment.auto_scaling_group_name``")
, validators.deprecated_key("env", "Use ``stack.<stack>.env`` instead``")
, timeout = defaulted(integer_spec(), 300)
, version_message = required(formatted(string_spec(), formatter=MergedOptionStringFormatter))
, deployment_queue = required(formatted(string_spec(), formatter=MergedOptionStringFormatter))
))
)
def authentications_spec(self):
"""Spec for a group of authentication options"""
return container_spec(authentication_objs.Authentication
, dictof(string_spec(), set_options(
reading = optional_spec(authentication_spec())
, writing = optional_spec(authentication_spec())
)
)
)
def normalise(self, meta, val):
return sb.create_spec(MockMethod
, http_method = sb.overridden(self.method)
, resource_name = sb.overridden(self.resource_name)
, request_mapping = sb.defaulted(mapping_spec(), Mapping("application/json", '{"statusCode": 200}'))
, mapping = mapping_spec()
, require_api_key = sb.defaulted(sb.boolean(), False)
, sample_event = sb.or_spec(sb.dictionary_spec(), sb.string_spec())
, desired_output_for_test = sb.or_spec(sb.dictionary_spec(), sb.string_spec())
).normalise(meta, val)
for key in ('sample_event', 'desired_output_for_test'):
if isinstance(result[key], six.string_types):
v = result[key]
if v.startswith("{") and v.endswith("}"):
v = sb.formatted(sb.string_spec(), formatter=MergedOptionStringFormatter).normalise(meta.at(key), v)
result[key] = v
def extra_configuration_collection(self, configuration):
"""Hook to do any extra configuration collection or converter registration"""
harpoon_spec = HarpoonSpec()
for image in configuration.get('images', {}).keys():
self.make_image_converters(image, configuration, harpoon_spec)
self.register_converters(
{
(0, ("content", )): sb.dictof(sb.string_spec(),
sb.string_spec()),
(0, ("harpoon", )): harpoon_spec.harpoon_spec,
(0, ("authentication", )): harpoon_spec.authentications_spec
}, Meta, configuration, sb.NotSpecified)
# Some other code works better when harpoon no existy
if configuration["harpoon"] is sb.NotSpecified:
del configuration["harpoon"]
def dns_environment_spec(self, this):
formatted_string = sb.formatted(sb.string_spec(), formatter=MergedOptionStringFormatter)
class spec(Spec):
def normalise_filled(s, meta, val):
meta.everything = meta.everything.wrapped()
meta.everything["__site_environments__"] = this["environments"].as_dict()
spec = sb.dictof(sb.string_spec(), sb.listof(formatted_string))
return spec.normalise(meta, val.as_dict())
return spec()
def normalise_filled(self, meta, val):
if "environments" not in meta.everything:
raise BespinError("Please specify {environments}")
val = string_spec().normalise(meta, val)
available = list(meta.everything["environments"].keys())
if val not in available:
raise BespinError("Please choose a valid environment", meta=meta, wanted=val, available=available)
return val
def formatted_dict(self, meta, val, chain=None):
result = {}
for key, val in val.items():
if type(val) is dict:
result[key] = self.formatted_dict(meta.at(key), val, chain)
elif isinstance(val, six.string_types):
result[key] = sb.formatted(sb.string_spec(), formatter=MergedOptionStringFormatter).normalise(meta.at(key), val)
else:
result[key] = val
return result
def tasks_spec(self, available_actions, default_action="run"):
"""Tasks for a particular image"""
return dictof(
self.task_name_spec
, create_spec(task_objs.Task, validators.deprecated_key("spec", "Use ``action`` and ``options`` instead (note that ``action`` defaults to run)")
, action = defaulted(string_choice_spec(available_actions, "No such task"), default_action)
, options = dictionary_spec()
, overrides = dictionary_spec()
, description = string_spec()
)
)
def normalise(self, meta, val):
if 'use' in val:
template = val['use']
if template not in meta.everything['templates']:
available = list(meta.everything['templates'].keys())
raise BadTemplate("Template doesn't exist!", wanted=template, available=available, meta=meta)
val = MergedOptions.using(meta.everything['templates'][template], val)
gateway_name = meta.key_names()['_key_name_0']
gateway_location = formatted_string().normalise(meta.at('location'), val.get('location', ''))
return sb.create_spec(Gateway
, name = sb.overridden(gateway_name)
, location = sb.required(formatted_string())
, stages = sb.listof(formatted_string())
, api_keys = sb.listof(api_key_spec())
, domain_names = sb.dictof(sb.string_spec(), custom_domain_name_spec(gateway_location))
, resources = sb.dictof(sb.string_spec(), gateway_resource_spec())
).normalise(meta, val)
def tasks_spec(self, available_actions, default_action="run"):
"""Tasks for a particular stack"""
return dictof(
self.task_name_spec
, create_spec(task_objs.Task
, action = defaulted(string_choice_spec(available_actions, "No such task"), default_action)
, options = dictionary_spec()
, overrides = dictionary_spec()
, description = string_spec()
)
)
def add_configuration(self, configuration, collect_another_source, done, result, src):
"""Used to add a file to the configuration, result here is the yaml.load of the src"""
configuration.update(result, dont_prefix=[dictobj], source=src)
if "bespin" in configuration:
if "extra_files" in configuration["bespin"]:
for extra in sb.listof(sb.formatted(sb.string_spec(), formatter=MergedOptionStringFormatter)).normalise(Meta(configuration, [("bespin", ""), ("extra_files", "")]), configuration["bespin"]["extra_files"]):
if os.path.abspath(extra) not in done:
if not os.path.exists(extra):
raise BadConfiguration("Specified extra file doesn't exist", extra=extra, source=src)
collect_another_source(extra)