def parameterize(self, call, host): """ Parameterize a Call with its Context set to a per-host Config. """ debug("Parameterizing {!r} for host {!r}".format(call, host)) # Generate a custom ConnectionCall that knows how to yield a Connection # in its make_context(), specifically one to the host requested here. clone = call.clone(into=ConnectionCall) # TODO: using bag-of-attrs is mildly gross but whatever, I'll take it. clone.host = host return clone
def init(c, project, quadrant): """ Before terraform can run we need to initialize it. The init process sets up the backend for state management and insures we don't collide quadrants. """ p_log("Task: Init") reform_root = settings.GetReformRoot() # TODO build this dynamically if project not in projects: debug("Init: Not a valid project: '%s'" % (project)) p_log("Init: Not a valid project: '%s'" % (project)) exit(1) project_path = "%s/projects/%s" % (reform_root, project) project_tf = Path(project_path) if not project_tf.is_dir(): debug("Init: Project path does not exists: '%s'" % (project_path)) p_log("Init: Project path does not exists: '%s'" % (project_path)) exit(2) # Run pre task clean(c, project) preform(c, quadrant) _cmd = "%s init " % (tf_bin) with c.cd(project_path): _fmt_ = c.run("%s fmt" % (tf_bin)).stdout.strip() debug("Init: '%s fmt' output '%s'" % (tf_bin, _fmt_)) _init_ = c.run(_cmd).stdout.strip() debug("Init: %s output '%s'" % (_cmd, _init_))
def parameterize(self, call, connection_init_kwargs): """ Parameterize a Call with its Context set to a per-host Connection. :param call: The generic `.Call` being parameterized. :param connection_init_kwargs: The dict of `.Connection` init params/kwargs to attach to the resulting `.ConnectionCall`. :returns: `.ConnectionCall`. """ msg = "Parameterizing {!r} with Connection kwargs {!r}" debug(msg.format(call, connection_init_kwargs)) # Generate a custom ConnectionCall that has init_kwargs (used for # creating the Connection at runtime) set to the requested params. new_call_kwargs = dict(init_kwargs=connection_init_kwargs) clone = call.clone(into=ConnectionCall, with_=new_call_kwargs) return clone
def mkS3Bucket(c, bucket, region): """ Create the Secure S3 bucket we will store our secret keys in. This will use the kms key with alias aws/s3 for encrypting contents of S3 bucket. """ p_log("Task: mkS3Bucket") # First lets find our kms key with alias kms/s3 client = boto3.client("kms", region) bucket_constraint = {} if region == "us-east-1": s3c = boto3.client("s3") else: s3c = boto3.client("s3", region_name=region) bucket_constraint = {"LocationConstraint": region} p_log("Region: %s" % (region)) create_args = {"ACL": "private", "Bucket": bucket} if region != "us-east-1": create_args["CreateBucketConfiguration"] = bucket_constraint response = s3c.create_bucket(**create_args) response = s3c.put_bucket_versioning( Bucket=bucket, VersioningConfiguration={"Status": "Enabled"} ) debug("mkS3Bucket: {}".format(response)) response = s3c.put_bucket_encryption( Bucket=bucket, ServerSideEncryptionConfiguration={ "Rules": [ { "ApplyServerSideEncryptionByDefault": { "SSEAlgorithm": "aws:kms", } } ] }, ) debug("mkS3Bucket secure: {}".format(response)) p_log("%s created in %s" % (bucket, region))
def plan(c, project, quadrant): """ This does a standard terraform plan in the project specified. It also requires to quadrant to specify what to propose changes for. """ p_log("Start: Plan") reform_root = settings.GetReformRoot() # TODO build this dynamically if project not in projects: debug("Plan: Not a valid project: '%s'" % (project)) p_log("Plan: Not a valid project: '%s'" % (project)) exit(1) project_path = "%s/projects/%s" % (reform_root, project) project_tf = Path(project_path) if not project_tf.is_dir(): debug("Plan: Project path does not exists: '%s'" % (project_path)) p_log("Plan: Project path does not exists: '%s'" % (project_path)) exit(2) # Run pre task init(c, project, quadrant) pl = os.getenv("TF_PARALLEL", 10) _cmd = "%s plan -out=tfplan -parallelism=%s" % (tf_bin, pl) with c.cd(project_path): _init_ = c.run(_cmd).stdout.strip() debug("Plan: %s output '%s'" % (_cmd, _init_)) p_log("Complete: Plan")
def apply(c, project, quadrant): """ This applies a set of changes to terraform. It will run a plan first if a tfplan file is not found """ p_log("Start: Apply") reform_root = settings.GetReformRoot() # TODO build this dynamically if project not in projects: debug("Apply: Not a valid project: '%s'" % (project)) p_log("Apply: Not a valid project: '%s'" % (project)) exit(1) project_path = "%s/projects/%s" % (reform_root, project) project_tf = Path(project_path) if not project_tf.is_dir(): debug("Apply: Project path does not exists: '%s'" % (project_path)) p_log("Apply: Project path does not exists: '%s'" % (project_path)) exit(2) # Run plan if no tfplan exists project_tfplan = "%s/tfplan" % (project_path) project_tfplan_path = Path(project_tfplan) if not project_tfplan_path.is_file(): plan(c, project, quadrant) debug("Apply: produce a plan") pl = os.getenv("TF_PARALLEL", 10) _cmd = "%s apply -parallelism=%s %s" % (tf_bin, pl, project_tfplan) with c.cd(project_path): _init_ = c.run(_cmd).stdout.strip() debug("Apply: %s output '%s'" % (_cmd, _init_)) p_log("Complete: Apply")
def put(self, local, remote=None, preserve_mode=True): """ Upload a file from the local filesystem to the current connection. :param local: Local path of file to upload, or a file-like object. **If a string is given**, it should be a path to a local (regular) file (not a directory). .. note:: When dealing with nonexistent file paths, normal Python file handling concerns come into play - for example, trying to upload a nonexistent ``local`` path will typically result in an `OSError`. **If a file-like object is given**, its contents are written to the remote file path. :param str remote: Remote path to which the local file will be written; is subject to similar behavior as that seen by common Unix utilities or OpenSSH's ``sftp`` or ``scp`` tools. If ``None`` or another 'falsey'/empty value is given (the default), the remote current working directory (typically the connecting user's home directory) is assumed. .. note:: When ``local`` is a file-like object, ``remote`` is required and must refer to a valid file path (not a directory). :param bool preserve_mode: Whether to ``chmod`` the remote file so it matches the local file's mode (default: ``True``). :returns: A `.Result` object. .. versionadded:: 2.0 """ # TODO: preserve honoring of "name" attribute of file-like objects as # in v1, so one CAN just upload to a directory? did we just make that # shit up or is it an actual part of the api in newer Pythons? sftp = self.connection.sftp() if not local: raise ValueError("Local path must not be empty!") is_file_like = hasattr(local, 'write') and callable(local.write) # Massage remote path orig_remote = remote if not remote: if is_file_like: raise ValueError("Must give non-empty remote path when local is a file-like object!") # noqa else: remote = os.path.basename(local) debug("Massaged empty remote path into {!r}".format(remote)) prejoined_remote = remote remote = posixpath.join(sftp.getcwd() or sftp.normalize('.'), remote) if remote != prejoined_remote: msg = "Massaged relative remote path {!r} into {!r}" debug(msg.format(prejoined_remote, remote)) # Massage local path orig_local = local if not is_file_like: local = os.path.abspath(local) if local != orig_local: debug("Massaged relative local path {!r} into {!r}".format(orig_local, local)) # noqa # Run Paramiko-level .put() (side-effects only. womp.) # TODO: push some of the path handling into Paramiko; it should be # responsible for dealing with path cleaning etc. # TODO: probably preserve warning message from v1 when overwriting # existing files. Use logging for that obviously. # # If local appears to be a file-like object, use sftp.putfo, not put if is_file_like: msg = "Uploading file-like object {!r} to {!r}" debug(msg.format(local, remote)) pointer = local.tell() try: local.seek(0) sftp.putfo(fl=local, remotepath=remote) finally: local.seek(pointer) else: debug("Uploading {!r} to {!r}".format(local, remote)) sftp.put(localpath=local, remotepath=remote) # Set mode to same as local end # TODO: Push this down into SFTPClient sometime (requires backwards # incompat release.) # if preserve_mode: local_mode = os.stat(local).st_mode mode = stat.S_IMODE(local_mode) sftp.chmod(remote, mode) # Return something useful return Result( orig_remote=orig_remote, remote=remote, orig_local=orig_local, local=local, connection=self.connection, )
def put(self, local, remote=None, preserve_mode=True): """ Upload a file from the local filesystem to the current connection. :param local: Local path of file to upload, or a file-like object. **If a string is given**, it should be a path to a local (regular) file (not a directory). .. note:: When dealing with nonexistent file paths, normal Python file handling concerns come into play - for example, trying to upload a nonexistent ``local`` path will typically result in an `OSError`. **If a file-like object is given**, its contents are written to the remote file path. :param str remote: Remote path to which the local file will be written. .. note:: Most SFTP servers set the remote working directory to the connecting user's home directory, and (unlike most shells) do *not* expand tildes (``~``). For example, instead of saying ``put("archive.tgz", "~/tmp/")``, say ``put("archive.tgz", "tmp/")``. In addition, this means that 'falsey'/empty values (such as the default value, ``None``) are allowed and result in uploading to the remote home directory. .. note:: When ``local`` is a file-like object, ``remote`` is required and must refer to a valid file path (not a directory). :param bool preserve_mode: Whether to ``chmod`` the remote file so it matches the local file's mode (default: ``True``). :returns: A `.Result` object. .. versionadded:: 2.0 """ # TODO: preserve honoring of "name" attribute of file-like objects as # in v1, so one CAN just upload to a directory? did we just make that # shit up or is it an actual part of the api in newer Pythons? sftp = self.connection.sftp() if not local: raise ValueError("Local path must not be empty!") is_file_like = hasattr(local, "write") and callable(local.write) # Massage remote path orig_remote = remote if not remote: if is_file_like: raise ValueError( "Must give non-empty remote path when local is a file-like object!" # noqa ) else: remote = os.path.basename(local) debug("Massaged empty remote path into {!r}".format(remote)) prejoined_remote = remote remote = posixpath.join(sftp.getcwd() or sftp.normalize("."), remote) if remote != prejoined_remote: msg = "Massaged relative remote path {!r} into {!r}" debug(msg.format(prejoined_remote, remote)) # Massage local path orig_local = local if not is_file_like: local = os.path.abspath(local) if local != orig_local: debug( "Massaged relative local path {!r} into {!r}".format( orig_local, local ) ) # noqa # Run Paramiko-level .put() (side-effects only. womp.) # TODO: push some of the path handling into Paramiko; it should be # responsible for dealing with path cleaning etc. # TODO: probably preserve warning message from v1 when overwriting # existing files. Use logging for that obviously. # # If local appears to be a file-like object, use sftp.putfo, not put if is_file_like: msg = "Uploading file-like object {!r} to {!r}" debug(msg.format(local, remote)) pointer = local.tell() try: local.seek(0) sftp.putfo(fl=local, remotepath=remote) finally: local.seek(pointer) else: debug("Uploading {!r} to {!r}".format(local, remote)) sftp.put(localpath=local, remotepath=remote) # Set mode to same as local end # TODO: Push this down into SFTPClient sometime (requires backwards # incompat release.) # if preserve_mode: local_mode = os.stat(local).st_mode mode = stat.S_IMODE(local_mode) sftp.chmod(remote, mode) # Return something useful return Result( orig_remote=orig_remote, remote=remote, orig_local=orig_local, local=local, connection=self.connection, )
def foo(): debug("my-sentinel")
def preform(c, quadrant): """ A simple preprocessor for terraform that processes *\*.tf.tpl* files. This is how we work around terraforms lack of loops and conditionals. This is also how we seed our dynamic reform configs for state backend and and configs we've defined. """ p_log("Start: Preform") projects_base_path = settings.GetReformRoot() # TODO Open this more to include modules work_dir = settings.GetReformRoot() modules_dir = "%s/modules" % (settings.GetReformRoot()) projects_dir = "%s/projects" % (settings.GetReformRoot()) template_suffix = ".tpl" env = Environment(loader=FileSystemLoader(work_dir), trim_blocks=True) # Custom Jinja Filters def is_list(value): return isinstance(value, list) def is_dict(value): return isinstance(value, dict) env.filters["is_list"] = is_list env.filters["is_dict"] = is_dict env.filters["jsonify"] = json.dumps config = ConfigManager.ConfigManager({"env": quadrant}).get_merge_configs() secret_manager = SecretsManager.SecretsManager( {"key": quadrant, "cipher": "RSA_AES"} ) env_secret = secret_manager.getSecretPath(quadrant) secrets = secret_manager.decryptSecretFile(env_secret) # Handle modules dir for directory, subdirectories, files in os.walk(modules_dir): for file in files: if file.endswith(template_suffix): debug("Found template file: %s" % (file)) full_file_path = os.path.join(directory, file) template = env.get_template(full_file_path.replace(work_dir, "")) new_full_file_path = re.sub( template_suffix, "", os.path.join(directory, "preform_" + file) ) debug("Generating file: %s" % (new_full_file_path)) try: with open(new_full_file_path, "w+") as outfile: redered_template = template.render( config=config, project=os.path.basename(directory), quadrant=quadrant, secrets=secrets, ) debug(redered_template) outfile.write( "##################################################\n" ) outfile.write( "# This file auto generated by preform, do not edit!\n" ) outfile.write("# Instead edit \n") outfile.write("# %s\n" % (full_file_path)) outfile.write( "##################################################\n" ) outfile.write("\n\n") outfile.write(redered_template) outfile.write("\n\n") outfile.close() except: pass # Handle projects dir for directory, subdirectories, files in os.walk(projects_dir): for file in files: if file.endswith(template_suffix): debug("Found template file: %s" % (file)) full_file_path = os.path.join(directory, file) template = env.get_template(full_file_path.replace(work_dir, "")) new_full_file_path = re.sub( template_suffix, "", os.path.join(directory, "preform_" + file) ) debug("Generating file: %s" % (new_full_file_path)) with open(new_full_file_path, "w+") as outfile: redered_template = template.render( config=config, project=os.path.basename(directory), quadrant=quadrant, secrets=secrets, ) debug(redered_template) outfile.write( "##################################################\n" ) outfile.write( "# This file auto generated by preform, do not edit!\n" ) outfile.write("# Instead edit \n") outfile.write("# %s\n" % (full_file_path)) outfile.write( "##################################################\n" ) outfile.write("\n\n") outfile.write(redered_template) outfile.write("\n\n") outfile.close() p_log("Complete: Preform")
def clean(c, project): """ This will clean up the terraform cache directory reform files from your project You need to do this between quadrants """ p_log("Task: Clean") reform_root = settings.GetReformRoot() if project not in projects: debug("Clean: Not a valid project: '%s'" % (project)) p_log("Clean: Not a valid project: '%s'" % (project)) exit(1) project_path = "%s/projects/%s/.terraform" % (reform_root, project) project_tf_cache = Path(project_path) if not project_tf_cache.is_dir(): debug("Clean: Project cache path does not exists: '%s'" % (project_path)) clean = c.run("rm -Rf %s" % (project_path)).stdout.strip() debug("Clean Result: %s" % (clean)) old_project_tfplan = "%s/projects/%s/tfplan" % (reform_root, project) old_project_tfplan_path = Path(old_project_tfplan) if old_project_tfplan_path.is_file(): os.remove(old_project_tfplan) debug("Clean: Removed '%s'" % (old_project_tfplan)) # We should also cleanup any preform files at this step just incase they # change and get abandoned preform_path = "%s/projects/%s/**/preform_*.tf" % (reform_root, project) for filename in glob.iglob(preform_path, recursive=True): debug("Clean: removing %s" % (filename)) os.remove(filename) # TODO Remove preform files from modules preform_path = "%s/modules/%s/**/preform_*.tf" % (reform_root, project) for filename in glob.iglob(preform_path, recursive=True): debug("Clean: removing %s" % (filename)) os.remove(filename)
def foo(ctx): debug("my-sentinel")
def mkS3Bucket(c, bucket, region): """ Create the Secure S3 bucket we will store our secret keys in. This will use the kms key with alias aws/s3 for encrypting contents of S3 bucket. """ p_log("Task: mkS3Bucket") # First lets find our kms key with alias kms/s3 client = boto3.client("kms", region) bucket_constraint = {} if region == 'us-east-1': s3c = boto3.client("s3") else: s3c = boto3.client("s3", region_name=region) bucket_constraint = {"LocationConstraint": region} response = client.list_aliases(Limit=100) more = True kms_id = False while more: more = False # print(response) for a in response["Aliases"]: if a["AliasName"] == "alias/aws/s3": if "TargetKeyId" not in a: p_log( "Can't create secure bucket '%s' in '%s', no initial kms alias/aws/s3 setup yet. This happens after you've created your first secure s3 bucket." % (bucket, region), 'error') kms_id = a["TargetKeyId"] break if response["Truncated"]: more = True response = client.list_aliases(Marker=response["NextMarker"], Limit=100) if not kms_id: debug("mkS3Bucket: Never found KMS ID for secure bucket creation") exit(3) p_log("Region: %s" % (region)) create_args = {"ACL": "private", "Bucket": bucket} if region != 'us-east-1': create_args['CreateBucketConfiguration'] = bucket_constraint response = s3c.create_bucket(**create_args) response = s3c.put_bucket_versioning( Bucket=bucket, VersioningConfiguration={"Status": "Enabled"}) debug("mkS3Bucket: {}".format(response)) response = s3c.put_bucket_encryption( Bucket=bucket, ServerSideEncryptionConfiguration={ "Rules": [{ "ApplyServerSideEncryptionByDefault": { "SSEAlgorithm": "aws:kms", "KMSMasterKeyID": kms_id, } }] }, ) debug("mkS3Bucket secure: {}".format(response)) p_log("%s created in %s" % (bucket, region))