def setup_terraform(version, environment, aws_region, terraform_bin_path,
s3_bucket_name):
print(f"deploy test to region {aws_region}")
tf = Terraform(working_dir=full_path,
terraform_bin_path=terraform_bin_path)
var_tf = {
"module_version": version,
"tiingo_api_key": os.environ.get("TIINGO_API_KEY"),
"aws_region": aws_region,
"environment": environment,
"s3_bucket_name": s3_bucket_name,
}
tf.init()
ret_code, out, err = tf.apply(skip_plan=True, var=var_tf)
if ret_code != 0:
print(err)
ret_code, out, err = tf.destroy(var=var_tf)
raise Exception("Error applying terraform. Error \n {}".format(err))
yield
ret_code, out, err = tf.destroy(var=var_tf)
if ret_code != 0:
print(err)
raise Exception("Error detroying terraform. Error \n {}".format(err))
class TerraformProvider():
def __init__(self, configuration, terraform_workspace):
log.info("Preparing terraform deployment")
log.debug("Using workspace: {}".format(terraform_workspace))
self._backend_provider = get_backend_provider(configuration,
terraform_workspace)
self._controller = Terraform(
working_dir=terraform_workspace,
variables=configuration["terraform"]["parameters"])
self._controller.init(capture_output=False, force_copy=IsFlagged)
@stage("Terraform deploy")
def deploy(self):
log.info("Deploying terraform infrastructure")
self._backend_provider.init_remote_backend()
self._controller.apply(capture_output=False, skip_plan=True)
output = self._controller.output()
artifact.create("terraform_output", content=json.dumps(output))
def destroy(self):
log.info("Destroying terraform infrastructure")
self._controller.destroy(capture_output=False)
self._backend_provider.remove_remote_backend()
class TerraformUtils:
VAR_FILE = "terraform.tfvars.json"
STATE_FILE = "terraform.tfstate"
def __init__(self, working_dir: str):
logging.info("TF FOLDER %s ", working_dir)
self.working_dir = working_dir
self.var_file_path = os.path.join(working_dir, self.VAR_FILE)
self.tf = Terraform(working_dir=working_dir,
state=self.STATE_FILE,
var_file=self.VAR_FILE)
self.init_tf()
def init_tf(self) -> None:
self.tf.cmd("init -plugin-dir=/root/.terraform.d/plugins/",
raise_on_error=True)
def apply(self, refresh: bool = True) -> None:
return_value, output, err = self.tf.apply(no_color=IsFlagged,
refresh=refresh,
input=False,
skip_plan=True)
if return_value != 0:
message = f"Terraform apply failed with return value {return_value}, output {output} , error {err}"
logging.error(message)
raise Exception(message)
def change_variables(self,
variables: Dict[str, str],
refresh: bool = True) -> None:
with open(self.var_file_path, "r+") as _file:
tfvars = json.load(_file)
tfvars.update(variables)
_file.seek(0)
_file.truncate()
json.dump(tfvars, _file)
self.apply(refresh=refresh)
def get_state(self) -> str:
self.tf.read_state_file(self.STATE_FILE)
return self.tf.tfstate
def set_new_vips(self, api_vip: str, ingress_vip: str) -> None:
self.change_variables(variables={
"api_vip": api_vip,
"ingress_vip": ingress_vip
},
refresh=True)
def destroy(self) -> None:
self.tf.destroy(force=True, input=False, auto_approve=True)
def vm_docker_deploy_old(config):
# This script should deploy the instance and return the output/logs after the test has finished
file_dir = os.path.dirname(os.path.realpath(__file__))
provider = config["params"]["Provider"][0]
### Check that a selection was made
if config["selection"]["instance"] is None:
config["logs"] = None
return config
### Setup terraform objects
instance_wkdir = file_dir + "/instance_deploy/" + provider
instance_tf = Terraform(working_dir=instance_wkdir)
docker_tf = Terraform(file_dir + "/docker_deploy")
tfstate_path = config["base_dir"] + '/tf_states/' + str(config["job_id"])
tfvars = config["base_dir"] + "/tfvars.tfvars"
## ALSO DIRECT TO A VARS.TF IN THE BASE_DIR
instance_tf.init(
backend_config={'path': tfstate_path + '/terraform.tfstate'})
instance_tf.apply(var_file=tfvars,
lock=False,
var={'instance_type': config["selection"]["instance"]},
skip_plan=True)
docker_tf.init(backend_config={
'path': tfstate_path + '/docker_tfstate/terraform.tfstate'
})
docker_tf.apply(var_file=tfvars,
lock=False,
var={'tfstate_path': tfstate_path},
skip_plan=True)
docker_tf.init(backend_config={
'path': tfstate_path + '/docker_tfstate/terraform.tfstate'
})
logs = docker_tf.output()
config["logs"] = logs
docker_tf.init(backend_config={
'path': tfstate_path + '/docker_tfstate/terraform.tfstate'
})
docker_tf.destroy(auto_approve=True)
instance_tf.init(
backend_config={'path': tfstate_path + '/terraform.tfstate'})
instance_tf.destroy(auto_approve=True)
return config
def destroy(pathToInf):
terra = Terraform(pathToInf)
return_code, stdout, stderr = terra.destroy()
return return_code
def delete_bastion(plateform):
tf = Terraform(working_dir='terraform/layer-bastion')
code, _, _ = tf.cmd("workspace select " + plateform['name'],
capture_output=False,
no_color=IsNotFlagged,
skip_plan=IsNotFlagged)
code, _, _ = tf.destroy(var={
'region':
plateform['region'],
'remote_bucket':
plateform['remote-bucket'],
'prefix_bucket':
plateform['prefix-bucket'],
'gcp-project':
plateform['gcp-project'],
'instance_type':
plateform['infrastructure']['bastion']['instance-type'],
'instance_image':
plateform['infrastructure']['bastion']['image'],
},
capture_output=False,
no_color=IsNotFlagged,
skip_plan=IsNotFlagged,
auto_approve=True)
if code != 0:
raise Exception("error in Terraform layer-data")
def delete_base(plateform):
tf = Terraform(working_dir='terraform/layer-base')
code, _, _ = tf.cmd("workspace select " + plateform['name'],
capture_output=False,
no_color=IsNotFlagged,
skip_plan=IsNotFlagged)
code, _, _ = tf.destroy(var={
'region':
plateform['region'],
'remote_bucket':
plateform['remote-bucket'],
'prefix_bucket':
plateform['prefix-bucket'],
'gcp-project':
plateform['gcp-project'],
'range-ip':
plateform['infrastructure']['range-ip'],
'range-ip-pod':
plateform['infrastructure']['range-ip-pod'],
'range-ip-svc':
plateform['infrastructure']['range-ip-svc'],
'range-plateform':
plateform['infrastructure']['range-plateform'],
'allowed-ips':
plateform['ips_whitelist'],
'env':
plateform['type']
},
capture_output=False,
no_color=IsNotFlagged,
skip_plan=IsNotFlagged,
auto_approve=True)
if code != 0:
raise Exception("error in Terraform layer-base")
def test_destroy(self):
tf = Terraform(working_dir=current_path,
variables={"test_var": "test"})
tf.init("var_to_output")
ret, out, err = tf.destroy("var_to_output")
assert ret == 0
assert "Destroy complete! Resources: 0 destroyed." in out
class Terraform:
def __init__(self, init):
from python_terraform import Terraform as PythonTerraform
self.terraform = PythonTerraform(working_dir='terraform')
Path(self.working_dir).mkdir(exist_ok=True)
if init:
return_code, _, err = self.terraform.init(
dir_or_plan=self.working_dir)
if return_code != 0:
raise CwsCommandError(err)
@property
def working_dir(self):
return self.terraform.working_dir
def init(self):
return_code, _, err = self.terraform.init()
if return_code != 0:
raise CwsCommandError(err)
def apply(self, workspace, targets):
self.select_workspace(workspace)
return_code, _, err = self.terraform.apply(target=targets,
skip_plan=True,
input=False,
raise_on_error=False,
parallelism=1)
if return_code != 0:
raise CwsCommandError(err)
def destroy(self, workspace, targets):
self.select_workspace(workspace)
return_code, _, err = self.terraform.destroy(target=targets)
if return_code != 0:
raise CwsCommandError(err)
def output(self):
self.select_workspace("default")
values = self.terraform.output(capture_output=True)
return {key: value['value']
for key, value in values.items()} if values else "{}"
def workspace_list(self):
self.select_workspace("default")
return_code, out, err = self.terraform.cmd('workspace', 'list')
if return_code != 0:
raise CwsCommandError(err)
values = out[1:].translate(str.maketrans('', '', ' \t\r')).split('\n')
return filter(None, values)
def select_workspace(self, workspace):
return_code, out, err = self.terraform.workspace('select', workspace)
if workspace != 'default' and return_code != 0:
_, out, err = self.terraform.workspace('new',
workspace,
raise_on_error=True)
if not (Path(self.working_dir) / '.terraform').exists():
self.terraform.init(input=False, raise_on_error=True)
def destroy_terraform(self, cluster):
if not self.working_path:
self.working_path = create_terrafrom_working_dir(cluster_name=cluster)
t = Terraform(working_dir=self.working_path)
p, _, _ = t.destroy('./', synchronous=False, no_color=IsNotFlagged, refresh=True)
for i in p.stdout:
print(i.decode())
_, err = p.communicate()
print(err.decode())
return p.returncode == 0
def delete_kubernetes(plateform):
tf = Terraform(working_dir='terraform/layer-kubernetes')
code, _, _ = tf.cmd("workspace select " + plateform['name'],
capture_output=False,
no_color=IsNotFlagged,
skip_plan=IsNotFlagged)
ip_1 = '10.0.0.1/32'
ip_2 = '10.0.0.1/32'
ip_3 = '10.0.0.1/32'
ip_4 = '10.0.0.1/32'
ip_5 = '10.0.0.1/32'
code, _, _ = tf.destroy(var={
'region':
plateform['region'],
'remote_bucket':
plateform['remote-bucket'],
'prefix_bucket':
plateform['prefix-bucket'],
'gcp-project':
plateform['gcp-project'],
'k8s-version':
plateform['infrastructure']['gke']['version'],
'preemptible':
plateform['infrastructure']['gke']['preemptible'],
'instance-type':
plateform['infrastructure']['gke']['instance-type'],
'white-ip-1':
ip_1,
'white-ip-2':
ip_2,
'white-ip-3':
ip_3,
'white-ip-4':
ip_4,
'white-ip-5':
ip_5,
'min_node':
plateform['infrastructure']['gke']['min'],
'max_node':
plateform['infrastructure']['gke']['max'],
'range_ip_master':
plateform['infrastructure']['range-ip-master']
},
capture_output=False,
no_color=IsNotFlagged,
skip_plan=IsNotFlagged,
auto_approve=True)
if code != 0:
raise Exception("error in Terraform layer-kubernetes")
def delete_terraform_stack(cluster_uuid, project_id, dir_path, state_path, proejct_deleted):
state_file = "{}/{}.tfstate".format(state_path, cluster_uuid)
tf_vars_file = "{}/vars.tf".format(state_path)
tf = Terraform(dir_path)
return_code, stdout, stderr = tf.get(capture_output=False)
return_code, stdout, stderr = tf.init(capture_output=False)
return_code, stdout, stderr = tf.destroy(var_file=tf_vars_file, auto_approve=IsFlagged, capture_output=False,
state=state_file)
shutil.rmtree(state_path)
if proejct_deleted:
shutil.rmtree("{}/{}".format(dir_path, project_id))
return return_code, stdout, stderr
def destroy():
tf = Terraform(working_dir='./terraform')
parser = SafeConfigParser()
parser = SafeConfigParser()
config = os.path.expanduser('~/.aws/config')
parser.read(config)
if not parser.has_section('profile pollexy'):
print "You need to run 'pollexy credentials configure'"
return
region = parser.get('profile pollexy', 'region')
print 'Destroying environment . . . '
code, stdout, stderr = tf.destroy(var={'aws_region': region})
if (stderr):
print stderr
else:
print stdout
def delete_data(plateform, admin_password, app_password, unique_id):
tf = Terraform(working_dir='terraform/layer-data')
code, _, _ = tf.cmd("workspace select " + plateform['name'],
capture_output=False,
no_color=IsNotFlagged,
skip_plan=IsNotFlagged)
code, _, _ = tf.destroy(var={
'region':
plateform['region'],
'remote_bucket':
plateform['remote-bucket'],
'prefix_bucket':
plateform['prefix-bucket'],
'gcp-project':
plateform['gcp-project'],
'database_version':
plateform['infrastructure']['cloudsql']['version'],
'database_instance_type':
plateform['infrastructure']['cloudsql']['instance-type'],
'database_disk_size':
plateform['infrastructure']['cloudsql']['disk-size'],
'admin_password':
admin_password,
'app_password':
app_password,
"unique_id":
unique_id,
'env':
plateform['type']
},
capture_output=False,
no_color=IsNotFlagged,
skip_plan=IsNotFlagged,
auto_approve=True)
if code != 0:
raise Exception("error in Terraform layer-data")
class TerraformUtils:
VAR_FILE = "terraform.tfvars.json"
STATE_FILE = "terraform.tfstate"
def __init__(self, working_dir: str, terraform_init: bool = True):
log.info("TF FOLDER %s ", working_dir)
self.working_dir = working_dir
self.var_file_path = os.path.join(working_dir, self.VAR_FILE)
self.tf = Terraform(working_dir=working_dir,
state=self.STATE_FILE,
var_file=self.VAR_FILE)
if terraform_init:
self.init_tf()
@retry(exceptions=TerraformCommandError, tries=10, delay=10)
def init_tf(self) -> None:
self.tf.cmd("init", raise_on_error=True, capture_output=True)
def select_defined_variables(self, **kwargs):
supported_variables = self.get_variable_list()
return {
k: v
for k, v in kwargs.items() if v and k in supported_variables
}
def get_variable_list(self):
results = list()
for tf_file in pathlib.Path(self.working_dir).glob("*.tf"):
with open(tf_file, "r") as fp:
terraform_file_dict = hcl2.load(fp)
results += terraform_file_dict[
"variable"] if "variable" in terraform_file_dict else list(
)
return list(map(lambda d: next(iter(d)), results))
def apply(self, refresh: bool = True) -> None:
return_value, output, err = self.tf.apply(no_color=IsFlagged,
refresh=refresh,
input=False,
skip_plan=True)
if return_value != 0:
message = f"Terraform apply failed with return value {return_value}, output {output} , error {err}"
log.error(message)
raise Exception(message)
def set_and_apply(self, refresh: bool = True, **kwargs) -> None:
defined_variables = self.select_defined_variables(**kwargs)
self.change_variables(defined_variables)
self.init_tf()
self.apply(refresh=refresh)
def change_variables(self,
variables: Dict[str, str],
refresh: bool = True) -> None:
with open(self.var_file_path, "r+") as _file:
tfvars = json.load(_file)
tfvars.update(variables)
_file.seek(0)
_file.truncate()
json.dump(tfvars, _file)
self.apply(refresh=refresh)
def get_state(self) -> Tfstate:
self.tf.read_state_file(self.STATE_FILE)
return self.tf.tfstate
def get_resources(self, resource_type: str = None) -> List[Dict[str, Any]]:
state = self.get_state()
return [
resource for resource in state.resources
if resource_type is None or resource["type"] == resource_type
]
def set_new_vips(self, api_vip: str, ingress_vip: str) -> None:
self.change_variables(variables={
"api_vip": api_vip,
"ingress_vip": ingress_vip
},
refresh=True)
def destroy(self) -> None:
self.tf.destroy(force=True, input=False, auto_approve=True)
parser.add_argument("--lambdas")
return parser.parse_args()
if __name__ == "__main__":
args = parse_args()
tf = Terraform(working_dir=args.workdir)
tf.init()
if args.apply:
for lf in args.lambdas.split(","):
name = lf.split("/")[-1].split(".")[0]
zip_lambda_function(lf, f"{args.workdir}/{name}.zip")
tf.apply(
no_color=IsFlagged,
refresh=False,
var_file=args.tfvars,
skip_plan=True,
capture_output=False,
)
elif args.destroy:
tf.destroy(
no_color=IsFlagged, var_file=args.tfvars, capture_output=False
)
else:
raise ValueError("Action not specified.")
class Component:
blocname = ""
component_name = ""
workspace = ""
def __init__(self, plateform):
self.plateform = plateform
self.get_constantes()
if self.blocname not in plateform:
return
self.check()
self.define_var()
self.workspace = self.plateform_name
def get_constantes(self):
self.bucket_component_state = self.plateform['bucket-component-state']
self.plateform_name = self.plateform['name']
if 'region' not in self.plateform:
self.plateform['region']='eu-west-1'
self.region = self.plateform['region']
def get_workspace(self):
return self.workspace
def define_var(self):
self.var = {}
def apply(self):
pass
def destroy(self):
pass
# to check:
# - dependancies (exemple: rds need network)
# - yaml validation for component
def check(self):
pass
def init(self, working_dir):
self.tf = Terraform(working_dir)
self.tf.cmd(
"init -backend-config=bucket=" + self.bucket_component_state + " -backend-config=region=" + self.region,
capture_output=True,
no_color=IsNotFlagged
)
def create(self, working_dir, var_component, skip_plan=True, workspace_name=""):
if len(workspace_name) == 0:
workspace_name = self.get_workspace()
if os.path.exists(working_dir+"/.terraform/environment"):
os.remove(working_dir+"/.terraform/environment")
else:
print("File environment not exist")
if os.path.exists(working_dir+"/.terraform/terraform.tfstate"):
os.remove(working_dir+"/.terraform/terraform.tfstate")
else:
print("File terraform.tfstate not exist")
self.init(working_dir=working_dir)
# select workspace
code, _, _ = self.tf.cmd("workspace select " + workspace_name, capture_output=False, no_color=IsNotFlagged, skip_plan=IsNotFlagged)
if code == 1:
self.tf.cmd("workspace new " + workspace_name, capture_output=False, no_color=IsNotFlagged, skip_plan=IsNotFlagged)
# terraform apply
code, _, _ = tf.apply(
var=var_component,
capture_output=False,
no_color=IsNotFlagged,
skip_plan=skip_plan,
auto_approve=True)
if code != 0:
raise Exception("error in component: " + self.component_name)
def delete(self, working_dir, var_component, skip_plan=True, workspace_name=""):
if len(workspace_name) == 0:
workspace_name = self.get_workspace()
if os.path.exists(working_dir+"/.terraform/environment"):
os.remove(working_dir+"/.terraform/environment")
else:
print("File environment not exist")
if os.path.exists(working_dir+"/.terraform/terraform.tfstate"):
os.remove(working_dir+"/.terraform/terraform.tfstate")
else:
print("File terraform.tfstate not exist")
self.init(working_dir=working_dir)
code, _, _ = self.tf.cmd("workspace select " + workspace_name, capture_output=False, no_color=IsNotFlagged, skip_plan=IsNotFlagged)
if code == 1:
print("workspace does not exist")
else:
code, _, _ = self.tf.destroy(
var=var_component,
capture_output=False,
no_color=IsNotFlagged,
skip_plan=IsNotFlagged,
auto_approve=True)
if code != 0:
raise Exception("error in component: " + self.component_name)
def output(self, var_name, working_dir, skip_plan=True, workspace_name=""):
if len(workspace_name) == 0:
workspace_name = self.get_workspace()
print("search output : " + var_name)
self.tf = Terraform(working_dir)
out = ''
code, _, _ = self.tf.cmd("workspace select " + workspace_name, capture_output=False, no_color=IsNotFlagged, skip_plan=IsNotFlagged)
if code == 1:
print("workspace does not exist")
else:
code, out, _ = self.tf.cmd(
"output " + var_name,
no_color=IsNotFlagged)
if code != 0:
raise Exception("error in component: " + self.component_name)
return out
class Terraform:
DEFAULT_DOCKER_HOST = 'unix:///var/run/docker.sock'
DEFAULT_DOCKER_ENTRYPOINT_PATH = '/docker-entrypoint.sh'
DEFAULT_NGINX_DOCKER_ENTRYPOINT_PATH = '/nginx.docker-entrypoint.sh'
DEFAULT_NGINX_DOCKER_IMAGE = 'nginx:stable-alpine'
DEFAULT_NGINX_DOCKER_CONTAINER_HTML_PATH = '/usr/share/nginx/html'
DEFAULT_UPLOAD_PATH = f"$HOME/.{PROJECT_NAME}/"
DEFAULT_SSH_USER = '******'
DEFAULT_SSH_PORT = 22
TERRAFORM_RESOURCE_FILE = 'file'
# trick for terrascript
class null_resource(Resource):
...
class vultr(Provider):
...
class vultr_server(Resource):
...
class vultr_ssh_key(Resource):
...
def __init__(self):
self.work_dir = TERRAFORM_WORK_DIR
self.app = TF(working_dir=self.work_dir)
@contextlib.contextmanager
def terraform_workspace(self):
workspace = f"terraform_workspace_{int(time.time())}"
self.app.create_workspace(workspace)
tmp_dir = TemporaryDirectory()
yield tmp_dir.name
self.app.set_workspace('default')
self.app.cmd('workspace delete', workspace, force=IsFlagged)
@contextlib.contextmanager
def patch_terraform_docker_ssh_conn(self):
# patch ssh config
yield
# clear ssh config
def write_terraform_config(self, config: Terrascript, dir_path: str):
tmp_config_file = NamedTemporaryFile(mode='wt',
suffix='.tf.json',
dir=dir_path,
delete=False)
tmp_config_file.write(str(config))
tmp_config_file.seek(0)
self.app.init(
dir_path,
plugin_dir=f"{self.work_dir}/plugins",
)
return tmp_config_file
def run_terraform_plan(self, config: Terrascript):
with self.terraform_workspace() as tw_dir:
self.write_terraform_config(config, tw_dir)
plan = self.app.plan(tw_dir, no_color=IsFlagged)
return plan
def run_terraform_apply(self, config: Terrascript):
with self.terraform_workspace() as tw_dir:
self.write_terraform_config(config, tw_dir)
print(config)
self.app.apply(tw_dir, skip_plan=True, no_color=IsFlagged)
output_result = self.app.output(json=IsFlagged, no_color=IsFlagged)
print(output_result)
output_var = {
output_var_key: output_result[output_var_key]['value']
for output_var_key in output_result
}
return output_var
def run_terraform_destroy(self, config: Terrascript):
with self.terraform_workspace() as tw_dir:
self.write_terraform_config(config, tw_dir)
destroy_result = self.app.destroy(tw_dir)
return destroy_result
@classmethod
def gen_digital_ocean_config(cls,
config_data: dict,
token: str,
public_key: str = None):
do_config = Terrascript()
do_provider = provider.digitalocean(token=token)
do_droplet_resource = resource.digitalocean_droplet(
f"server",
image=config_data['os_code'],
name=config_data['hostname'],
region=config_data['region_code'],
size=config_data['plan_code'],
ssh_keys=config_data['ssh_keys']
if config_data.get('ssh_keys') else [])
if public_key:
digitalocean_ssh_key = resource.digitalocean_ssh_key(
"digitalocean_ssh_key",
name="default",
public_key=public_key,
)
do_droplet_resource['ssh_keys'] += [
"${digitalocean_ssh_key.digitalocean_ssh_key.id}"
]
do_config += digitalocean_ssh_key
do_output_ip = Output(
'ip', value="${digitalocean_droplet.server.ipv4_address}")
do_output_id = Output('server_id',
value="${digitalocean_droplet.server.id}")
do_config += do_provider
do_config += do_droplet_resource
do_config += do_output_ip
do_config += do_output_id
return do_config
@classmethod
def gen_vultr_config(cls,
config_data: dict,
token: str,
public_key: str = None):
vultr_config = Terrascript()
vultr_provider = cls.vultr(api_key=token,
rate_limit=700,
retry_limit=3)
vultr_server = cls.vultr_server(f"server",
plan_id=config_data['plan_code'],
region_id=config_data['region_code'],
os_id=config_data['os_code'],
hostname=config_data['hostname'],
ssh_key_ids=config_data['ssh_keys']
if config_data.get('ssh_keys') else [])
vultr_output_ip = Output('ip', value="${vultr_server.server.main_ip}")
vultr_output_id = Output('server_id',
value="${vultr_server.server.id}")
if public_key:
vultr_ssh_key = cls.vultr_ssh_key('vultr_ssh_key',
name='default_key',
ssh_key=public_key)
vultr_server["ssh_key_ids"] += [
"${vultr_ssh_key.vultr_ssh_key.id}"
]
vultr_config += vultr_ssh_key
vultr_config += vultr_provider
vultr_config += vultr_server
vultr_config += vultr_output_ip
vultr_config += vultr_output_id
return vultr_config
@classmethod
def add_ssh_key_config(cls, public_key: str):
return provisioner("remote-exec",
provisioner=provisioner(
"remote-exec",
inline=[
'mkdir -p ~/.ssh',
f"{public_key} >> ~/.ssh/authorized_keys"
],
))
@classmethod
def gen_ssh_conn_config(cls,
*,
ssh_user: str = DEFAULT_SSH_USER,
ssh_private_key: str,
ssh_host: str,
ssh_port: int = DEFAULT_SSH_PORT) -> dict:
# see more in https://www.terraform.io/docs/provisioners/connection.html
return {
'type': 'ssh',
'user': ssh_user,
'private_key': ssh_private_key,
'host': ssh_host,
'port': ssh_port,
'timeout': '30s'
}
@classmethod
def gen_site_docker_deploy_config(cls,
*,
docker_host: str = DEFAULT_DOCKER_HOST,
site_name: str = None,
template_tar_bytes: bytes = None,
script: str = None,
ssh_user: str = DEFAULT_SSH_USER,
ssh_private_key: str,
ssh_host: str,
ssh_port: int = DEFAULT_SSH_PORT):
config = Terrascript()
docker_provider = provider.docker(host=docker_host,
connection=cls.gen_ssh_conn_config(
ssh_user=ssh_user,
ssh_private_key=ssh_private_key,
ssh_host=ssh_host,
ssh_port=ssh_port))
docker_image_resource = resource.docker_image(
'nginx_image',
name=cls.DEFAULT_NGINX_DOCKER_IMAGE,
)
docker_container_resource = resource.docker_container(
'nginx_container',
name=f"{site_name}-container-${{random_pet.docker_pet_name.id}}",
image="${docker_image.nginx_image.latest}",
restart="always",
start=True,
ports={'internal': 80},
upload=[])
docker_name_resource = resource.random_pet(
'docker_pet_name',
length=1,
)
if template_tar_bytes:
template_tar_file = f"{site_name}-tar-${{random_pet.docker_pet_name.id}}.tar.gz",
template_tar_file_content = base64.b64encode(
template_tar_bytes).decode('utf8')
template_tar_path = f"{cls.DEFAULT_NGINX_DOCKER_CONTAINER_HTML_PATH}/${template_tar_file}"
# self.upload_file(
# content='conf/myapp.conf',
# destination=f"{self.DEFAULT_UPLOAD_PATH}/${template_tar_file}",
# ssh_user=ssh_user,
# ssh_private_key=ssh_private_key,
# ssh_host=ssh_host,
# ssh_port=ssh_port
# )
docker_container_resource['upload'].append({
'content_base64':
template_tar_file_content,
'file':
template_tar_path
})
if script:
entrypoint_sh_content = TemplateRender().render(
cls.DEFAULT_NGINX_DOCKER_ENTRYPOINT_PATH,
init_script_path=cls.DEFAULT_DOCKER_ENTRYPOINT_PATH,
html_path=cls.DEFAULT_NGINX_DOCKER_CONTAINER_HTML_PATH)
docker_container_resource['upload'].append({
'content':
entrypoint_sh_content,
'file':
cls.DEFAULT_DOCKER_ENTRYPOINT_PATH
})
config += docker_provider
config += docker_image_resource
config += docker_container_resource
config += docker_name_resource
return config
def remote_exec(self,
*,
ssh_user: str = DEFAULT_SSH_USER,
ssh_private_key: str,
ssh_host: str,
ssh_port: int = DEFAULT_SSH_PORT):
exec_config = Terrascript()
ssh_conn = self.gen_ssh_conn_config(ssh_user=ssh_user,
ssh_private_key=ssh_private_key,
ssh_host=ssh_host,
ssh_port=ssh_port)
exec_resource = self.null_resource('remote-exec',
provisioner=provisioner(
"remote-exec",
inline=['ls -la'],
connection=ssh_conn))
exec_config += exec_resource
return exec_config
def upload_file(self,
content: str,
*,
destination: str = DEFAULT_UPLOAD_PATH,
ssh_user: str = DEFAULT_SSH_USER,
ssh_private_key: str,
ssh_host: str,
ssh_port: int = DEFAULT_SSH_PORT):
upload_config = Terrascript()
ssh_conn = self.gen_ssh_conn_config(ssh_user=ssh_user,
ssh_private_key=ssh_private_key,
ssh_host=ssh_host,
ssh_port=ssh_port)
file_resource = self.null_resource('upload_file_resource',
provisioner=provisioner(
self.TERRAFORM_RESOURCE_FILE,
content=content,
destination=destination,
connection=ssh_conn))
upload_config += file_resource
return upload_config
#!python from python_terraform import Terraform, IsFlagged tf_base = Terraform(working_dir='iac/layer-base') tf_users = Terraform(working_dir='iac/layer-users') tf_users.destroy(force=IsFlagged, capture_output=False) tf_base.destroy(force=IsFlagged, capture_output=False)
class Terraform:
DEFAULT_DOCKER_HOST = "unix:///var/run/docker.sock"
DEFAULT_DOCKER_ENTRYPOINT_PATH = "/docker-entrypoint.sh"
DEFAULT_NGINX_DOCKER_ENTRYPOINT_PATH = "/nginx.docker-entrypoint.sh"
DEFAULT_NGINX_DOCKER_IMAGE = "nginx:stable-alpine"
DEFAULT_NGINX_DOCKER_CONTAINER_HTML_PATH = "/usr/share/nginx/html"
DEFAULT_UPLOAD_PATH = f"$HOME/.{PROJECT_NAME}/"
DEFAULT_SSH_USER = "******"
DEFAULT_SSH_PORT = 22
TERRAFORM_RESOURCE_FILE = "file"
# trick for terrascript
class null_resource(Resource):
...
class tencentcloud(Provider):
...
class tencentcloud_availability_zones(Data):
...
class tencentcloud_images(Data):
...
class tencentcloud_instance_types(Data):
...
class tencentcloud_security_group(Resource):
...
class tencentcloud_security_group_lite_rule(Resource):
...
class tencentcloud_instance(Resource):
...
class tencentcloud_key_pair(Resource):
...
class alicloud(Provider):
...
class alicloud_vpc(Resource):
...
class alicloud_key_pair(Resource):
...
class alicloud_security_group(Resource):
...
class alicloud_security_group_rule(Resource):
...
class alicloud_instance(Resource):
...
class alicloud_vswitch(Resource):
...
class alicloud_zones(Data):
...
class vultr(Provider):
...
class vultr_server(Resource):
...
class vultr_ssh_key(Resource):
...
def __init__(self):
self.work_dir = TERRAFORM_WORK_DIR
self.app = TF(working_dir=self.work_dir)
@contextlib.contextmanager
def terraform_workspace(self):
workspace = f"terraform_workspace_{int(time.time())}"
self.app.create_workspace(workspace)
tmp_dir = TemporaryDirectory()
yield tmp_dir.name
self.app.set_workspace("default")
self.app.cmd("workspace delete", workspace, force=IsFlagged)
@contextlib.contextmanager
def patch_terraform_docker_ssh_conn(self):
# patch ssh config
yield
# clear ssh config
def write_terraform_config(self, config: Terrascript, dir_path: str):
tmp_config_file = NamedTemporaryFile(mode="wt",
suffix=".tf.json",
dir=dir_path,
delete=False)
logging.info(str(config))
tmp_config_file.write(str(config))
tmp_config_file.seek(0)
self.app.init(
dir_path
# disable maual plugin because it changes toooo fast
# dir_path, plugin_dir=f"{self.work_dir}/plugins",
)
return tmp_config_file
def run_terraform_plan(self, config: Terrascript):
with self.terraform_workspace() as tw_dir:
self.write_terraform_config(config, tw_dir)
plan = self.app.plan(tw_dir, no_color=IsFlagged)
return plan
def run_terraform_apply(self, config: Terrascript):
with self.terraform_workspace() as tw_dir:
self.write_terraform_config(config, tw_dir)
self.app.apply(tw_dir, skip_plan=True, no_color=IsFlagged)
output_result = self.app.output(json=IsFlagged, no_color=IsFlagged)
output_var = {
output_var_key: output_result[output_var_key]["value"]
for output_var_key in output_result
}
return output_var
def run_terraform_destroy(self, config: Terrascript):
with self.terraform_workspace() as tw_dir:
self.write_terraform_config(config, tw_dir)
destroy_result = self.app.destroy(tw_dir)
return destroy_result
@classmethod
def gen_digital_ocean_config(cls,
config_data: dict,
token: str,
public_key: str = None):
do_config = Terrascript()
do_provider = provider.digitalocean(token=token)
do_droplet_resource = resource.digitalocean_droplet(
"server",
image=config_data["os_code"],
name=config_data["hostname"],
region=config_data["region_code"],
size=config_data["plan_code"],
ssh_keys=config_data["ssh_keys"]
if config_data.get("ssh_keys") else [],
)
if public_key:
digitalocean_ssh_key = resource.digitalocean_ssh_key(
"digitalocean_ssh_key",
name="default",
public_key=public_key,
)
do_droplet_resource["ssh_keys"] += [
"${digitalocean_ssh_key.digitalocean_ssh_key.id}"
]
do_config += digitalocean_ssh_key
do_output_ip = Output(
"ip", value="${digitalocean_droplet.server.ipv4_address}")
do_output_id = Output("server_id",
value="${digitalocean_droplet.server.id}")
do_config += do_provider
do_config += do_droplet_resource
do_config += do_output_ip
do_config += do_output_id
return do_config
@classmethod
def gen_vultr_config(cls,
config_data: dict,
token: str,
public_key: str = None):
vultr_config = Terrascript()
vultr_provider = cls.vultr(api_key=token,
rate_limit=700,
retry_limit=3)
vultr_server = cls.vultr_server(
"server",
plan_id=config_data["plan_code"],
region_id=config_data["region_code"],
os_id=config_data["os_code"],
hostname=config_data["hostname"],
ssh_key_ids=config_data["ssh_keys"]
if config_data.get("ssh_keys") else [],
)
vultr_output_ip = Output("ip", value="${vultr_server.server.main_ip}")
vultr_output_id = Output("server_id",
value="${vultr_server.server.id}")
if public_key:
vultr_ssh_key = cls.vultr_ssh_key("vultr_ssh_key",
name="default_key",
ssh_key=public_key)
vultr_server["ssh_key_ids"] += [
"${vultr_ssh_key.vultr_ssh_key.id}"
]
vultr_config += vultr_ssh_key
vultr_config += vultr_provider
vultr_config += vultr_server
vultr_config += vultr_output_ip
vultr_config += vultr_output_id
return vultr_config
@classmethod
def gen_tencent_cloud_config(
cls,
config_data: dict,
token: str,
public_key_name: str = None,
secret_id: str = None,
):
tencent_cloud_config = Terrascript()
tencent_terraform = terraform(
**{
"required_providers": {
"tencentcloud": {
"source": "terraform-providers/tencentcloud",
"version": "~> 1.40.3",
},
}
})
tencent_cloud_provider = cls.tencentcloud(
secret_id=secret_id,
secret_key=token,
region=config_data["region_code"],
)
tencent_zone = cls.tencentcloud_availability_zones("default")
tencent_security_group = cls.tencentcloud_security_group(
"default", name="all-open", description="open all ports")
tencent_security_group_rule = cls.tencentcloud_security_group_lite_rule(
"rule",
security_group_id="${tencentcloud_security_group.default.id}",
ingress=[
"ACCEPT#10.0.0.0/8#ALL#ALL",
],
egress=[
"ACCEPT#10.0.0.0/8#ALL#ALL",
],
)
tencent_cloud_server = cls.tencentcloud_instance(
"server",
instance_name=config_data["hostname"],
availability_zone=
"${data.tencentcloud_availability_zones.default.zones.0.name}",
image_id=config_data["os_code"],
instance_type=config_data["plan_code"],
disable_monitor_service=True,
disable_security_service=True,
allocate_public_ip=True,
internet_max_bandwidth_out=5,
instance_charge_type="POSTPAID_BY_HOUR",
internet_charge_type="TRAFFIC_POSTPAID_BY_HOUR",
system_disk_type="CLOUD_SSD",
count=1,
)
tencent_output_ip = Output(
"ip", value="${tencentcloud_instance.server.0.public_ip}")
tencent_output_id = Output(
"server_id", value="${tencentcloud_instance.server.0.id}")
if public_key_name:
tencent_cloud_server["key_name"] = public_key_name
tencent_cloud_config += tencent_terraform
tencent_cloud_config += tencent_cloud_provider
tencent_cloud_config += tencent_zone
tencent_cloud_config += tencent_security_group
tencent_cloud_config += tencent_security_group_rule
tencent_cloud_config += tencent_cloud_server
tencent_cloud_config += tencent_output_ip
tencent_cloud_config += tencent_output_id
return tencent_cloud_config
@classmethod
def gen_ali_cloud_config(
cls,
config_data: dict,
token: str,
ssh_key_name: str = None,
access_key: str = None,
security_groups: List[str] = [],
):
ali_cloud_config = Terrascript()
ali_cloud_provider = cls.alicloud(
access_key=access_key,
secret_key=token,
region=config_data["region_code"],
)
ali_zone = cls.alicloud_zones(
"default",
available_disk_category="cloud_efficiency",
available_resource_creation="Instance",
)
ali_vpc = cls.alicloud_vpc(
"vpc",
cidr_block="172.16.0.0/12",
)
ali_vswitch = cls.alicloud_vswitch(
"vswitch",
vpc_id="${alicloud_vpc.vpc.id}",
cidr_block="172.16.0.0/29",
availability_zone="${data.alicloud_zones.default.zones.0.id}",
)
ali_security_group = cls.alicloud_security_group(
"group",
name="all-open",
vpc_id="${alicloud_vpc.vpc.id}",
description="open all ports",
inner_access_policy="Accept",
)
ali_internet_security_group_rule = cls.alicloud_security_group_rule(
"internet",
# nic_type="internet",
security_group_id="${alicloud_security_group.group.id}",
type="ingress",
port_range="-1/-1",
cidr_ip="0.0.0.0/0",
ip_protocol="all",
policy="accept",
)
ali_intranet_security_group_rule = cls.alicloud_security_group_rule(
"intranet",
# nic_type="intranet",
security_group_id="${alicloud_security_group.group.id}",
port_range="-1/-1",
type="egress",
cidr_ip="0.0.0.0/0",
ip_protocol="all",
policy="accept",
priority=1,
)
ali_cloud_server = cls.alicloud_instance(
"server",
instance_name=config_data["hostname"],
availability_zone="${data.alicloud_zones.default.zones.0.id}",
# security_groups=security_groups,
security_groups="${alicloud_security_group.group.*.id}",
vswitch_id="${alicloud_vswitch.vswitch.id}",
image_id=config_data["os_code"],
instance_type=config_data["plan_code"],
security_enhancement_strategy="Deactive",
instance_charge_type="PostPaid",
internet_charge_type="PayByTraffic",
internet_max_bandwidth_out=2,
)
ali_output_ip = Output("ip",
value="${alicloud_instance.server.public_ip}")
ali_output_id = Output("server_id",
value="${alicloud_instance.server.id}")
if ssh_key_name:
ali_cloud_server["key_name"] = ssh_key_name
ali_cloud_config += ali_cloud_provider
ali_cloud_config += ali_zone
ali_cloud_config += ali_vpc
ali_cloud_config += ali_vswitch
ali_cloud_config += ali_security_group
ali_cloud_config += ali_internet_security_group_rule
ali_cloud_config += ali_intranet_security_group_rule
ali_cloud_config += ali_cloud_server
ali_cloud_config += ali_output_ip
ali_cloud_config += ali_output_id
return ali_cloud_config
@classmethod
def add_ssh_key_config(cls, public_key: str):
return provisioner(
"remote-exec",
provisioner=provisioner(
"remote-exec",
inline=[
"mkdir -p ~/.ssh",
f"{public_key} >> ~/.ssh/authorized_keys"
],
),
)
@classmethod
def gen_ssh_conn_config(
cls,
*,
ssh_user: str = DEFAULT_SSH_USER,
ssh_private_key: str,
ssh_host: str,
ssh_port: int = DEFAULT_SSH_PORT,
) -> dict:
# see more in https://www.terraform.io/docs/provisioners/connection.html
return {
"type": "ssh",
"user": ssh_user,
"private_key": ssh_private_key,
"host": ssh_host,
"port": ssh_port,
"timeout": "30s",
}
@classmethod
def gen_site_docker_deploy_config(
cls,
*,
docker_host: str = DEFAULT_DOCKER_HOST,
site_name: str = None,
template_tar_bytes: bytes = None,
script: str = None,
ssh_user: str = DEFAULT_SSH_USER,
ssh_private_key: str,
ssh_host: str,
ssh_port: int = DEFAULT_SSH_PORT,
):
config = Terrascript()
docker_provider = provider.docker(
host=docker_host,
connection=cls.gen_ssh_conn_config(
ssh_user=ssh_user,
ssh_private_key=ssh_private_key,
ssh_host=ssh_host,
ssh_port=ssh_port,
),
)
docker_image_resource = resource.docker_image(
"nginx_image",
name=cls.DEFAULT_NGINX_DOCKER_IMAGE,
)
docker_container_resource = resource.docker_container(
"nginx_container",
name=f"{site_name}-container-${{random_pet.docker_pet_name.id}}",
image="${docker_image.nginx_image.latest}",
restart="always",
start=True,
ports={"internal": 80},
upload=[],
)
docker_name_resource = resource.random_pet(
"docker_pet_name",
length=1,
)
if template_tar_bytes:
template_tar_file = (
f"{site_name}-tar-${{random_pet.docker_pet_name.id}}.tar.gz", )
template_tar_file_content = base64.b64encode(
template_tar_bytes).decode("utf8")
template_tar_path = (
f"{cls.DEFAULT_NGINX_DOCKER_CONTAINER_HTML_PATH}/${template_tar_file}"
)
# self.upload_file(
# content='conf/myapp.conf',
# destination=f"{self.DEFAULT_UPLOAD_PATH}/${template_tar_file}",
# ssh_user=ssh_user,
# ssh_private_key=ssh_private_key,
# ssh_host=ssh_host,
# ssh_port=ssh_port
# )
docker_container_resource["upload"].append({
"content_base64":
template_tar_file_content,
"file":
template_tar_path
})
if script:
entrypoint_sh_content = TemplateRender().render(
cls.DEFAULT_NGINX_DOCKER_ENTRYPOINT_PATH,
init_script_path=cls.DEFAULT_DOCKER_ENTRYPOINT_PATH,
html_path=cls.DEFAULT_NGINX_DOCKER_CONTAINER_HTML_PATH,
)
docker_container_resource["upload"].append({
"content":
entrypoint_sh_content,
"file":
cls.DEFAULT_DOCKER_ENTRYPOINT_PATH,
})
config += docker_provider
config += docker_image_resource
config += docker_container_resource
config += docker_name_resource
return config
def remote_exec(
self,
*,
ssh_user: str = DEFAULT_SSH_USER,
ssh_private_key: str,
ssh_host: str,
ssh_port: int = DEFAULT_SSH_PORT,
):
exec_config = Terrascript()
ssh_conn = self.gen_ssh_conn_config(
ssh_user=ssh_user,
ssh_private_key=ssh_private_key,
ssh_host=ssh_host,
ssh_port=ssh_port,
)
exec_resource = self.null_resource(
"remote-exec",
provisioner=provisioner("remote-exec",
inline=["ls -la"],
connection=ssh_conn),
)
exec_config += exec_resource
return exec_config
def upload_file(
self,
content: str,
*,
destination: str = DEFAULT_UPLOAD_PATH,
ssh_user: str = DEFAULT_SSH_USER,
ssh_private_key: str,
ssh_host: str,
ssh_port: int = DEFAULT_SSH_PORT,
):
upload_config = Terrascript()
ssh_conn = self.gen_ssh_conn_config(
ssh_user=ssh_user,
ssh_private_key=ssh_private_key,
ssh_host=ssh_host,
ssh_port=ssh_port,
)
file_resource = self.null_resource(
"upload_file_resource",
provisioner=provisioner(
self.TERRAFORM_RESOURCE_FILE,
content=content,
destination=destination,
connection=ssh_conn,
),
)
upload_config += file_resource
return upload_config
