def main(username, password):
#get_default_cli().invoke(['login', "--use-device-code"], out_file=sys.stdout)
#
# Destroy Infrastructure
#
tfstate_file = 'terraform.tfstate'
tfstate_files = ['terraform.tfstate', 'terraform.tfstate.backup']
fpath = './WebInDeploy/' + tfstate_file
if os.path.isfile(fpath):
tf = Terraform(working_dir='./WebInDeploy')
rg_name = tf.output('RESOURCE_GROUP_NAME')
#rg_name1 = tf.output('Resource_Group')
delete_rg_cmd = 'group delete --name ' + rg_name + ' --yes'
az_cli(delete_rg_cmd)
#
# Delete state files WebInDeploy
#
delete_state_files('./WebInDeploy/', tfstate_files)
fpath = './WebInBootstrap/' + tfstate_file
if os.path.isfile(fpath):
tf = Terraform(working_dir='./WebInBootstrap')
rg_name1 = tf.output('Resource_Group')
delete_rg_cmd = 'group delete --name ' + rg_name1 + ' --yes'
az_cli(delete_rg_cmd)
#
# Delete state files WebInBootstrap
#
delete_state_files('./WebInBootstrap/', tfstate_files)
#
# Delete state files WebInFWConf
#
delete_state_files('./WebInFWConf/', tfstate_files)
class TerraformProvider():
def __init__(self, configuration, terraform_workspace):
log.info("Preparing terraform deployment")
log.debug("Using workspace: {}".format(terraform_workspace))
self._backend_provider = get_backend_provider(configuration,
terraform_workspace)
self._controller = Terraform(
working_dir=terraform_workspace,
variables=configuration["terraform"]["parameters"])
self._controller.init(capture_output=False, force_copy=IsFlagged)
@stage("Terraform deploy")
def deploy(self):
log.info("Deploying terraform infrastructure")
self._backend_provider.init_remote_backend()
self._controller.apply(capture_output=False, skip_plan=True)
output = self._controller.output()
artifact.create("terraform_output", content=json.dumps(output))
def destroy(self):
log.info("Destroying terraform infrastructure")
self._controller.destroy(capture_output=False)
self._backend_provider.remove_remote_backend()
def terraform_apply(env_data, tf: Terraform):
retry_count = 0
return_code = 0
while retry_count < 5:
logger.debug("Try {}".format(retry_count))
return_code, stdout, stderr = tf.apply(skip_plan=True,
var_file=env_data,
capture_output=True)
logger.debug('Terraform apply return code is {}'.format(return_code))
logger.debug('Terraform apply stdout is {}'.format(stdout))
logger.debug("Terraform apply stderr is {}".format(stderr))
retry_count += 1
if return_code == 0:
break
time.sleep(30)
if return_code == 0:
show_return_code, tf_state, stdout = tf.show(json=True)
logger.debug(
'Terraform show return code is {}'.format(show_return_code))
logger.debug('Terraform show stdout is {}'.format(stdout))
tf_outputs = tf.output()
for output_value in tf_outputs:
logger.debug('Terraform output value is {}'.format(output_value))
else:
# TODO get output for errors
tf_state = {}
tf_outputs = {}
traceback.print_stack()
return {
"tf_return_code": return_code,
"tf_outputs": tf_outputs,
"tf_state": tf_state
}
class Terraform:
def __init__(self, init):
from python_terraform import Terraform as PythonTerraform
self.terraform = PythonTerraform(working_dir='terraform')
Path(self.working_dir).mkdir(exist_ok=True)
if init:
return_code, _, err = self.terraform.init(
dir_or_plan=self.working_dir)
if return_code != 0:
raise CwsCommandError(err)
@property
def working_dir(self):
return self.terraform.working_dir
def init(self):
return_code, _, err = self.terraform.init()
if return_code != 0:
raise CwsCommandError(err)
def apply(self, workspace, targets):
self.select_workspace(workspace)
return_code, _, err = self.terraform.apply(target=targets,
skip_plan=True,
input=False,
raise_on_error=False,
parallelism=1)
if return_code != 0:
raise CwsCommandError(err)
def destroy(self, workspace, targets):
self.select_workspace(workspace)
return_code, _, err = self.terraform.destroy(target=targets)
if return_code != 0:
raise CwsCommandError(err)
def output(self):
self.select_workspace("default")
values = self.terraform.output(capture_output=True)
return {key: value['value']
for key, value in values.items()} if values else "{}"
def workspace_list(self):
self.select_workspace("default")
return_code, out, err = self.terraform.cmd('workspace', 'list')
if return_code != 0:
raise CwsCommandError(err)
values = out[1:].translate(str.maketrans('', '', ' \t\r')).split('\n')
return filter(None, values)
def select_workspace(self, workspace):
return_code, out, err = self.terraform.workspace('select', workspace)
if workspace != 'default' and return_code != 0:
_, out, err = self.terraform.workspace('new',
workspace,
raise_on_error=True)
if not (Path(self.working_dir) / '.terraform').exists():
self.terraform.init(input=False, raise_on_error=True)
class TerraformBaseAction(Action):
def __init__(self, config):
"""Creates a new BaseAction given a StackStorm config object (kwargs works too)
Also stores the Terraform class from python_terraform in a class variable
:param config: StackStorm configuration object for the pack
:returns: a new BaseAction
"""
super(TerraformBaseAction, self).__init__(config)
self.terraform = Terraform()
def check_result(self,
return_code,
stdout,
stderr,
return_output=False,
valid_return_codes=None):
"""Check the return code from the terraform action and return the output with
the error message (if there is one)
:param return_code: return code from the Terraform command that was run
:param stdout: output from the Terraform command that was run
:param stderr: error message (if any) from the Terraform command
:param return_output: whether to return the results of `terraform output` command
:param valid_return_codes: list of valid return codes
:returns: success flag for the st2 action and output from the Terraform command
"""
if valid_return_codes is None:
valid_return_codes = [0]
if return_output:
output = None
if return_code in valid_return_codes:
output = self.terraform.output(state=self.terraform.state)
else:
output = TerraformBaseAction.concat_std_output(stdout, stderr)
# Capture success status vs valid return codes and return result
success = (return_code in valid_return_codes)
return success, output
@staticmethod
def concat_std_output(stdout, stderr):
"""Combines stdout and stderr from terraform execution, one or the other
can be None
:param stdout: results from stdout
:param stderr: results from stderr
:returns: string concatenation
"""
output = None
if stderr or stdout:
output = ''
if stdout:
output += stdout
if stderr:
if stdout:
output += "\n"
output += stderr
return output
def terraform_output(setup_terraform,
terraform_bin_path) -> Dict[str, Dict[str, str]]:
tf = Terraform(working_dir=full_path,
terraform_bin_path=terraform_bin_path)
outputs = tf.output()
if outputs is not None:
return outputs
raise Exception("Cannot retrieve the outputs")
def main(username, password):
#get_default_cli().invoke(['login', "--use-device-code"], out_file=sys.stdout)
#
# Destroy Infrastructure
#
tfstate_file = 'terraform.tfstate'
fpath = './WebInDeploy/' + tfstate_file
if os.path.isfile(fpath):
tf = Terraform(working_dir='./WebInDeploy')
rg_name = tf.output('RG_Name')
rg_name1 = tf.output('Attacker_RG_Name')
delete_rg_cmd = 'group delete --name ' + rg_name + ' --yes'
az_cli(delete_rg_cmd)
delete_file(fpath)
fpath = './WebInBootstrap/' + tfstate_file
if os.path.isfile(fpath):
delete_rg_cmd = 'group delete --name ' + rg_name1 + ' --yes'
az_cli(delete_rg_cmd)
delete_file(fpath)
def test_override_default(self, folder, variables):
tf = Terraform(working_dir=current_path, variables=variables)
tf.init(folder)
ret, out, err = tf.apply(
folder,
var={"test_var": "test2"},
no_color=IsNotFlagged,
)
out = out.replace("\n", "")
assert "\x1b[0m\x1b[1m\x1b[32mApply" in out
out = tf.output("test_output")
assert "test2" in out
def test_output(self, caplog: LogCaptureFixture, output_all: bool):
expected_value = "test"
required_output = "test_output"
with caplog.at_level(logging.INFO):
tf = Terraform(working_dir=current_path,
variables={"test_var": expected_value})
tf.init("var_to_output")
tf.apply("var_to_output")
params = tuple() if output_all else (required_output, )
result = tf.output(*params)
if output_all:
assert result[required_output]["value"] == expected_value
else:
assert result == expected_value
assert expected_value in caplog.messages[-1]
def vm_docker_deploy_old(config):
# This script should deploy the instance and return the output/logs after the test has finished
file_dir = os.path.dirname(os.path.realpath(__file__))
provider = config["params"]["Provider"][0]
### Check that a selection was made
if config["selection"]["instance"] is None:
config["logs"] = None
return config
### Setup terraform objects
instance_wkdir = file_dir + "/instance_deploy/" + provider
instance_tf = Terraform(working_dir=instance_wkdir)
docker_tf = Terraform(file_dir + "/docker_deploy")
tfstate_path = config["base_dir"] + '/tf_states/' + str(config["job_id"])
tfvars = config["base_dir"] + "/tfvars.tfvars"
## ALSO DIRECT TO A VARS.TF IN THE BASE_DIR
instance_tf.init(
backend_config={'path': tfstate_path + '/terraform.tfstate'})
instance_tf.apply(var_file=tfvars,
lock=False,
var={'instance_type': config["selection"]["instance"]},
skip_plan=True)
docker_tf.init(backend_config={
'path': tfstate_path + '/docker_tfstate/terraform.tfstate'
})
docker_tf.apply(var_file=tfvars,
lock=False,
var={'tfstate_path': tfstate_path},
skip_plan=True)
docker_tf.init(backend_config={
'path': tfstate_path + '/docker_tfstate/terraform.tfstate'
})
logs = docker_tf.output()
config["logs"] = logs
docker_tf.init(backend_config={
'path': tfstate_path + '/docker_tfstate/terraform.tfstate'
})
docker_tf.destroy(auto_approve=True)
instance_tf.init(
backend_config={'path': tfstate_path + '/terraform.tfstate'})
instance_tf.destroy(auto_approve=True)
return config
def apply_tf(working_dir, vars, description):
"""
Handles terraform operations and returns variables in outputs.tf as a dict.
:param working_dir: Directory that contains the tf files
:param vars: Additional variables passed in to override defaults equivalent to -var
:param description: Description of the deployment for logging purposes
:return: return_code - 0 for success or other for failure
outputs - Dictionary of the terraform outputs defined in the outputs.tf file
"""
# Set run_plan to TRUE is you wish to run terraform plan before apply
run_plan = False
kwargs = {"auto-approve": True}
# Class Terraform uses subprocess and setting capture_output to True will capture output
capture_output = kwargs.pop('capture_output', False)
if capture_output is True:
stderr = subprocess.PIPE
stdout = subprocess.PIPE
else:
# if capture output is False, then everything will essentially go to stdout and stderrf
stderr = sys.stderr
stdout = sys.stdout
start_time = time.asctime()
print('Starting Deployment at {}\n'.format(start_time))
# Create Bootstrap
tf = Terraform(working_dir=working_dir)
tf.cmd('init')
if run_plan:
# print('Calling tf.plan')
tf.plan(capture_output=False)
return_code, stdout, stderr = tf.apply(vars=vars,
capture_output=capture_output,
skip_plan=True,
**kwargs)
outputs = tf.output()
logger.debug('Got Return code {} for deployment of {}'.format(
return_code, description))
return (return_code, outputs)
def create(pathToInf):
# Initalize a terraform object
# Make the working directory the openstack directory
terra = Terraform(pathToInf)
# Apply the IAC in the openstack directory and skip the planning Stage
return_code, stdout, stderr = terra.apply(skip_plan=True)
# Get the outputs from the apply
outputs = terra.output()
print(stderr)
print(stdout)
print(outputs)
# Return the outputs
return outputs, return_code
def build_controller():
bucket_backend = 'bucket=' + str(s3_bucket)
table_backend = 'dynamodb_table=' + str(s3_bucket)
backend_configs = [bucket_backend, table_backend]
tf_base = Terraform(workdir_ctrl)
tf_base.init(backend_config=backend_configs)
#tf_base.plan(capture_output=False, var_file=tfvars)
return_code = tf_base.apply(skip_plan=True,
capture_output=False,
var_file=tfvars)
controller_outputs = tf_base.output(capture_output=True)
if return_code[0] == 1:
print("Something went wrong!")
sys.exit(1)
else:
print("All good!")
pass
return controller_outputs
def orchestration():
# Initalize a terraform object
# Make the working directory the openstack directory
terra = Terraform("./openstack")
# Apply the IAC in the openstack directory and skip the planning Stage
return_code, stdout, stderr = terra.apply(skip_plan=True)
# Print the results
print(return_code)
print(stdout)
print(stderr)
# Get the outputs from the apply
outputs = terra.output()
# Return the outputs
return outputs
print("confirmed password")
wait_until_channel_endswith(ssh_channel, b'# ')
ssh_channel.send(f'commit\n')
print("sent commit")
# longer timeout of 60s to cater to commit time
wait_until_channel_endswith(ssh_channel, b'# ', 60)
print("changed admin password")
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
working_dir = "./"
tf = Terraform(working_dir=working_dir)
outputs = tf.output()
fw1_mgmt = outputs['FW-1-Mgmt']['value']
fw2_mgmt = outputs['FW-2-Mgmt']['value']
username = "******"
new_password = outputs['password_new']['value']
ssh_key_path = outputs['ssh_key_path']['value']
time.sleep(600)
fw_init(fw1_mgmt, username, ssh_key_path, new_password)
fw_init(fw2_mgmt, username, ssh_key_path, new_password)
# Get new API Key
url = "https://%s/api/?type=keygen&user=%s&password=%s" % (fw1_mgmt, username,
new_password)
response = requests.get(url, verify=False)
def main(username, password, aws_access_key, aws_secret_key, aws_region,
ec2_key_pair, bootstrap_bucket):
username = username
password = password
aws_access_key = aws_access_key
aws_secret_key = aws_secret_key
aws_region = aws_region
ec2_key_pair = ec2_key_pair
albDns = ''
nlbDns = ''
fwMgt = ''
default_vars = {
'aws_access_key': aws_access_key,
'aws_secret_key': aws_secret_key,
'aws_region': aws_region
}
WebInDeploy_vars = {
'aws_access_key': aws_access_key,
'aws_secret_key': aws_secret_key,
'aws_region': aws_region,
'ServerKeyName': ec2_key_pair,
'bootstrap_s3bucket': bootstrap_bucket
}
waf_conf_vars = {
'aws_access_key': aws_access_key,
'aws_secret_key': aws_secret_key,
'aws_region': aws_region,
'ServerKeyName': ec2_key_pair,
'alb_arn': albDns,
'nlb-dns': nlbDns
}
WebInFWConf_vars = {
'aws_access_key': aws_access_key,
'aws_secret_key': aws_secret_key,
'aws_region': aws_region,
'ServerKeyName': ec2_key_pair,
'mgt-ipaddress-fw1': fwMgt,
'nlb-dns': nlbDns,
'username': username,
'password': password
}
# Set run_plan to TRUE is you wish to run terraform plan before apply
run_plan = False
kwargs = {"auto-approve": True}
# Class Terraform uses subprocess and setting capture_output to True will capture output
capture_output = kwargs.pop('capture_output', False)
if capture_output is True:
stderr = subprocess.PIPE
stdout = subprocess.PIPE
else:
# if capture output is False, then everything will essentially go to stdout and stderrf
stderr = sys.stderr
stdout = sys.stdout
start_time = time.asctime()
print(f'Starting Deployment at {start_time}\n')
# Build Infrastructure
tf = Terraform(working_dir='./WebInDeploy')
tf.cmd('init')
if run_plan:
# print('Calling tf.plan')
tf.plan(capture_output=False, var=WebInDeploy_vars)
return_code1, stdout, stderr = tf.apply(var=WebInDeploy_vars,
capture_output=capture_output,
skip_plan=True,
**kwargs)
web_in_deploy_output = tf.output()
logger.debug(
'Got Return code for deploy WebInDeploy {}'.format(return_code1))
# update_status('web_in_deploy_stdout', stdout)
update_status('web_in_deploy_output', web_in_deploy_output)
if return_code1 != 0:
logger.info("WebInDeploy failed")
update_status('web_in_deploy_status', 'error')
update_status('web_in_deploy_stderr', stderr)
print(json.dumps(status_output))
exit(1)
else:
update_status('web_in_deploy_status', 'success')
albDns = tf.output('ALB-DNS')
fwMgt = tf.output('MGT-IP-FW-1')
nlbDns = tf.output('NLB-DNS')
fwMgtIP = fwMgt
WebInFWConf_vars['mgt-ipaddress-fw1'] = fwMgt
WebInFWConf_vars['nlb-dns'] = nlbDns
WebInDeploy_vars['alb_dns'] = albDns
WebInDeploy_vars['nlb-dns'] = nlbDns
#
# Apply WAF Rules
#
tf = Terraform(working_dir='./waf_conf')
tf.cmd('init')
kwargs = {"auto-approve": True}
logger.info("Applying WAF config to App LB")
if run_plan:
tf.plan(capture_output=capture_output, var=vars, **kwargs)
return_code3, stdout, stderr = tf.apply(capture_output=capture_output,
skip_plan=True,
var=waf_conf_vars,
**kwargs)
waf_conf_out = tf.output()
update_status('waf_conf_output', waf_conf_out)
# update_status('waf_conf_stdout', stdout)
# update_status('waf_conf_stderr', stderr)
logger.debug('Got Return code to deploy waf_conf {}'.format(return_code3))
if return_code3 != 0:
logger.info("waf_conf failed")
update_status('waf_conf_status', 'error')
update_status('waf_conf_stderr', stderr)
print(json.dumps(status_output))
exit(1)
else:
update_status('waf_conf_status', 'success')
logger.info("Got these values from output of first run\n\n")
logger.info("ALB address is {}".format(albDns))
logger.info("nlb address is {}".format(nlbDns))
logger.info("Firewall Mgt address is {}".format(fwMgt))
#
# Check firewall is up and running
# #
api_key = getApiKey(fwMgtIP, username, password)
while True:
err = getFirewallStatus(fwMgtIP, api_key)
if err == 'cmd_error':
logger.info("Command error from fw ")
elif err == 'no':
logger.info("FW is not up...yet")
# print("FW is not up...yet")
time.sleep(60)
continue
elif err == 'almost':
logger.info("MGT up waiting for dataplane")
time.sleep(20)
continue
elif err == 'yes':
logger.info("FW is up")
break
logger.debug(
'Giving the FW another 10 seconds to fully come up to avoid race conditions'
)
time.sleep(10)
fw = firewall.Firewall(hostname=fwMgtIP,
api_username=username,
api_password=password)
logger.info("Updating firewall with latest content pack")
update_fw(fwMgtIP, api_key)
updateHandle = updater.ContentUpdater(fw)
# updateHandle.download(fw)
# logger.info("Waiting 3 minutes for content update to download")
# time.sleep(210)
# updateHandle.install()
#
# Configure Firewall
#
tf = Terraform(working_dir='./WebInFWConf')
tf.cmd('init')
kwargs = {"auto-approve": True}
logger.info("Applying addtional config to firewall")
WebInFWConf_vars['mgt-ipaddress-fw1'] = fwMgt
if run_plan:
tf.plan(capture_output=capture_output, var=WebInFWConf_vars)
# update initial vars with generated fwMgt ip
return_code2, stdout, stderr = tf.apply(capture_output=capture_output,
skip_plan=True,
var=WebInFWConf_vars,
**kwargs)
web_in_fw_conf_out = tf.output()
update_status('web_in_fw_conf_output', web_in_fw_conf_out)
# update_status('web_in_fw_conf_stdout', stdout)
logger.debug(
'Got Return code for deploy WebInFwConf {}'.format(return_code2))
if return_code2 != 0:
logger.error("WebFWConfy failed")
update_status('web_in_fw_conf_status', 'error')
update_status('web_in_fw_conf_stderr', stderr)
print(json.dumps(status_output))
exit(1)
else:
update_status('web_in_fw_conf_status', 'success')
logger.info("Commit changes to firewall")
fw.commit()
logger.info("waiting for commit")
time.sleep(60)
logger.info("waiting for commit")
#
# Check Jenkins
#
logger.info('Checking if Jenkins Server is ready')
# FIXME - add outputs for all 3 dirs
res = getServerStatus(albDns)
if res == 'server_up':
logger.info('Jenkins Server is ready')
logger.info('\n\n ### Deployment Complete ###')
logger.info(
'\n\n Connect to Jenkins Server at http://{}'.format(albDns))
else:
logger.info('Jenkins Server is down')
logger.info('\n\n ### Deployment Complete ###')
# dump out status to stdout
print(json.dumps(status_output))
def main(fwUsername, fwPasswd):
albDns = ''
nlbDns = ''
fwMgt = ''
# Set run_plan to TRUE is you wish to run terraform plan before apply
run_plan = False
deployment_status = {}
kwargs = {"auto-approve": True}
# Class Terraform uses subprocess and setting capture_output to True will capture output
# capture_output = kwargs.pop('capture_output', True)
#
# if capture_output is True:
# stderr = subprocess.PIPE
# stdout = subprocess.PIPE
# else:
# stderr = sys.stderr
# stdout = sys.stdout
#
# Build Infrastructure
#
tf = Terraform(working_dir='./WebInDeploy')
tf.cmd('init')
if run_plan:
print('Calling tf.plan')
tf.plan(capture_output=False)
return_code1, stdout, stderr = tf.apply(capture_output=False,
skip_plan=True,
**kwargs)
#return_code1 =0
print('Got return code {}'.format(return_code1))
if return_code1 != 0:
logger.info("WebInDeploy failed")
deployment_status = {'WebInDeploy': 'Fail'}
write_status_file(deployment_status)
exit()
else:
deployment_status = {'WebInDeploy': 'Success'}
write_status_file(deployment_status)
albDns = tf.output('ALB-DNS')
fwMgt = tf.output('MGT-IP-FW-1')
nlbDns = tf.output('NLB-DNS')
# fwUsername = "******"
# fwPasswd = "PaloAlt0!123!!"
fw_trust_ip = fwMgt
#
# Apply WAF Rules
#
tf = Terraform(working_dir='./waf_conf')
tf.cmd('init')
kwargs = {"auto-approve": True}
logger.info("Applying WAF config to App LB")
if run_plan:
tf.plan(capture_output=False, var={'alb_arn': nlbDns}, **kwargs)
return_code3, stdout, stderr = tf.apply(capture_output=False,
skip_plan=True,
var={
'alb_arn': nlbDns,
'int-nlb-fqdn': nlbDns
},
**kwargs)
if return_code3 != 0:
logger.info("waf_conf failed")
deployment_status.update({'waf_conf': 'Fail'})
write_status_file(deployment_status)
exit()
else:
deployment_status.update({'waf_conf': 'Success'})
write_status_file(deployment_status)
logger.info("Got these values from output of first run\n\n")
logger.info("ALB address is {}".format(albDns))
logger.info("nlb address is {}".format(nlbDns))
logger.info("Firewall Mgt address is {}".format(fwMgt))
#
# Check firewall is up and running
#
class FWNotUpException(Exception):
pass
err = 'no'
api_key = ''
api_key = getApiKey(fw_trust_ip, fwUsername, fwPasswd)
while True:
err = getFirewallStatus(fw_trust_ip, api_key)
if err == 'cmd_error':
logger.info("Command error from fw ")
#raise FWNotUpException('FW is not up! Request Timeout')
elif err == 'no':
logger.info("FW is not up...yet")
print("FW is not up...yet")
time.sleep(60)
continue
#raise FWNotUpException('FW is not up!')
elif err == 'almost':
logger.info("MGT up waiting for dataplane")
time.sleep(20)
continue
elif err == 'yes':
logger.info("[INFO]: FW is up")
break
fw = firewall.Firewall(hostname=fw_trust_ip,
api_username=fwUsername,
api_password=fwPasswd)
logger.info("Updating firewall with latest content pack")
updateHandle = updater.ContentUpdater(fw)
updateHandle.download()
logger.info("Waiting 3 minutes for content update to download")
time.sleep(210)
updateHandle.install()
#
# Configure Firewall
#
tf = Terraform(working_dir='./WebInFWConf')
tf.cmd('init')
kwargs = {"auto-approve": True}
logger.info("Applying addtional config to firewall")
if run_plan:
tf.plan(capture_output=False,
var={
'mgt-ipaddress-fw1': fwMgt,
'int-nlb-fqdn': nlbDns
})
return_code2, stdout, stderr = tf.apply(capture_output=False,
skip_plan=True,
var={
'mgt-ipaddress-fw1': fwMgt,
'nlb-dns': nlbDns,
'aws_access_key':
aws_access_key,
'aws_secret_key':
aws_secret_key
},
**kwargs)
#return_code2 = 0
if return_code2 != 0:
logger.info("WebFWConfy failed")
deployment_status.update({'WebFWConfy': 'Fail'})
write_status_file(deployment_status)
exit()
else:
deployment_status.update({'WebFWConf': 'Success'})
write_status_file(deployment_status)
logger.info("Commit changes to firewall")
fw.commit()
logger.info('Checking if Jenkins Server is ready')
# tf = Terraform(working_dir='./WebInDeploy')
# albDns = tf.output('ALB-DNS')
count = 0
max_tries = 3
while True:
if count < max_tries:
res = getServerStatus(albDns)
if res == 'server_down':
count = count + 1
time.sleep(2)
continue
elif res == 'server_up':
break
else:
break
logger.info('Jenkins Server is ready')
logger.info('\n\n ### Deployment Complete ###')
logger.info('\n\n Connect to Jenkins Server at http://{}'.format(albDns))
import pytest
import json
import os
from python_terraform import Terraform
tf = Terraform(working_dir='infrastructure/')
stack = tf.output()
@pytest.fixture
def reader_lambda_runtime():
os.environ['QUEUE_URL'] = stack['reader_queue_url']['value']
yield
del os.environ['QUEUE_URL']
@pytest.fixture
def writer_lambda_runtime():
os.environ['DYNAMODB_TABLE_NAME'] = stack['dynamodb_table']['value']
yield
del os.environ['DYNAMODB_TABLE_NAME']
@pytest.fixture
def model_lambda_runtime():
os.environ["OUTPUT_QUEUE_URL"] = stack["writer_queue_url"]["value"]
os.environ[
"MODEL_PATH"] = "s3://kalemiller-model-artifacts/real-time-wine/model.joblib"
yield # TODO
del os.environ["OUTPUT_QUEUE_URL"]
def main(username, password, rg_name, azure_region):
username = username
password = password
WebInBootstrap_vars = {'RG_Name': rg_name, 'Azure_Region': azure_region}
WebInDeploy_vars = {
'Admin_Username': username,
'Admin_Password': password,
'Azure_Region': azure_region
}
WebInFWConf_vars = {'Admin_Username': username, 'Admin_Password': password}
# Set run_plan to TRUE is you wish to run terraform plan before apply
run_plan = False
kwargs = {"auto-approve": True}
# Class Terraform uses subprocess and setting capture_output to True will capture output
capture_output = kwargs.pop('capture_output', False)
if capture_output is True:
stderr = subprocess.PIPE
stdout = subprocess.PIPE
else:
# if capture output is False, then everything will essentially go to stdout and stderrf
stderr = sys.stderr
stdout = sys.stdout
start_time = time.asctime()
print(f'Starting Deployment at {start_time}\n')
# Create Bootstrap
tf = Terraform(working_dir='./WebInBootstrap')
tf.cmd('init')
if run_plan:
# print('Calling tf.plan')
tf.plan(capture_output=False)
return_code1, stdout, stderr = tf.apply(vars=WebInBootstrap_vars,
capture_output=capture_output,
skip_plan=True,
**kwargs)
resource_group = tf.output('Resource_Group')
bootstrap_bucket = tf.output('Bootstrap_Bucket')
storage_account_access_key = tf.output('Storage_Account_Access_Key')
web_in_bootstrap_output = tf.output()
logger.debug(
'Got Return code for deploy WebInDeploy {}'.format(return_code1))
update_status('web_in_deploy_stdout', stdout)
update_status('web_in_bootstrap_output', web_in_bootstrap_output)
if return_code1 != 0:
logger.info("WebInBootstrap failed")
update_status('web_in_bootstap_status', 'error')
update_status('web_in_bootstrap_stderr', stderr)
print(json.dumps(status_output))
exit(1)
else:
update_status('web_in_bootstrap_status', 'success')
share_prefix = 'jenkins-demo'
share_name = create_azure_fileshare(share_prefix, bootstrap_bucket,
storage_account_access_key)
WebInDeploy_vars.update(
{'Storage_Account_Access_Key': storage_account_access_key})
WebInDeploy_vars.update({'Bootstrap_Storage_Account': bootstrap_bucket})
WebInDeploy_vars.update({'RG_Name': resource_group})
WebInDeploy_vars.update({'Attack_RG_Name': resource_group})
WebInDeploy_vars.update({'Storage_Account_Fileshare': share_name})
# Build Infrastructure
tf = Terraform(working_dir='./WebInDeploy')
# print("vars {}".format(WebInDeploy_vars))
tf.cmd('init')
if run_plan:
# print('Calling tf.plan')
tf.plan(capture_output=False, var=WebInDeploy_vars)
return_code1, stdout, stderr = tf.apply(var=WebInDeploy_vars,
capture_output=capture_output,
skip_plan=True,
**kwargs)
web_in_deploy_output = tf.output()
logger.debug(
'Got Return code for deploy WebInDeploy {}'.format(return_code1))
update_status('web_in_deploy_stdout', stdout)
update_status('web_in_deploy_output', web_in_deploy_output)
if return_code1 != 0:
logger.info("WebInDeploy failed")
update_status('web_in_deploy_status', 'error')
update_status('web_in_deploy_stderr', stderr)
print(json.dumps(status_output))
exit(1)
else:
update_status('web_in_deploy_status', 'success')
albDns = tf.output('ALB-DNS')
fwMgt = tf.output('MGT-IP-FW-1')
nlbDns = tf.output('NLB-DNS')
fwMgtIP = tf.output('MGT-IP-FW-1')
logger.info("Got these values from output \n\n")
logger.info("AppGateway address is {}".format(albDns))
logger.info("Internal loadbalancer address is {}".format(nlbDns))
logger.info("Firewall Mgt address is {}".format(fwMgt))
#
# Check firewall is up and running
# #
api_key = getApiKey(fwMgtIP, username, password)
while True:
err = getFirewallStatus(fwMgtIP, api_key)
if err == 'cmd_error':
logger.info("Command error from fw ")
elif err == 'no':
logger.info("FW is not up...yet")
# print("FW is not up...yet")
time.sleep(60)
continue
elif err == 'almost':
logger.info("MGT up waiting for dataplane")
time.sleep(20)
continue
elif err == 'yes':
logger.info("FW is up")
break
logger.debug(
'Giving the FW another 10 seconds to fully come up to avoid race conditions'
)
time.sleep(10)
fw = firewall.Firewall(hostname=fwMgtIP,
api_username=username,
api_password=password)
logger.info("Updating firewall with latest content pack")
update_fw(fwMgtIP, api_key)
#
# Configure Firewall
#
WebInFWConf_vars.update({'FW_Mgmt_IP': fwMgtIP})
tf = Terraform(working_dir='./WebInFWConf')
tf.cmd('init')
kwargs = {"auto-approve": True}
logger.info("Applying addtional config to firewall")
WebInFWConf_vars['mgt-ipaddress-fw1'] = fwMgt
if run_plan:
tf.plan(capture_output=capture_output, var=WebInFWConf_vars)
# update initial vars with generated fwMgt ip
return_code2, stdout, stderr = tf.apply(capture_output=capture_output,
skip_plan=True,
var=WebInFWConf_vars,
**kwargs)
web_in_fw_conf_out = tf.output()
update_status('web_in_fw_conf_output', web_in_fw_conf_out)
# update_status('web_in_fw_conf_stdout', stdout)
logger.debug(
'Got Return code for deploy WebInFwConf {}'.format(return_code2))
if return_code2 != 0:
logger.error("WebInFWConf failed")
update_status('web_in_fw_conf_status', 'error')
update_status('web_in_fw_conf_stderr', stderr)
print(json.dumps(status_output))
exit(1)
else:
update_status('web_in_fw_conf_status', 'success')
logger.info("Commit changes to firewall")
fw.commit()
logger.info("waiting for commit")
time.sleep(60)
logger.info("waiting for commit")
#
# Check Jenkins
#
logger.info('Checking if Jenkins Server is ready')
# FIXME - add outputs for all 3 dirs
res = getServerStatus(albDns)
if res == 'server_up':
logger.info('Jenkins Server is ready')
logger.info('\n\n ### Deployment Complete ###')
logger.info(
'\n\n Connect to Jenkins Server at http://{}'.format(albDns))
else:
logger.info('Jenkins Server is down')
logger.info('\n\n ### Deployment Complete ###')
# dump out status to stdout
print(json.dumps(status_output))
def main(username, password):
username = username
password = password
WebInDeploy_vars = {'Admin_Username': username, 'Admin_Password': password}
WebInBootstrap_vars = {
'Admin_Username': username,
'Admin_Password': password
}
albDns = ''
nlbDns = ''
fwMgt = ''
# Set run_plan to TRUE is you wish to run terraform plan before apply
run_plan = False
deployment_status = {}
kwargs = {"auto-approve": True}
#
# Destroy Infrastructure
#
tf = Terraform(working_dir='./WebInDeploy')
rg_name = tf.output('RG_Name')
attack_rg_name = tf.output('Attacker_RG_Name')
logger.info('Got RG_Name {} and Attacker_RG_Name {}'.format(
rg_name, attack_rg_name))
WebInDeploy_vars.update({'RG_Name': rg_name})
WebInDeploy_vars.update({'Attack_RG_Name': attack_rg_name})
if run_plan:
print('Calling tf.plan')
tf.plan(capture_output=False)
return_code1, stdout, stderr = tf.cmd('destroy',
var=WebInDeploy_vars,
capture_output=False,
**kwargs)
# return_code1 =0
print('Got return code {}'.format(return_code1))
if return_code1 != 0:
logger.info("Failed to destroy build ")
exit()
else:
logger.info("Destroyed WebInDeploy ")
WebInBootstrap_vars.update({'RG_Name': rg_name})
WebInBootstrap_vars.update({'Attack_RG_Name': attack_rg_name})
tf = Terraform(working_dir='./WebInBootstrap')
if run_plan:
print('Calling tf.plan')
tf.plan(capture_output=False)
return_code1, stdout, stderr = tf.cmd('destroy',
var=WebInBootstrap_vars,
capture_output=False,
**kwargs)
# return_code1 =0
print('Got return code {}'.format(return_code1))
if return_code1 != 0:
logger.info("WebInBootstrap destroyed")
deployment_status = {'WebInDeploy': 'Fail'}
exit()
else:
deployment_status = {'WebInDeploy': 'Success'}
exit()
#!python
from python_terraform import Terraform, IsFlagged
tf_base = Terraform(working_dir='iac/layer-base')
tf_users = Terraform(working_dir='iac/layer-users')
tf_base.apply(auto_approve=IsFlagged, capture_output=False)
tf_users.apply(auto_approve=IsFlagged, capture_output=False)
url_pf = tf_users.output('env_url')
print("Test with: " + url_pf)
def remotestate():
tf_state = Terraform(remotestate_dic)
tf_state.init()
#tf_state.plan(capture_output=False, var_file=tfvars)
tf_state.apply(skip_plan=True, capture_output=False, var_file=tfvars)
state_output = tf_state.output(capture_output=True)
class Terraform:
DEFAULT_DOCKER_HOST = 'unix:///var/run/docker.sock'
DEFAULT_DOCKER_ENTRYPOINT_PATH = '/docker-entrypoint.sh'
DEFAULT_NGINX_DOCKER_ENTRYPOINT_PATH = '/nginx.docker-entrypoint.sh'
DEFAULT_NGINX_DOCKER_IMAGE = 'nginx:stable-alpine'
DEFAULT_NGINX_DOCKER_CONTAINER_HTML_PATH = '/usr/share/nginx/html'
DEFAULT_UPLOAD_PATH = f"$HOME/.{PROJECT_NAME}/"
DEFAULT_SSH_USER = '******'
DEFAULT_SSH_PORT = 22
TERRAFORM_RESOURCE_FILE = 'file'
# trick for terrascript
class null_resource(Resource):
...
class vultr(Provider):
...
class vultr_server(Resource):
...
class vultr_ssh_key(Resource):
...
def __init__(self):
self.work_dir = TERRAFORM_WORK_DIR
self.app = TF(working_dir=self.work_dir)
@contextlib.contextmanager
def terraform_workspace(self):
workspace = f"terraform_workspace_{int(time.time())}"
self.app.create_workspace(workspace)
tmp_dir = TemporaryDirectory()
yield tmp_dir.name
self.app.set_workspace('default')
self.app.cmd('workspace delete', workspace, force=IsFlagged)
@contextlib.contextmanager
def patch_terraform_docker_ssh_conn(self):
# patch ssh config
yield
# clear ssh config
def write_terraform_config(self, config: Terrascript, dir_path: str):
tmp_config_file = NamedTemporaryFile(mode='wt',
suffix='.tf.json',
dir=dir_path,
delete=False)
tmp_config_file.write(str(config))
tmp_config_file.seek(0)
self.app.init(
dir_path,
plugin_dir=f"{self.work_dir}/plugins",
)
return tmp_config_file
def run_terraform_plan(self, config: Terrascript):
with self.terraform_workspace() as tw_dir:
self.write_terraform_config(config, tw_dir)
plan = self.app.plan(tw_dir, no_color=IsFlagged)
return plan
def run_terraform_apply(self, config: Terrascript):
with self.terraform_workspace() as tw_dir:
self.write_terraform_config(config, tw_dir)
print(config)
self.app.apply(tw_dir, skip_plan=True, no_color=IsFlagged)
output_result = self.app.output(json=IsFlagged, no_color=IsFlagged)
print(output_result)
output_var = {
output_var_key: output_result[output_var_key]['value']
for output_var_key in output_result
}
return output_var
def run_terraform_destroy(self, config: Terrascript):
with self.terraform_workspace() as tw_dir:
self.write_terraform_config(config, tw_dir)
destroy_result = self.app.destroy(tw_dir)
return destroy_result
@classmethod
def gen_digital_ocean_config(cls,
config_data: dict,
token: str,
public_key: str = None):
do_config = Terrascript()
do_provider = provider.digitalocean(token=token)
do_droplet_resource = resource.digitalocean_droplet(
f"server",
image=config_data['os_code'],
name=config_data['hostname'],
region=config_data['region_code'],
size=config_data['plan_code'],
ssh_keys=config_data['ssh_keys']
if config_data.get('ssh_keys') else [])
if public_key:
digitalocean_ssh_key = resource.digitalocean_ssh_key(
"digitalocean_ssh_key",
name="default",
public_key=public_key,
)
do_droplet_resource['ssh_keys'] += [
"${digitalocean_ssh_key.digitalocean_ssh_key.id}"
]
do_config += digitalocean_ssh_key
do_output_ip = Output(
'ip', value="${digitalocean_droplet.server.ipv4_address}")
do_output_id = Output('server_id',
value="${digitalocean_droplet.server.id}")
do_config += do_provider
do_config += do_droplet_resource
do_config += do_output_ip
do_config += do_output_id
return do_config
@classmethod
def gen_vultr_config(cls,
config_data: dict,
token: str,
public_key: str = None):
vultr_config = Terrascript()
vultr_provider = cls.vultr(api_key=token,
rate_limit=700,
retry_limit=3)
vultr_server = cls.vultr_server(f"server",
plan_id=config_data['plan_code'],
region_id=config_data['region_code'],
os_id=config_data['os_code'],
hostname=config_data['hostname'],
ssh_key_ids=config_data['ssh_keys']
if config_data.get('ssh_keys') else [])
vultr_output_ip = Output('ip', value="${vultr_server.server.main_ip}")
vultr_output_id = Output('server_id',
value="${vultr_server.server.id}")
if public_key:
vultr_ssh_key = cls.vultr_ssh_key('vultr_ssh_key',
name='default_key',
ssh_key=public_key)
vultr_server["ssh_key_ids"] += [
"${vultr_ssh_key.vultr_ssh_key.id}"
]
vultr_config += vultr_ssh_key
vultr_config += vultr_provider
vultr_config += vultr_server
vultr_config += vultr_output_ip
vultr_config += vultr_output_id
return vultr_config
@classmethod
def add_ssh_key_config(cls, public_key: str):
return provisioner("remote-exec",
provisioner=provisioner(
"remote-exec",
inline=[
'mkdir -p ~/.ssh',
f"{public_key} >> ~/.ssh/authorized_keys"
],
))
@classmethod
def gen_ssh_conn_config(cls,
*,
ssh_user: str = DEFAULT_SSH_USER,
ssh_private_key: str,
ssh_host: str,
ssh_port: int = DEFAULT_SSH_PORT) -> dict:
# see more in https://www.terraform.io/docs/provisioners/connection.html
return {
'type': 'ssh',
'user': ssh_user,
'private_key': ssh_private_key,
'host': ssh_host,
'port': ssh_port,
'timeout': '30s'
}
@classmethod
def gen_site_docker_deploy_config(cls,
*,
docker_host: str = DEFAULT_DOCKER_HOST,
site_name: str = None,
template_tar_bytes: bytes = None,
script: str = None,
ssh_user: str = DEFAULT_SSH_USER,
ssh_private_key: str,
ssh_host: str,
ssh_port: int = DEFAULT_SSH_PORT):
config = Terrascript()
docker_provider = provider.docker(host=docker_host,
connection=cls.gen_ssh_conn_config(
ssh_user=ssh_user,
ssh_private_key=ssh_private_key,
ssh_host=ssh_host,
ssh_port=ssh_port))
docker_image_resource = resource.docker_image(
'nginx_image',
name=cls.DEFAULT_NGINX_DOCKER_IMAGE,
)
docker_container_resource = resource.docker_container(
'nginx_container',
name=f"{site_name}-container-${{random_pet.docker_pet_name.id}}",
image="${docker_image.nginx_image.latest}",
restart="always",
start=True,
ports={'internal': 80},
upload=[])
docker_name_resource = resource.random_pet(
'docker_pet_name',
length=1,
)
if template_tar_bytes:
template_tar_file = f"{site_name}-tar-${{random_pet.docker_pet_name.id}}.tar.gz",
template_tar_file_content = base64.b64encode(
template_tar_bytes).decode('utf8')
template_tar_path = f"{cls.DEFAULT_NGINX_DOCKER_CONTAINER_HTML_PATH}/${template_tar_file}"
# self.upload_file(
# content='conf/myapp.conf',
# destination=f"{self.DEFAULT_UPLOAD_PATH}/${template_tar_file}",
# ssh_user=ssh_user,
# ssh_private_key=ssh_private_key,
# ssh_host=ssh_host,
# ssh_port=ssh_port
# )
docker_container_resource['upload'].append({
'content_base64':
template_tar_file_content,
'file':
template_tar_path
})
if script:
entrypoint_sh_content = TemplateRender().render(
cls.DEFAULT_NGINX_DOCKER_ENTRYPOINT_PATH,
init_script_path=cls.DEFAULT_DOCKER_ENTRYPOINT_PATH,
html_path=cls.DEFAULT_NGINX_DOCKER_CONTAINER_HTML_PATH)
docker_container_resource['upload'].append({
'content':
entrypoint_sh_content,
'file':
cls.DEFAULT_DOCKER_ENTRYPOINT_PATH
})
config += docker_provider
config += docker_image_resource
config += docker_container_resource
config += docker_name_resource
return config
def remote_exec(self,
*,
ssh_user: str = DEFAULT_SSH_USER,
ssh_private_key: str,
ssh_host: str,
ssh_port: int = DEFAULT_SSH_PORT):
exec_config = Terrascript()
ssh_conn = self.gen_ssh_conn_config(ssh_user=ssh_user,
ssh_private_key=ssh_private_key,
ssh_host=ssh_host,
ssh_port=ssh_port)
exec_resource = self.null_resource('remote-exec',
provisioner=provisioner(
"remote-exec",
inline=['ls -la'],
connection=ssh_conn))
exec_config += exec_resource
return exec_config
def upload_file(self,
content: str,
*,
destination: str = DEFAULT_UPLOAD_PATH,
ssh_user: str = DEFAULT_SSH_USER,
ssh_private_key: str,
ssh_host: str,
ssh_port: int = DEFAULT_SSH_PORT):
upload_config = Terrascript()
ssh_conn = self.gen_ssh_conn_config(ssh_user=ssh_user,
ssh_private_key=ssh_private_key,
ssh_host=ssh_host,
ssh_port=ssh_port)
file_resource = self.null_resource('upload_file_resource',
provisioner=provisioner(
self.TERRAFORM_RESOURCE_FILE,
content=content,
destination=destination,
connection=ssh_conn))
upload_config += file_resource
return upload_config
class Terraform:
DEFAULT_DOCKER_HOST = "unix:///var/run/docker.sock"
DEFAULT_DOCKER_ENTRYPOINT_PATH = "/docker-entrypoint.sh"
DEFAULT_NGINX_DOCKER_ENTRYPOINT_PATH = "/nginx.docker-entrypoint.sh"
DEFAULT_NGINX_DOCKER_IMAGE = "nginx:stable-alpine"
DEFAULT_NGINX_DOCKER_CONTAINER_HTML_PATH = "/usr/share/nginx/html"
DEFAULT_UPLOAD_PATH = f"$HOME/.{PROJECT_NAME}/"
DEFAULT_SSH_USER = "******"
DEFAULT_SSH_PORT = 22
TERRAFORM_RESOURCE_FILE = "file"
# trick for terrascript
class null_resource(Resource):
...
class tencentcloud(Provider):
...
class tencentcloud_availability_zones(Data):
...
class tencentcloud_images(Data):
...
class tencentcloud_instance_types(Data):
...
class tencentcloud_security_group(Resource):
...
class tencentcloud_security_group_lite_rule(Resource):
...
class tencentcloud_instance(Resource):
...
class tencentcloud_key_pair(Resource):
...
class alicloud(Provider):
...
class alicloud_vpc(Resource):
...
class alicloud_key_pair(Resource):
...
class alicloud_security_group(Resource):
...
class alicloud_security_group_rule(Resource):
...
class alicloud_instance(Resource):
...
class alicloud_vswitch(Resource):
...
class alicloud_zones(Data):
...
class vultr(Provider):
...
class vultr_server(Resource):
...
class vultr_ssh_key(Resource):
...
def __init__(self):
self.work_dir = TERRAFORM_WORK_DIR
self.app = TF(working_dir=self.work_dir)
@contextlib.contextmanager
def terraform_workspace(self):
workspace = f"terraform_workspace_{int(time.time())}"
self.app.create_workspace(workspace)
tmp_dir = TemporaryDirectory()
yield tmp_dir.name
self.app.set_workspace("default")
self.app.cmd("workspace delete", workspace, force=IsFlagged)
@contextlib.contextmanager
def patch_terraform_docker_ssh_conn(self):
# patch ssh config
yield
# clear ssh config
def write_terraform_config(self, config: Terrascript, dir_path: str):
tmp_config_file = NamedTemporaryFile(mode="wt",
suffix=".tf.json",
dir=dir_path,
delete=False)
logging.info(str(config))
tmp_config_file.write(str(config))
tmp_config_file.seek(0)
self.app.init(
dir_path
# disable maual plugin because it changes toooo fast
# dir_path, plugin_dir=f"{self.work_dir}/plugins",
)
return tmp_config_file
def run_terraform_plan(self, config: Terrascript):
with self.terraform_workspace() as tw_dir:
self.write_terraform_config(config, tw_dir)
plan = self.app.plan(tw_dir, no_color=IsFlagged)
return plan
def run_terraform_apply(self, config: Terrascript):
with self.terraform_workspace() as tw_dir:
self.write_terraform_config(config, tw_dir)
self.app.apply(tw_dir, skip_plan=True, no_color=IsFlagged)
output_result = self.app.output(json=IsFlagged, no_color=IsFlagged)
output_var = {
output_var_key: output_result[output_var_key]["value"]
for output_var_key in output_result
}
return output_var
def run_terraform_destroy(self, config: Terrascript):
with self.terraform_workspace() as tw_dir:
self.write_terraform_config(config, tw_dir)
destroy_result = self.app.destroy(tw_dir)
return destroy_result
@classmethod
def gen_digital_ocean_config(cls,
config_data: dict,
token: str,
public_key: str = None):
do_config = Terrascript()
do_provider = provider.digitalocean(token=token)
do_droplet_resource = resource.digitalocean_droplet(
"server",
image=config_data["os_code"],
name=config_data["hostname"],
region=config_data["region_code"],
size=config_data["plan_code"],
ssh_keys=config_data["ssh_keys"]
if config_data.get("ssh_keys") else [],
)
if public_key:
digitalocean_ssh_key = resource.digitalocean_ssh_key(
"digitalocean_ssh_key",
name="default",
public_key=public_key,
)
do_droplet_resource["ssh_keys"] += [
"${digitalocean_ssh_key.digitalocean_ssh_key.id}"
]
do_config += digitalocean_ssh_key
do_output_ip = Output(
"ip", value="${digitalocean_droplet.server.ipv4_address}")
do_output_id = Output("server_id",
value="${digitalocean_droplet.server.id}")
do_config += do_provider
do_config += do_droplet_resource
do_config += do_output_ip
do_config += do_output_id
return do_config
@classmethod
def gen_vultr_config(cls,
config_data: dict,
token: str,
public_key: str = None):
vultr_config = Terrascript()
vultr_provider = cls.vultr(api_key=token,
rate_limit=700,
retry_limit=3)
vultr_server = cls.vultr_server(
"server",
plan_id=config_data["plan_code"],
region_id=config_data["region_code"],
os_id=config_data["os_code"],
hostname=config_data["hostname"],
ssh_key_ids=config_data["ssh_keys"]
if config_data.get("ssh_keys") else [],
)
vultr_output_ip = Output("ip", value="${vultr_server.server.main_ip}")
vultr_output_id = Output("server_id",
value="${vultr_server.server.id}")
if public_key:
vultr_ssh_key = cls.vultr_ssh_key("vultr_ssh_key",
name="default_key",
ssh_key=public_key)
vultr_server["ssh_key_ids"] += [
"${vultr_ssh_key.vultr_ssh_key.id}"
]
vultr_config += vultr_ssh_key
vultr_config += vultr_provider
vultr_config += vultr_server
vultr_config += vultr_output_ip
vultr_config += vultr_output_id
return vultr_config
@classmethod
def gen_tencent_cloud_config(
cls,
config_data: dict,
token: str,
public_key_name: str = None,
secret_id: str = None,
):
tencent_cloud_config = Terrascript()
tencent_terraform = terraform(
**{
"required_providers": {
"tencentcloud": {
"source": "terraform-providers/tencentcloud",
"version": "~> 1.40.3",
},
}
})
tencent_cloud_provider = cls.tencentcloud(
secret_id=secret_id,
secret_key=token,
region=config_data["region_code"],
)
tencent_zone = cls.tencentcloud_availability_zones("default")
tencent_security_group = cls.tencentcloud_security_group(
"default", name="all-open", description="open all ports")
tencent_security_group_rule = cls.tencentcloud_security_group_lite_rule(
"rule",
security_group_id="${tencentcloud_security_group.default.id}",
ingress=[
"ACCEPT#10.0.0.0/8#ALL#ALL",
],
egress=[
"ACCEPT#10.0.0.0/8#ALL#ALL",
],
)
tencent_cloud_server = cls.tencentcloud_instance(
"server",
instance_name=config_data["hostname"],
availability_zone=
"${data.tencentcloud_availability_zones.default.zones.0.name}",
image_id=config_data["os_code"],
instance_type=config_data["plan_code"],
disable_monitor_service=True,
disable_security_service=True,
allocate_public_ip=True,
internet_max_bandwidth_out=5,
instance_charge_type="POSTPAID_BY_HOUR",
internet_charge_type="TRAFFIC_POSTPAID_BY_HOUR",
system_disk_type="CLOUD_SSD",
count=1,
)
tencent_output_ip = Output(
"ip", value="${tencentcloud_instance.server.0.public_ip}")
tencent_output_id = Output(
"server_id", value="${tencentcloud_instance.server.0.id}")
if public_key_name:
tencent_cloud_server["key_name"] = public_key_name
tencent_cloud_config += tencent_terraform
tencent_cloud_config += tencent_cloud_provider
tencent_cloud_config += tencent_zone
tencent_cloud_config += tencent_security_group
tencent_cloud_config += tencent_security_group_rule
tencent_cloud_config += tencent_cloud_server
tencent_cloud_config += tencent_output_ip
tencent_cloud_config += tencent_output_id
return tencent_cloud_config
@classmethod
def gen_ali_cloud_config(
cls,
config_data: dict,
token: str,
ssh_key_name: str = None,
access_key: str = None,
security_groups: List[str] = [],
):
ali_cloud_config = Terrascript()
ali_cloud_provider = cls.alicloud(
access_key=access_key,
secret_key=token,
region=config_data["region_code"],
)
ali_zone = cls.alicloud_zones(
"default",
available_disk_category="cloud_efficiency",
available_resource_creation="Instance",
)
ali_vpc = cls.alicloud_vpc(
"vpc",
cidr_block="172.16.0.0/12",
)
ali_vswitch = cls.alicloud_vswitch(
"vswitch",
vpc_id="${alicloud_vpc.vpc.id}",
cidr_block="172.16.0.0/29",
availability_zone="${data.alicloud_zones.default.zones.0.id}",
)
ali_security_group = cls.alicloud_security_group(
"group",
name="all-open",
vpc_id="${alicloud_vpc.vpc.id}",
description="open all ports",
inner_access_policy="Accept",
)
ali_internet_security_group_rule = cls.alicloud_security_group_rule(
"internet",
# nic_type="internet",
security_group_id="${alicloud_security_group.group.id}",
type="ingress",
port_range="-1/-1",
cidr_ip="0.0.0.0/0",
ip_protocol="all",
policy="accept",
)
ali_intranet_security_group_rule = cls.alicloud_security_group_rule(
"intranet",
# nic_type="intranet",
security_group_id="${alicloud_security_group.group.id}",
port_range="-1/-1",
type="egress",
cidr_ip="0.0.0.0/0",
ip_protocol="all",
policy="accept",
priority=1,
)
ali_cloud_server = cls.alicloud_instance(
"server",
instance_name=config_data["hostname"],
availability_zone="${data.alicloud_zones.default.zones.0.id}",
# security_groups=security_groups,
security_groups="${alicloud_security_group.group.*.id}",
vswitch_id="${alicloud_vswitch.vswitch.id}",
image_id=config_data["os_code"],
instance_type=config_data["plan_code"],
security_enhancement_strategy="Deactive",
instance_charge_type="PostPaid",
internet_charge_type="PayByTraffic",
internet_max_bandwidth_out=2,
)
ali_output_ip = Output("ip",
value="${alicloud_instance.server.public_ip}")
ali_output_id = Output("server_id",
value="${alicloud_instance.server.id}")
if ssh_key_name:
ali_cloud_server["key_name"] = ssh_key_name
ali_cloud_config += ali_cloud_provider
ali_cloud_config += ali_zone
ali_cloud_config += ali_vpc
ali_cloud_config += ali_vswitch
ali_cloud_config += ali_security_group
ali_cloud_config += ali_internet_security_group_rule
ali_cloud_config += ali_intranet_security_group_rule
ali_cloud_config += ali_cloud_server
ali_cloud_config += ali_output_ip
ali_cloud_config += ali_output_id
return ali_cloud_config
@classmethod
def add_ssh_key_config(cls, public_key: str):
return provisioner(
"remote-exec",
provisioner=provisioner(
"remote-exec",
inline=[
"mkdir -p ~/.ssh",
f"{public_key} >> ~/.ssh/authorized_keys"
],
),
)
@classmethod
def gen_ssh_conn_config(
cls,
*,
ssh_user: str = DEFAULT_SSH_USER,
ssh_private_key: str,
ssh_host: str,
ssh_port: int = DEFAULT_SSH_PORT,
) -> dict:
# see more in https://www.terraform.io/docs/provisioners/connection.html
return {
"type": "ssh",
"user": ssh_user,
"private_key": ssh_private_key,
"host": ssh_host,
"port": ssh_port,
"timeout": "30s",
}
@classmethod
def gen_site_docker_deploy_config(
cls,
*,
docker_host: str = DEFAULT_DOCKER_HOST,
site_name: str = None,
template_tar_bytes: bytes = None,
script: str = None,
ssh_user: str = DEFAULT_SSH_USER,
ssh_private_key: str,
ssh_host: str,
ssh_port: int = DEFAULT_SSH_PORT,
):
config = Terrascript()
docker_provider = provider.docker(
host=docker_host,
connection=cls.gen_ssh_conn_config(
ssh_user=ssh_user,
ssh_private_key=ssh_private_key,
ssh_host=ssh_host,
ssh_port=ssh_port,
),
)
docker_image_resource = resource.docker_image(
"nginx_image",
name=cls.DEFAULT_NGINX_DOCKER_IMAGE,
)
docker_container_resource = resource.docker_container(
"nginx_container",
name=f"{site_name}-container-${{random_pet.docker_pet_name.id}}",
image="${docker_image.nginx_image.latest}",
restart="always",
start=True,
ports={"internal": 80},
upload=[],
)
docker_name_resource = resource.random_pet(
"docker_pet_name",
length=1,
)
if template_tar_bytes:
template_tar_file = (
f"{site_name}-tar-${{random_pet.docker_pet_name.id}}.tar.gz", )
template_tar_file_content = base64.b64encode(
template_tar_bytes).decode("utf8")
template_tar_path = (
f"{cls.DEFAULT_NGINX_DOCKER_CONTAINER_HTML_PATH}/${template_tar_file}"
)
# self.upload_file(
# content='conf/myapp.conf',
# destination=f"{self.DEFAULT_UPLOAD_PATH}/${template_tar_file}",
# ssh_user=ssh_user,
# ssh_private_key=ssh_private_key,
# ssh_host=ssh_host,
# ssh_port=ssh_port
# )
docker_container_resource["upload"].append({
"content_base64":
template_tar_file_content,
"file":
template_tar_path
})
if script:
entrypoint_sh_content = TemplateRender().render(
cls.DEFAULT_NGINX_DOCKER_ENTRYPOINT_PATH,
init_script_path=cls.DEFAULT_DOCKER_ENTRYPOINT_PATH,
html_path=cls.DEFAULT_NGINX_DOCKER_CONTAINER_HTML_PATH,
)
docker_container_resource["upload"].append({
"content":
entrypoint_sh_content,
"file":
cls.DEFAULT_DOCKER_ENTRYPOINT_PATH,
})
config += docker_provider
config += docker_image_resource
config += docker_container_resource
config += docker_name_resource
return config
def remote_exec(
self,
*,
ssh_user: str = DEFAULT_SSH_USER,
ssh_private_key: str,
ssh_host: str,
ssh_port: int = DEFAULT_SSH_PORT,
):
exec_config = Terrascript()
ssh_conn = self.gen_ssh_conn_config(
ssh_user=ssh_user,
ssh_private_key=ssh_private_key,
ssh_host=ssh_host,
ssh_port=ssh_port,
)
exec_resource = self.null_resource(
"remote-exec",
provisioner=provisioner("remote-exec",
inline=["ls -la"],
connection=ssh_conn),
)
exec_config += exec_resource
return exec_config
def upload_file(
self,
content: str,
*,
destination: str = DEFAULT_UPLOAD_PATH,
ssh_user: str = DEFAULT_SSH_USER,
ssh_private_key: str,
ssh_host: str,
ssh_port: int = DEFAULT_SSH_PORT,
):
upload_config = Terrascript()
ssh_conn = self.gen_ssh_conn_config(
ssh_user=ssh_user,
ssh_private_key=ssh_private_key,
ssh_host=ssh_host,
ssh_port=ssh_port,
)
file_resource = self.null_resource(
"upload_file_resource",
provisioner=provisioner(
self.TERRAFORM_RESOURCE_FILE,
content=content,
destination=destination,
connection=ssh_conn,
),
)
upload_config += file_resource
return upload_config
