def handler(system: flow_api.System, this: flow_api.Execution):
vault_config = 'my-vaultconfig'
secret_path = 'my-git-secret'
secret_key_user = '******'
secret_key_password = '******'
my_connector_name = 'git-example-connector'
# Create connector of type GIT and link it with vault secrets
system.connector(my_connector_name).save(
connector_type='GIT',
value={
'username': f'vault.secret({secret_key_user})',
'password': f'vault.secret({secret_key_password})',
},
vault_secrets=[{
'vault_name': vault_config,
'engine_path': 'kv',
'secret_path': secret_path,
}],
)
# Use the connector:
git_metadata = this.connect(
my_connector_name,
name='get git-metadata',
repository_url='https://mygitrepo.com/repo/',
command='metadata',
).get('output_value')
this.log(git_metadata)
return this.success('all done')
def handler(system: flow_api.System, this: flow_api.Execution):
inputs = this.get('input_value')
gcloud_connector = inputs.get('gcloud_connector')
project_id = system.connector(gcloud_connector).get('value').get(
'key').get('project_id')
if project_id is None or gcloud_connector is None:
return this.error('Missing inputs')
delete_task = this.connect(
gcloud_connector,
name='delete webpagetest-server',
api_name='compute',
api_version='v1',
collection='instances',
request='delete',
params={
'instance': 'webpagetest-server',
'project': project_id,
'zone': 'europe-west1-b',
},
wait=system.return_when.ALL_ENDED,
)
status = delete_task.get('status')
message = delete_task.get('message')
operation = delete_task.get('output_value').get('result')
this.flow(
'google_operation_wait',
operation=operation,
)
system.setting('webpagetestserver').release()
if status == 'ENDED_ERROR' and 'HttpError 404' not in message:
return this.error('failed to delete webpagetest-server')
webpagetestserver = system.setting('webpagetestserver').get('value')
webpagetestserver['active'] = False
system.setting('webpagetestserver').save(value=webpagetestserver)
return this.success('all done')
def handler(system: flow_api.System, this: flow_api.Execution):
inputs = this.get('input_value')
gcloud_connector = inputs.get('gcloud_connector')
project_id = system.connector(gcloud_connector).get('value').get(
'key').get('project_id')
if project_id is None or gcloud_connector is None:
return this.error('Missing inputs')
try:
webpagetestserver = system.setting('webpagetestserver').get('value')
if webpagetestserver.get('active'):
this.set_output(wait=False)
return this.success('Server already active')
except:
webpagetestserver = system.setting('webpagetestserver')
this.save(message=f'waiting for webpagetestserver lock')
system.setting('webpagetestserver').acquire(timeout=None)
this.save(message=f'creating webpagetest server')
webpagetest_server_config_str = system.file(
'webpagetest-server.json').get_text_content()
webpagetest_server_config = json.loads(webpagetest_server_config_str)
webpagetest_server_config['name'] = 'webpagetest-server'
webpagetest_server_config['disks'][0]['deviceName'] = 'webpagetest-server'
webpagetest_server_config['disks'][0]['initializeParams'][
'sourceSnapshot'] = f'projects/{project_id}/global/snapshots/webpagetest-server-template'
launch_task = this.connect(
gcloud_connector,
name='launch webpagetest-server',
api_name='compute',
api_version='v1',
collection='instances',
request='insert',
params={
'body': webpagetest_server_config,
'project': project_id,
'zone': 'europe-west1-b',
},
wait=system.return_when.ALL_ENDED,
)
status = launch_task.get('status')
message = launch_task.get('message')
if status == 'ENDED_ERROR' and 'HttpError 409' in message:
return this.success('using existing webpagetestserver')
if status == 'ENDED_ERROR':
return this.error('failed to launch webpagetest-server')
operation = launch_task.get('output_value')['result']
this.flow(
'google_operation_wait',
operation=operation,
)
for _ in range(60):
instance_list = this.connect(
gcloud_connector,
name='find webpagetest-server',
api_name='compute',
api_version='v1',
collection='instances',
request='list',
params={
'filter': 'name=webpagetest-server',
'project': project_id,
'zone': 'europe-west1-b',
},
).get('output_value')
if len(instance_list['result'].get('items', [])) != 0:
break
this.sleep(1)
else:
return this.error('did not find webpagetest server within 60 tries')
external_ip = instance_list['result']['items'][0]['networkInterfaces'][0][
'accessConfigs'][0]['natIP']
webpagetestserver = system.setting('webpagetestserver').get('value')
if webpagetestserver is None:
webpagetestserver = {}
webpagetestserver['hostname'] = external_ip
webpagetestserver['created_at'] = datetime.timestamp(datetime.now())
webpagetestserver['active'] = True
system.setting('webpagetestserver').save(value=webpagetestserver)
this.set_output(wait=True) # wait until the docker containers are ready
return this.success('all done')
def handler(system: flow_api.System, this: flow_api.Execution):
inputs = this.get('input_value')
gcloud_connector = inputs.get('gcloud_connector')
project_id = system.connector(gcloud_connector).get('value').get(
'key').get('project_id')
if project_id is None or gcloud_connector is None:
return this.error('Missing inputs')
webpagetestserver_template_setting = system.setting(
'webpagetest-server-template')
webpagetestserver_template_setting.acquire(timeout=None)
instance_list = this.connect(
gcloud_connector,
name='find webpagetest-server-template',
api_name='compute',
api_version='v1',
collection='instances',
request='list',
params={
'filter': 'name=webpagetest-server-template',
'project': project_id,
'zone': 'europe-west1-b',
},
).get('output_value')
if len(instance_list['result'].get('items', [])) > 0:
this.connect(
gcloud_connector,
name='delete webpagetest-template',
api_name='compute',
api_version='v1',
collection='instances',
request='delete',
params={
'instance': 'webpagetest-server-template',
'project': project_id,
'zone': 'europe-west1-b',
},
)
for _ in range(60):
instance_list = this.connect(
gcloud_connector,
name='find webpagetest-server-template',
api_name='compute',
api_version='v1',
collection='instances',
request='list',
params={
'filter': f'name=webpagetest-server-template',
'project': project_id,
'zone': 'europe-west1-b',
},
).get('output_value')
if len(instance_list['result'].get('items', [])) == 0:
break
this.sleep(1)
else:
return this.error(
'old webpagetest-server-template still exists after 60 tries')
images = this.connect(gcloud_connector,
name='find ubuntu image name',
api_name='compute',
api_version='v1',
collection='images',
request='list',
params={
'project': 'ubuntu-os-cloud'
}).load('output_value')['result']['items']
image_name = None
for image in images:
with contextlib.suppress(KeyError):
if image['deprecated']['state'] == 'DEPRECATED':
continue
if image['name'].startswith('ubuntu-1804-bionic-'):
image_name = image['name']
break
setup_user_key = RSA.generate(2048)
setup_user_priv = setup_user_key.export_key().decode()
setup_user_pub = setup_user_key.publickey().export_key(
format='OpenSSH').decode()
setup_user_key = None
webpagetest_user_key = RSA.generate(2048)
webpagetest_user_priv = webpagetest_user_key.export_key().decode()
webpagetest_user_pub = webpagetest_user_key.publickey().export_key(
format='OpenSSH').decode()
webpagetest_user_key = None
webpagetest_server_startup_script = system.file(
'webpagetest-server-template-startup.sh').get_text_content()
webpagetest_server_startup_script = webpagetest_server_startup_script.format(
setup_user_pub=setup_user_pub,
webpagetest_user_pub=webpagetest_user_pub,
)
webpagetest_server_config_str = system.file(
'webpagetest-server-template.json').get_text_content()
webpagetest_server_config_str.replace('{PROJECT_ID}', project_id)
webpagetest_server_config = json.loads(webpagetest_server_config_str)
webpagetest_server_config['metadata']['items'][0][
'value'] = webpagetest_server_startup_script
webpagetest_server_config['name'] = 'webpagetest-server-template'
webpagetest_server_config['disks'][0][
'deviceName'] = 'webpagetest-server-template'
webpagetest_server_config['disks'][0]['initializeParams'][
'sourceImage'] = f'projects/ubuntu-os-cloud/global/images/{image_name}'
operation = this.connect(
gcloud_connector,
name='launch webpagetest-server-template',
api_name='compute',
api_version='v1',
collection='instances',
request='insert',
params={
'body': webpagetest_server_config,
'project': project_id,
'zone': 'europe-west1-b',
},
).get('output_value')['result']
this.flow(
'google_operation_wait',
operation=operation,
)
try:
for _ in range(60):
instance_list = this.connect(
gcloud_connector,
name='find webpagetest-server-template',
api_name='compute',
api_version='v1',
collection='instances',
request='list',
params={
'filter': 'name=webpagetest-server-template',
'project': project_id,
'zone': 'europe-west1-b',
},
).get('output_value')
if len(instance_list['result'].get('items', [])) != 0:
break
this.sleep(1)
else:
return this.error(
'did not find webpagetest server template within 60 tries')
external_ip = instance_list['result']['items'][0]['networkInterfaces'][
0]['accessConfigs'][0]['natIP']
failed_tries = []
for _ in range(60):
this.sleep(1)
guest_attribute_task = this.connect(
gcloud_connector,
name='get webpagetest-server-template hostkey',
api_name='compute',
api_version='beta',
collection='instances',
request='getGuestAttributes',
params={
'instance': 'webpagetest-server-template',
'queryPath': 'hostkeys/ssh-rsa',
'project': project_id,
'zone': 'europe-west1-b',
},
wait=system.return_when.ALL_ENDED,
)
if guest_attribute_task.load('status') == 'ENDED_SUCCESS':
break
failed_tries.append(guest_attribute_task)
else:
return this.error('did not find hostkey within 60 tries')
for failed_try in failed_tries:
failed_try.delete()
hostkey = guest_attribute_task.get(
'output_value')['result']['queryValue']['items'][0]['value']
system.connector('webpagetestserver-setup').save(
connector_type='SSH',
value={
'key': setup_user_priv,
'username': '******',
'hostname': external_ip,
'hostkey': hostkey,
},
)
system.connector('webpagetestserver').save(
connector_type='SSH',
value={
'key': webpagetest_user_priv,
'username': '******',
'hostname': external_ip,
'hostkey': hostkey,
},
)
system.connector('webpagetestserver-copy').save(
connector_type='SCP',
value={
'key': webpagetest_user_priv,
'username': '******',
'hostname': external_ip,
'hostkey': hostkey,
},
)
webpagetestserver_template = {
'hostname': external_ip,
'hostkey': hostkey,
'setup-user': {
'username': '******',
'ssh-key': setup_user_priv,
'public-key': setup_user_pub,
},
'webpagetest-user': {
'username': '******',
'ssh-key': webpagetest_user_priv,
'public-key': webpagetest_user_pub,
},
}
for _ in range(10):
try:
this.connect(
'webpagetestserver-setup',
script="""
sudo apt-get update
sudo apt-get install -y --no-install-recommends python-pip python3-pip
sudo apt-get install -y --no-install-recommends build-essential python3-dev iftop iotop zip unzip
# DO NOT remove python3-cryptography! or instance will not be able to setup networking on boot
# sudo apt-get remove -y python3-cryptography
""",
name='install packages',
)
except Exception:
this.sleep(3)
else:
break
else:
return this.error(
'failed to ssh to webpagetestserver-template within 10 tries')
this.connect(
'webpagetestserver',
script="""
pip install wheel setuptools
pip3 install wheel setuptools
pip3 install alembic psycopg2-binary
""",
name='install pip packages',
)
this.connect(
'webpagetestserver-setup',
script="""
set +e
ssh -V
ret=$?
set -e
if [ "$ret" -ne "0" ]; then
sudo apt-get install -y --no-install-recommends openssh-client
fi
""",
name='install openssh client',
)
this.connect(
'webpagetestserver-setup',
script="""
set +e
jq -V
ret=$?
set -e
if [ "$ret" -ne "0" ]; then
sudo apt-get install -y --no-install-recommends jq
fi
set +e
jq -V
ret=$?
set -e
if [ "$ret" -ne "0" ]; then
sudo snap install jq
fi
""",
name='install jq',
)
this.connect(
'webpagetestserver-setup',
script="""
set +e
yq -V
ret=$?
set -e
if [ "$ret" -ne "0" ]; then
sudo snap install yq
fi
""",
name='install yq',
)
this.connect(
'webpagetestserver-setup',
script="""
set +e
docker --version
ret=$?
set -e
if [ "$ret" -ne "0" ]; then
curl -fsSL https://get.docker.com -o get-docker.sh
sh get-docker.sh
RELEASE=$(lsb_release -is | tr '[:upper:]' '[:lower:]')
sudo apt-get install -y --no-install-recommends \\
apt-transport-https \\
ca-certificates \\
curl \\
gnupg2 \\
software-properties-common
curl -fsSL https://download.docker.com/linux/${RELEASE}/gpg | sudo apt-key add -
sudo add-apt-repository \\
"deb [arch=amd64] https://download.docker.com/linux/${RELEASE} \\
$(lsb_release -cs) \\
stable"
sudo apt-get update
sudo apt-get install -y --no-install-recommends docker-ce
fi
sudo usermod -a -G docker webpagetest-user
""",
name='install docker',
)
this.connect(
'webpagetestserver',
name='create webpagetest folders',
script="""
mkdir -p webpagetest/server
mkdir -p webpagetest/agent
""",
)
this.connect(
'webpagetestserver-copy',
name='copy webpagetest server Dockerfile',
src='cloudomation:webpagetest-server-Dockerfile',
dst='webpagetest/server/Dockerfile',
)
this.connect(
'webpagetestserver-copy',
name='copy webpagetest server locations.ini',
src='cloudomation:webpagetest-server-locations.ini',
dst='webpagetest/server/locations.ini',
)
this.connect(
'webpagetestserver-copy',
name='copy webpagetest agent Dockerfile',
src='cloudomation:webpagetest-agent-Dockerfile',
dst='webpagetest/agent/Dockerfile',
)
this.connect(
'webpagetestserver-copy',
name='copy webpagetest agent script.sh',
src='cloudomation:webpagetest-agent-script.sh',
dst='webpagetest/agent/script.sh',
)
this.connect(
'webpagetestserver',
name='create webpagetest server and agent',
input_value={'script_timeout': 600},
script="""
pushd webpagetest/server
docker build -t wpt-server .
docker run -d -p 4000:80 --restart unless-stopped wpt-server
popd
pushd webpagetest/agent
chmod u+x script.sh
docker build -t wpt-agent .
docker run -d -p 4001:80 \\
--restart unless-stopped \\
--network="host" \\
-e "SERVER_URL=http://localhost:4000/work/" \\
-e "LOCATION=Test" \\
wpt-agent
""",
)
this.connect(
gcloud_connector,
name='stop webpagetest-server-template',
api_name='compute',
api_version='v1',
collection='instances',
request='stop',
params={
'instance': 'webpagetest-server-template',
'project': project_id,
'zone': 'europe-west1-b',
},
)
try:
operation = this.connect(
gcloud_connector,
name='delete old webpagetest-server-template snapshot',
api_name='compute',
api_version='v1',
collection='snapshots',
request='delete',
params={
'snapshot': 'webpagetest-server-template',
'project': project_id,
},
).get('output_value')['result']
this.flow(
'google_operation_wait',
operation=operation,
)
except Exception:
pass
today = date.today()
operation = this.connect(
gcloud_connector,
name='create webpagetest-server-template snapshot',
api_name='compute',
api_version='v1',
collection='disks',
request='createSnapshot',
params={
'disk': 'webpagetest-server-template',
'body': {
'name': 'webpagetest-server-template',
'description':
f'auto-generated at {today} by setup webpagetest server',
'storageLocations': ['europe-west1'],
},
'project': project_id,
'zone': 'europe-west1-b',
},
).get('output_value')['result']
this.flow(
'google_operation_wait',
operation=operation,
)
webpagetestserver_template.pop('hostname')
webpagetestserver_template.pop('hostkey')
webpagetestserver_template_setting.save(
value=webpagetestserver_template)
webpagetestserver_template_setting.release()
except Exception as ex:
this.save(message=repr(ex))
if this.get('input_value').get('interactive'):
this.sleep(120)
# this.pause()
raise
finally:
instance_list = this.connect(
gcloud_connector,
name='find webpagetest-server-template',
api_name='compute',
api_version='v1',
collection='instances',
request='list',
params={
'filter': f'name=webpagetest-server-template',
'project': project_id,
'zone': 'europe-west1-b'
},
).get('output_value')
if len(instance_list['result'].get('items', [])) > 0:
this.connect(
gcloud_connector,
name='delete webpagetest-server-template',
api_name='compute',
api_version='v1',
collection='instances',
request='delete',
params={
'instance': 'webpagetest-server-template',
'project': project_id,
'zone': 'europe-west1-b',
},
)
while True:
instance_list = this.connect(
gcloud_connector,
name='find webpagetest-server-template',
api_name='compute',
api_version='v1',
collection='instances',
request='list',
params={
'filter': 'name=webpagetest-server-template',
'project': project_id,
'zone': 'europe-west1-b',
},
).get('output_value')
if len(instance_list['result'].get('items', [])) == 0:
break
this.sleep(1)
return this.success('all done')
def handler(system: flow_api.System, this: flow_api.Execution):
"""
In this example we setup a Virtual Machine (VM) in the google-cloud.
Then, Ubuntu will be installed and a small bash script is executed.
Bash scripts can be used to install more software on the VM or
change settings, anything you could do on a local machine in a terminal.
This flow also requires the two files:
* create google cloud vm example.sh
* create google cloud vm example.json
In the future, you can take these as an example, adopt them and
upload them with a new name in Cloudomation > Files.
"""
# This is a pop-up message form, it will ask for some information
# The last element is an OK-button, after clicking it the
# values are passed further and the pop-up message closes.
form_response = system.message(
subject='Input information to setup a VM in google-cloud',
message_type='POPUP',
body={
'type':
'object',
'properties': {
'connector_name': {
'element': 'string',
'type': 'string',
'label': 'Enter name for the connector: \n\
(This name can be searched for and is used to connect with your VM)',
'example': 'my-vm-on-gcloud',
'order': 1,
},
'project_id': {
'element': 'string',
'type': 'string',
'label': 'Enter your google cloud project-ID:',
'default': 'my-first-project-id',
'order': 2,
},
'machine_type': {
'element': 'string',
'type': 'string',
'label':
'Enter the machine type you want to use on google-cloud:',
'default': 'n1-standard-1',
'order': 3,
},
'server_location': {
'element': 'string',
'type': 'string',
'label':
'Enter the desired location of the hosting server from google-cloud:',
'default': 'europe-west1-b',
'order': 4,
},
'Ok': {
'element': 'submit',
'label': 'OK',
'type': 'boolean',
'order': 5,
},
},
'required': [
'connector_name',
'project_id',
'machine_type',
'server_location',
],
},
).wait().get('response')
connector_name = form_response['connector_name']
project_id = form_response['project_id']
machine_type = form_response['machine_type']
server_location = form_response['server_location']
# generate a ssh key-pair for the setup-user:
setup_user_key = RSA.generate(2048)
setup_user_priv = setup_user_key.export_key().decode()
setup_user_pub = setup_user_key.publickey().export_key(
format='OpenSSH').decode()
setup_user_key = None
# load the startup script and pass the public ssh key to the script:
vm_startup_script = system.file(
'create google cloud vm example.sh').get_text_content()
vm_startup_script = vm_startup_script.format(
setup_user_pub=setup_user_pub, )
# load the configuration for the VM and setup last parameters:
vm_config_str = system.file(
'create google cloud vm example.json').get_text_content()
vm_config = json.loads(vm_config_str)
vm_config['name'] = connector_name
vm_config['machineType'] = \
f'projects/{project_id}/zones/{server_location}/machineTypes/{machine_type}'
vm_config['zone'] = f'projects/{project_id}/zones/{server_location}'
vm_config['metadata']['items'][0]['value'] = vm_startup_script
vm_config['disks'][0]['deviceName'] = connector_name
vm_config['disks'][0]['initializeParams']['diskType'] = \
f'projects/{project_id}/zones/{server_location}/diskTypes/pd-ssd'
vm_config['networkInterfaces'][0]['subnetwork'] = \
f'projects/{project_id}/regions/{server_location[:-2]}/subnetworks/default'
# NOTE: two sources can be used for initial system installation:
# a) from 'sourceSnapshot'
# b) from 'sourceImage'
# This flow script will install Ubuntu, using the parameter 'sourceImage'.
# For installing a system from snapshot use the respective parameter.
# This paramter is set under: disks / initializeParams
# Connect to the VM
operation = this.connect(
'gcloud',
name=f'launch {connector_name}',
api_name='compute',
api_version='v1',
collection='instances',
request='insert',
params={
'project': project_id,
'zone': server_location,
'body': vm_config,
},
).get('output_value')['result']
this.flow(
'google_operation_wait',
operation=operation,
)
# retrieve external IP and hostkey (required for creating a SSH connector):
for _ in range(60):
try:
instance_list = this.connect(
'gcloud',
name=f'find {connector_name}',
api_name='compute',
api_version='v1',
collection='instances',
request='list',
params={
'project': project_id,
'zone': server_location,
'filter': f'name={connector_name}',
},
).get('output_value')
except flow_api.DependencyFailedError:
this.sleep(1)
if len(instance_list['result'].get('items', [])) != 0:
break
this.sleep(1)
else:
return this.error('did not find VM within 60 tries')
external_ip = instance_list['result']['items'][0]['networkInterfaces'][0][
'accessConfigs'][0]['natIP']
last_try_guest_attribute = None
for _ in range(60):
if last_try_guest_attribute is not None:
last_try_guest_attribute.archive()
try:
last_try_guest_attribute = this.connect(
'gcloud',
name=f'get {connector_name} hostkey',
api_name='compute',
api_version='v1',
collection='instances',
request='getGuestAttributes',
params={
'project': project_id,
'zone': server_location,
'instance': connector_name,
'queryPath': 'hostkeys/ssh-rsa',
},
wait=system.return_when.ALL_ENDED,
)
except Exception:
this.sleep(1)
else:
if last_try_guest_attribute.load('status') == 'ENDED_SUCCESS':
break
else:
return this.error('did not find hostkey within 60 tries')
hostkey = last_try_guest_attribute.get(
'output_value')['result']['queryValue']['items'][0]['value']
# setup the SSH connector for the VM
system.connector(f'{connector_name}-ssh').save(
connector_type='SSH',
value={
'hostname': external_ip,
'hostkey': hostkey,
'username': '******',
'key': setup_user_priv,
'script_timeout': 600,
'interpreter': '/usr/bin/env bash -ex',
},
)
# Run a little bash script on the VM:
last_try_script_echo = None
for _ in range(20):
if last_try_script_echo is not None:
last_try_script_echo.archive()
try:
last_try_script_echo = this.connect(
f'{connector_name}-ssh',
script="""
echo "litte test content" >> ~/test_file.txt
cat ~/test_file.txt
rm ~/test_file.txt
""",
name='run test script on new vm',
)
except:
this.sleep(1)
else:
if last_try_script_echo.load('status') == 'ENDED_SUCCESS':
break
else:
return this.error('Failed to run the little test script.')
script_echo = last_try_script_echo.get('output_value')['report']
this.set_output(f'Echo from test-script:\n{script_echo}')
# Save flow state and pause, manually resume this script's execution to remove the VM
this.save(
message=
'The VM was successfully deployed. Resume this execution to remove the VM'
)
this.pause()
# Remove the VM
operation = this.connect(
'gcloud',
name=f'remove {connector_name}',
api_name='compute',
api_version='v1',
collection='instances',
request='delete',
params={
'project': project_id,
'zone': server_location,
'instance': connector_name,
},
).get('output_value')['result']
this.flow(
'google_operation_wait',
operation=operation,
)
return this.success('all done')