def handler(system: flow_api.System, this: flow_api.Execution):
"""
Clone a git repository, create Cloudomation objects from the files.
All .py files in the flows/ subdirectory will be stored as Flows.
All .yaml files in the settings/ subdirectory will be stored as Settings.
All files in the files/ subdirectory will be stored as Files.
"""
inputs = this.get('input_value')
# this flow is registered as webhook, triggered by a commit to the
# repository. The commit sha is passed in .json.commit_sha
# when started manually, it will sync from master
try:
ref = inputs['json']['commit_sha']
except (KeyError, TypeError):
ref = 'master'
# read the connection information of the private repository
repo_info = system.setting('private git repo').get('value')
# the git 'get' command fetches the content of the repository.
# since no files_path is passed, the files will be returned in the
# output_value of the task
files = this.connect(
connector_type='GIT',
command='get',
repository_url=repo_info['repository_url'],
httpCookie=repo_info['httpCookie'],
ref=ref,
).get('output_value')['files']
# iterate over all files
for file_ in files:
# split the path and filename
path, filename = os.path.split(file_['name'])
# split the filename and file extension
name, ext = os.path.splitext(filename)
if path == 'flows' and ext == '.py':
# decode the base64 file content to text
text_content = base64.b64decode(file_['content']).decode()
# create or update Flow object
system.flow(name).save(script=text_content)
elif path == 'settings' and ext == '.yaml':
# decode the base64 file content to text
text_content = base64.b64decode(file_['content']).decode()
# load the yaml string in the file content
value = yaml.safe_load(text_content)
# create or update Setting object
system.setting(name).save(value=value)
elif path == 'files':
# create or update File object
# we use the name with the extension
# pass the base64 encoded binary file content directly
system.file(filename).save(content=file_['content'],
convert_binary=False)
return this.success('all done')
def handler(system: flow_api.System, this: flow_api.Execution):
"""
Clone a git repository, create Cloudomation objects from the files.
All .py files in the flows/ subdirectory will be stored as Flows.
All .yaml files in the settings/ subdirectory will be stored as Settings.
All files in the files/ subdirectory will be stored as Files.
A pop-up message form will request all necessary user information for the repository.
The inputs are not checked for validity!
"""
# This is a pop-up message form, it will ask for some information
# The last element is an OK-button, after clicking it the
# values are passed further and the pop-up message closes.
git_login_form_response = system.message(
subject='Input Git repository information to connect',
message_type='POPUP',
body={
'type': 'object',
'properties': {
'username': {
'element': 'string',
'type': 'string',
'label': 'Enter your username for the GIT repository:',
'order': 1,
},
'password': {
'element': 'password',
'type': 'string',
'label': 'Enter your password for the GIT repository:',
'order': 2,
},
# Here an example input is shown in the field, but its value is empty:
'repository url': {
'element': 'string',
'type': 'string',
'label':
'Enter the URL for the repository you want to clone into Cloudomation:',
'example': 'https://repository.url/user/repository.git',
'order': 3,
},
# Here a default value is provided with the field:
'reference': {
'element': 'string',
'type': 'string',
'label':
'Enter the "branch" you want to clone from or a "commit sha":',
'default': 'develop',
'order': 4,
},
'Ok': {
'element': 'submit',
'label': 'OK',
'type': 'boolean',
'order': 5,
},
},
'required': [
'username',
'password',
'repository url',
'reference',
],
},
).wait().get('response')
git_username = git_login_form_response['username']
git_password = git_login_form_response['password']
git_repo_url = git_login_form_response['repository url']
git_reference = git_login_form_response['reference']
# Now we use a "GIT" task.
# The git 'get' command fetches the content of the repository.
# Since no files_path is passed, the files will be returned in the
# output_value of the task
files = None
try:
files = this.connect(
connector_type='GIT',
command='get',
repository_url=git_repo_url,
# httpCookie=repo_info.get('httpCookie'),
ref=git_reference,
username=git_username,
password=git_password,
).get('output_value')['files']
except flow_api.exceptions.DependencyFailedError as err:
# send the error message to the output of this execution:
return this.error(repr(err))
# iterate over all files and save them on Cloudomation:
for file_ in files:
# split the path and filename
path, filename = os.path.split(file_['name'])
# split the filename and file extension
name, ext = os.path.splitext(filename)
if 'flows' in path and ext == '.py':
# decode the base64 file content to text
text_content = base64.b64decode(file_['content']).decode()
# create or update Flow object
system.flow(name).save(script=text_content)
elif 'settings' in path and ext == '.yaml':
# decode the base64 file content to text
text_content = base64.b64decode(file_['content']).decode()
# load the yaml string in the file content
value = yaml.safe_load(text_content)
# create or update Setting object
system.setting(name).save(value=value)
elif 'files' in path:
# create or update File object
# we use the name with the extension
# pass the base64 encoded binary file content directly
system.file(filename).save(content=file_['content'],
convert_binary=False)
return this.success('all done')
def handler(system: flow_api.System, this: flow_api.Execution):
inputs = this.get('input_value')
gcloud_connector = inputs.get('gcloud_connector')
project_id = system.connector(gcloud_connector).get('value').get(
'key').get('project_id')
if project_id is None or gcloud_connector is None:
return this.error('Missing inputs')
try:
webpagetestserver = system.setting('webpagetestserver').get('value')
if webpagetestserver.get('active'):
this.set_output(wait=False)
return this.success('Server already active')
except:
webpagetestserver = system.setting('webpagetestserver')
this.save(message=f'waiting for webpagetestserver lock')
system.setting('webpagetestserver').acquire(timeout=None)
this.save(message=f'creating webpagetest server')
webpagetest_server_config_str = system.file(
'webpagetest-server.json').get_text_content()
webpagetest_server_config = json.loads(webpagetest_server_config_str)
webpagetest_server_config['name'] = 'webpagetest-server'
webpagetest_server_config['disks'][0]['deviceName'] = 'webpagetest-server'
webpagetest_server_config['disks'][0]['initializeParams'][
'sourceSnapshot'] = f'projects/{project_id}/global/snapshots/webpagetest-server-template'
launch_task = this.connect(
gcloud_connector,
name='launch webpagetest-server',
api_name='compute',
api_version='v1',
collection='instances',
request='insert',
params={
'body': webpagetest_server_config,
'project': project_id,
'zone': 'europe-west1-b',
},
wait=system.return_when.ALL_ENDED,
)
status = launch_task.get('status')
message = launch_task.get('message')
if status == 'ENDED_ERROR' and 'HttpError 409' in message:
return this.success('using existing webpagetestserver')
if status == 'ENDED_ERROR':
return this.error('failed to launch webpagetest-server')
operation = launch_task.get('output_value')['result']
this.flow(
'google_operation_wait',
operation=operation,
)
for _ in range(60):
instance_list = this.connect(
gcloud_connector,
name='find webpagetest-server',
api_name='compute',
api_version='v1',
collection='instances',
request='list',
params={
'filter': 'name=webpagetest-server',
'project': project_id,
'zone': 'europe-west1-b',
},
).get('output_value')
if len(instance_list['result'].get('items', [])) != 0:
break
this.sleep(1)
else:
return this.error('did not find webpagetest server within 60 tries')
external_ip = instance_list['result']['items'][0]['networkInterfaces'][0][
'accessConfigs'][0]['natIP']
webpagetestserver = system.setting('webpagetestserver').get('value')
if webpagetestserver is None:
webpagetestserver = {}
webpagetestserver['hostname'] = external_ip
webpagetestserver['created_at'] = datetime.timestamp(datetime.now())
webpagetestserver['active'] = True
system.setting('webpagetestserver').save(value=webpagetestserver)
this.set_output(wait=True) # wait until the docker containers are ready
return this.success('all done')
def handler(system: flow_api.System, this: flow_api.Execution):
DT_FORMAT = '%Y-%m-%dT%H:%M:%S.%f%z'
number_of_steps = 12
user_name = system.get_own_user().get('name')
client_name = system.get_own_client().get('name')
user_name = user_name[0].upper() + user_name[1:]
execution_name = ''
execution_status = ''
output_value_str = ''
setting_id = None
setting_name = ''
text = ''
iterations = 0
delay = 0
file_content = ''
message = ''
time_out = 30 * 60
poll_interval = 10
def get_message(step):
messages = [
textwrap.dedent( # step 1
f'''
### Hi {user_name}, Welcome to Cloudomation!
This wizard will guide you through some first steps to explore the functionality of Cloudomation.
You will be given a task and the wizard continues once the task is done.
If at any time you want to view the instructions again, switch to the <strong><i class="fa fa-envelope-o"></i> Messages</strong> resource.
<div class="alert alert-primary">
<strong class="d-block pb-1">Your first task:</strong>
<div><i class="fa fa-check-square-o"></i> Close this message by pressing "OK".</div>
</div>
'''),
textwrap.dedent( # step 2
f'''
The Cloudomation user-interface uses the following layout:
<div class="p-row p-justify-center mb-3">
<div class="p-col" style="border: 1px solid black; width: 300px; height: 150px">
<div class="p-row p-align-center p-justify-center p-1 bg-blue text-blue-contrast">Header</div>
<div class="p-row c-grow">
<div class="p-row p-align-center p-justify-center p-1 bg-green text-green-contrast" style="width: 15%; writing-mode: vertical-lr">Navigation</div>
<div class="p-col c-grow">
<div class="p-row p-align-center p-justify-center p-1 bg-yellow text-yellow-contrast">Action bar</div>
<div class="p-row c-grow p-align-center p-justify-center p-1 bg-light-darker text-light-darker-contrast">Main area</div>
</div>
</div>
</div>
</div>
The navigation is structured in levels:
- Categories: e.g. **- App - <i class="fa fa-chevron-down"></i>**
- Resources: e.g. **<i class="fa fa-desktop"></i> Executions**
- Opened records: e.g. <i class="fa fa-file-code-o"></i> my first flow
<div class="alert alert-primary">
<strong class="d-block pb-1">Your tasks:</strong>
<div><i class="fa fa-check-square-o"></i> Familiarize yourself with the components of the user-interface.</div>
<div><i class="fa fa-check-square-o"></i> Select the <strong><i class="fa fa-code"></i> Flows</strong> resource in the <strong>- App -</strong> category.</div>
<div><i class="fa fa-check-square-o"></i> Create a new flow by pressing <span class="text-primary"><i class="fa fa-file-o"></i> New</span> in the action bar.</div>
</div>
'''),
textwrap.dedent( # step 3
f'''
A flow contains the automation logic which is processed by Cloudomation when the flow is run.
You can learn more about flows in the [Flows](/documentation/Flows){{ext}} documentation.
The newly created flow is opened for you.
You can see all opened flow records below the <strong><i class="fa fa-code"></i> Flows</strong> resource in the menu on the left.
Flows contain two important fields:
- The flow name
- The flow script
Flow names must be unique among all flows.
The flow script has to be Python code with a handler function.
There are many Python tutorials out there, for example [The Python Tutorial](https://docs.python.org/3/tutorial/){{ext}} by the Python Software Foundation
<div class="alert alert-info p-row p-align-center">
<i class="fa fa-info fa-2x text-info pr-3"></i> The rest of the wizard assumes basic understanding of Python syntax.
</div>
The new flow script already contains the handler function and some boilerplate.
<div class="alert alert-primary">
<strong class="d-block pb-1">Your tasks:</strong>
<div><i class="fa fa-check-square-o"></i> Change the name of the flow to something more useful, like "my first flow".</div>
<div><i class="fa fa-check-square-o"></i> Replace the script with the following:
<pre class="ml-3"><code>import flow_api
def handler(system: flow_api.System, this: flow_api.Execution):
this.log('Hello {user_name}!')
return this.success('all done')</code></pre>
</div>
<div><i class="fa fa-check-square-o"></i> Create an execution of the flow by pressing <span class="text-success"><i class="fa fa-play"></i> Try</span> in the action bar.</div>
</div>
'''),
textwrap.dedent( # step 4
f'''
**Congratulations!**
Your execution `{execution_name}` just ended with status `{execution_status}`.
When you pressed <span class="text-success"><i class="fa fa-play"></i> Try</span>, the user-interface switched to the executions list.
**The difference between <span class="text-success"><i class="fa fa-play"></i> Try</span> and <span class="text-success"><i class="fa fa-play"></i> Run</span>**
Both actions start the execution of your flow. For development, you can use <span class="text-success"><i class="fa fa-play"></i> Try</span> in the browser.
<span class="text-success"><i class="fa fa-play"></i> Run</span> is meant for productive use.
**Executions**
You can see an overview of all running executions in the execution list.
By clicking on the name of your execution, you open the execution record view.
When scrolling down in the execution record view, you will see the "Outputs" field which contains:
```yaml
{textwrap.indent(output_value_str,' ')}
```
---
Next, let's look at the <strong><i class="fa fa-sliders"></i> Settings</strong> resource.
<div class="alert alert-primary">
<strong class="d-block pb-1">Your tasks:</strong>
<div><i class="fa fa-check-square-o"></i> Select the <strong><i class="fa fa-sliders"></i> Settings</strong> resource.</div>
<div><i class="fa fa-check-square-o"></i> Create a new setting by pressing <span class="text-primary"><i class="fa fa-file-o"></i> New</span> in the action bar.</div>
</div>
'''),
textwrap.dedent( # step 5
f'''
Just like flows, settings contain two important fields:
- The setting name
- The setting value
Setting names must be unique among all settings.
The setting value must be a valid [YAML document](https://learnxinyminutes.com/docs/yaml/){{ext}}.
<div class="alert alert-primary">
<strong class="d-block pb-1">Your tasks:</strong>
<div><i class="fa fa-check-square-o"></i> Change the name of the setting to something more useful, like "test setting".</div>
<div><i class="fa fa-check-square-o"></i> Replace the setting's value with the following:
<pre class="ml-3"><code>text: Hello {user_name}!
iterations: 10
delay: 6</code></pre>
</div>
<div><i class="fa fa-check-square-o"></i> Save the setting by pressing <span class="text-primary"><i class="fa fa-save"></i> Save</span> in the action bar.</div>
</div>
'''),
textwrap.dedent( # step 6
f'''
Let's create another flow which uses the newly created setting.
This flow will read the values of `text`, `iterations`, and `delay` from the setting.
With the values you put in the setting it will log the text "{text}" "{iterations}" times with a delay of "{delay}" seconds.
<div class="alert alert-primary">
<strong class="d-block pb-1">Your tasks:</strong>
<div><i class="fa fa-check-square-o"></i> Create a new flow.</div>
<div><i class="fa fa-check-square-o"></i> Paste the following script:
<pre class="ml-3"><code>import flow_api
def handler(system: flow_api.System, this: flow_api.Execution):
# read the value of the setting record
test_setting = system.setting('{setting_name}').get('value')
# loop `iterations` times
for i in range(test_setting['iterations']):
# log the text
this.log(test_setting['text'])
# sleep for `delay` seconds
this.sleep(test_setting['delay'])
return this.success('all done')</code></pre>
</div>
<div><i class="fa fa-check-square-o"></i> Create an execution of the flow by pressing <span class="text-success"><i class="fa fa-play"></i> Try</span> in the action bar.</div>
</div>
'''),
textwrap.dedent( # step 7
f'''
Your execution "{execution_name}" is now running. It will take about {iterations * delay} seconds to end.
In the <strong><i class="fa fa-fw fa-desktop"></i> Executions</strong> resource you can see the list of all your executions.
There you can monitor the current status of your executions.
<div class="alert alert-primary">
<strong class="d-block pb-1">Your tasks:</strong>
<div><i class="fa fa-check-square-o"></i> Switch to the <strong><i class="fa fa-fw fa-desktop"></i> Executions</strong> resource. You'll see a list of all your executions.</div>
<div><i class="fa fa-check-square-o"></i> Select your execution by checking it in the list: <i class="fa fa-check-square-o"></i></div>
<div><i class="fa fa-check-square-o"></i> Pause the execution by pressing <span class="text-primary"><i class="fa fa-pause"></i> Pause</span> in the action bar.</div>
</div>
'''),
textwrap.dedent( # step 8
f'''
Let's recap what we've already seen:
**The user-interface**
- Layout: There are four main areas: header, navigation, main area, and action bar.
- The **- App -** category lists resources: Executions, Flows, Settings, and others.
- Selecting a resource shows a list of all records in the main area.
- It is possible to check records in the list view and perform actions on _all_ checked records at once.
- In the record view more details are seen and the record can be modified.
**The resources**
- Flows contain automation logic.
- Executions are created by "running" or "trying" a flow.
- Flows can access settings to read or write values.
<div class="alert alert-primary">
<strong class="d-block pb-1">Your task:</strong>
<div><i class="fa fa-check-square-o"></i> Close this message by pressing "OK".</div>
</div>
'''),
textwrap.dedent( # step 9
f'''
Now, let's see how to work with files.
The **Files** section is right beneath the settings section on the left.
In this section, you can store arbitrary files like shell scripts, markup files, zip files etc.
All these files can be loaded and manipulated through flow scripts.
<div class="alert alert-primary">
<strong class="d-block pb-1">Your tasks:</strong>
<div><i class="fa fa-check-square-o"></i> Create a file by pressing <span class="text-primary"><i class="fa fa-file-o"></i> New</span> in the action bar.</div>
<div><i class="fa fa-check-square-o"></i> Rename the file to <code>myfile.txt</code> and write something into the text field and press <span class="text-primary"><i class="fa fa-save"></i> Save</span> in the action bar.</div>
<div><i class="fa fa-check-square-o"></i> Create a new flow and paste the following script into the code editor:
<pre class="ml-3"><code>import flow_api
def handler(system: flow_api.System, this: flow_api.Execution):
# read the value of the file
read_file = system.file(
name='myfile.txt'
)
# print contents of the file to the log output
this.log(read_file.get('content'))
return this.success(message='all done')</code></pre>
</div>
<div><i class="fa fa-check-square-o"></i> Create an execution of the flow by pressing <span class="text-success"><i class="fa fa-play"></i> Try</span> in the action bar.</div>
</div>
'''),
textwrap.dedent( # step 10
f'''
You can see the content of the file in the **Outputs** section in the execution <code>{execution_name}</code>:
<pre class="ml-3"><code>{file_content}</code></pre>
You can learn more about files under [Files](/documentation/File%20handling%20with%20Cloudomation){{ext}} in the documentation.
---
Next, let's explore tasks.
As already covered, flows contain the automation logic.
Tasks on the other hand provide a way to access outside systems.
There are several task types available, each of which specializes on a certain protocol to talk with outside systems.
The `REST` task for example can be used to communicate with any REST service. Let's try!
<div class="alert alert-primary">
<strong class="d-block pb-1">Your tasks:</strong>
<div><i class="fa fa-check-square-o"></i> Create a new flow.</div>
<div><i class="fa fa-check-square-o"></i> Paste the following script:
<pre><code>import flow_api
def handler(system: flow_api.System, this: flow_api.Execution):
# create a REST task and run it
task = this.connect(connector_type='REST', url='https://api.icndb.com/jokes/random')
# access a field of the JSON response
joke = task.get('output_value')['json']['value']['joke']
# end with a joke
return this.success(message=joke)</code></pre>
</div>
<div><i class="fa fa-check-square-o"></i> Create an execution of the flow by pressing <span class="text-success"><i class="fa fa-play"></i> Try</span> in the action bar.</div>
</div>
'''),
textwrap.dedent( # step 11
f'''
Your execution `{execution_name}` ended with the message:
{message}
Aside from learning something fascinating about Chuck Norris, we also used two additional important features of Cloudomation:
- Child executions
- Execution inputs and outputs
Let's look at line 3 of the flow `{execution_name}` you've just created:
```python
task = this.connect(connector_type='REST', url='https://api.icndb.com/jokes/random')
```
There are several things to notice:
- `this.task` instructs Cloudomation to create a task execution which is a child of the currently running execution.
- `REST` specifies which task type to use.
- `url=...` passes an input value named `url` to the task.
All task types provide access to outside systems and accept a different set of inputs.
You can learn more about the available task types in the [Tasks](/documentation/Tasks){{ext}} documentation.
Once the child execution ends, your flow continues running.
The child execution provides output values which can be used by your flow.
This we can see in line 5:
```python
joke = task.get('output_value')['json']['value']['joke']
```
- `task.get` instructs Cloudomation to read a field of the child execution.
- `output_value` specifies which field to read.
- `['json']['value']['joke']` selects the part of the output value we are interested in.
<div class="alert alert-primary">
<strong class="d-block pb-1">Your task:</strong>
<div><i class="fa fa-check-square-o"></i> Close this message by pressing "OK".</div>
</div>
'''),
textwrap.dedent( # step 12
f'''
We've reached the end of the first steps wizard. Here are some links you can follow to deepen your understanding of Cloudomation:
- Learn more about using Cloudomation in the [Tutorial](https://cloudomation.com/documentation/tutorial/){{ext}}
- [Invite users](/users){{ext}} to join your Cloudomation client "{client_name}"
- Set up [integration with Git](https://cloudomation.com/documentation/using-flow-scripts-from-git/){{ext}}
- Read the flow [script function reference](/documentation/Flow%20script%20function%20reference){{ext}}
- Explore available [task types](/documentation/Tasks){{ext}}
- Set up [webhooks](https://cloudomation.com/documentation/webhooks/){{ext}}.
- If there are any questions or issues, please do not hesitate to [contact us](/contact){{ext}}.
Enjoy automating!
'''),
]
return messages[step - 1]
for step in range(1, number_of_steps + 1):
m = system.message(subject=f'Step {step} of {number_of_steps}',
message_type='POPUP',
body={
'type': 'object',
'properties': {
'content': {
'element': 'markdown',
'description': get_message(step),
'order': 1,
},
'acknowledged': {
'label': 'OK',
'element': 'submit',
'type': 'boolean',
'order': 2,
},
},
'required': [
'acknowledged',
],
})
m.wait()
dt_now = datetime.datetime.now(tz=datetime.timezone.utc)
if step == 2:
start_time = time.time()
flow = None
while flow is None and time.time() < start_time + time_out:
for cur_flow in system.flows():
try:
if cur_flow.get(
'created_at'
) is not None and datetime.datetime.strptime(
cur_flow.get('created_at'),
DT_FORMAT) > dt_now:
flow = cur_flow
break
except Exception:
pass
this.sleep(poll_interval)
elif step == 3 or step == 6:
start_time = time.time()
execution = None
while execution is None and time.time() < start_time + time_out:
for cur_execution in system.executions():
try:
if cur_execution.get(
'created_at'
) is not None and datetime.datetime.strptime(
cur_execution.get('created_at'),
DT_FORMAT) > dt_now:
execution = cur_execution
break
except Exception:
pass
this.sleep(poll_interval)
if step == 3:
execution.wait()
execution_name, execution_status, output_value = execution.load(
'name', 'status', 'output_value')
output_value_str = yaml.safe_dump(output_value,
default_flow_style=False)
else:
execution_name = execution.load('name')
elif step == 4:
start_time = time.time()
setting = None
while setting is None and time.time() < start_time + time_out:
for cur_setting in system.settings():
try:
if cur_setting is not None and cur_setting.get(
'created_at'
) is not None and datetime.datetime.strptime(
cur_setting.get('created_at'),
DT_FORMAT) > dt_now:
setting = cur_setting
break
except Exception:
pass
this.sleep(poll_interval)
setting_id = setting.get('id')
elif step == 5:
start_time = time.time()
before_modified_at = setting.load('modified_at')
while time.time() < start_time + time_out:
setting = None
for cur_setting in system.settings():
try:
if cur_setting is not None and cur_setting.load(
'modified_at') > before_modified_at:
setting = cur_setting
break
except Exception:
pass
if setting is None:
continue
modified_at = setting.load('modified_at')
if modified_at is not None and modified_at > before_modified_at:
before_modified_at = modified_at
value = setting.get('value')
if all(k in value
for k in ('text', 'iterations', 'delay')):
break
else:
check_m = system.message(
subject=
f'Please check - Step {step} of {number_of_steps}',
message_type='POPUP',
body={
'type': 'object',
'properties': {
'content': {
'element':
'markdown',
'description':
textwrap.dedent(f'''
Please make sure that the setting contains all the keys:
- text
- iterations
- delay
Like in this example:
```yaml
text: Hello {user_name}!
iterations: 10
delay: 6
```
The values of the keys will be used in the next step.
'''),
'order':
1,
},
'acknowledged': {
'label': 'OK',
'element': 'submit',
'type': 'boolean',
'order': 2,
},
},
'required': [
'acknowledged',
],
})
check_m.wait()
continue
this.sleep(poll_interval)
setting_name, setting_value = setting.load('name', 'value')
text = setting_value['text']
iterations = setting_value['iterations']
delay = setting_value['delay']
elif step == 7:
start_time = time.time()
while time.time() < start_time + time_out:
if execution.load('status') in ('PAUSED', 'ENDED_SUCCESS',
'ENDED_ERROR',
'ENDED_CANCELLED'):
break
this.sleep(poll_interval)
elif step == 9:
start_time = time.time()
file_content = None
execution = None
while time.time() < start_time + time_out:
if file_content is None:
for cur_files in system.files():
try:
if cur_files.load('name') == 'myfile.txt':
myfile = system.file('myfile.txt')
file_content = myfile.get('content')
except Exception:
pass
if execution is None:
for cur_execution in system.executions():
try:
if cur_execution.get(
'created_at'
) is not None and datetime.datetime.strptime(
cur_execution.get('created_at'),
DT_FORMAT) > dt_now:
execution = cur_execution
except Exception:
pass
if file_content and execution:
break
this.sleep(poll_interval)
execution_name = execution.load('name')
elif step == 10:
start_time = time.time()
task = None
execution = None
while time.time() < start_time + time_out:
for cur_execution in system.executions():
try:
created_at = cur_execution.get('created_at')
if created_at is not None and datetime.datetime.strptime(
created_at, DT_FORMAT) > dt_now:
if cur_execution.get('type') == 'TASK':
task = cur_execution
else:
execution = cur_execution
except Exception:
pass
if task and execution:
break
this.sleep(poll_interval)
execution.wait(return_when=system.return_when.ALL_SUCCEEDED)
execution_name = execution.load('name')
message = unescape(
task.get('output_value')['json']['value']['joke'])
return this.success('all done')
def handler(system: flow_api.System, this: flow_api.Execution):
inputs = this.get('input_value')
gcloud_connector = inputs.get('gcloud_connector')
project_id = system.connector(gcloud_connector).get('value').get(
'key').get('project_id')
if project_id is None or gcloud_connector is None:
return this.error('Missing inputs')
webpagetestserver_template_setting = system.setting(
'webpagetest-server-template')
webpagetestserver_template_setting.acquire(timeout=None)
instance_list = this.connect(
gcloud_connector,
name='find webpagetest-server-template',
api_name='compute',
api_version='v1',
collection='instances',
request='list',
params={
'filter': 'name=webpagetest-server-template',
'project': project_id,
'zone': 'europe-west1-b',
},
).get('output_value')
if len(instance_list['result'].get('items', [])) > 0:
this.connect(
gcloud_connector,
name='delete webpagetest-template',
api_name='compute',
api_version='v1',
collection='instances',
request='delete',
params={
'instance': 'webpagetest-server-template',
'project': project_id,
'zone': 'europe-west1-b',
},
)
for _ in range(60):
instance_list = this.connect(
gcloud_connector,
name='find webpagetest-server-template',
api_name='compute',
api_version='v1',
collection='instances',
request='list',
params={
'filter': f'name=webpagetest-server-template',
'project': project_id,
'zone': 'europe-west1-b',
},
).get('output_value')
if len(instance_list['result'].get('items', [])) == 0:
break
this.sleep(1)
else:
return this.error(
'old webpagetest-server-template still exists after 60 tries')
images = this.connect(gcloud_connector,
name='find ubuntu image name',
api_name='compute',
api_version='v1',
collection='images',
request='list',
params={
'project': 'ubuntu-os-cloud'
}).load('output_value')['result']['items']
image_name = None
for image in images:
with contextlib.suppress(KeyError):
if image['deprecated']['state'] == 'DEPRECATED':
continue
if image['name'].startswith('ubuntu-1804-bionic-'):
image_name = image['name']
break
setup_user_key = RSA.generate(2048)
setup_user_priv = setup_user_key.export_key().decode()
setup_user_pub = setup_user_key.publickey().export_key(
format='OpenSSH').decode()
setup_user_key = None
webpagetest_user_key = RSA.generate(2048)
webpagetest_user_priv = webpagetest_user_key.export_key().decode()
webpagetest_user_pub = webpagetest_user_key.publickey().export_key(
format='OpenSSH').decode()
webpagetest_user_key = None
webpagetest_server_startup_script = system.file(
'webpagetest-server-template-startup.sh').get_text_content()
webpagetest_server_startup_script = webpagetest_server_startup_script.format(
setup_user_pub=setup_user_pub,
webpagetest_user_pub=webpagetest_user_pub,
)
webpagetest_server_config_str = system.file(
'webpagetest-server-template.json').get_text_content()
webpagetest_server_config_str.replace('{PROJECT_ID}', project_id)
webpagetest_server_config = json.loads(webpagetest_server_config_str)
webpagetest_server_config['metadata']['items'][0][
'value'] = webpagetest_server_startup_script
webpagetest_server_config['name'] = 'webpagetest-server-template'
webpagetest_server_config['disks'][0][
'deviceName'] = 'webpagetest-server-template'
webpagetest_server_config['disks'][0]['initializeParams'][
'sourceImage'] = f'projects/ubuntu-os-cloud/global/images/{image_name}'
operation = this.connect(
gcloud_connector,
name='launch webpagetest-server-template',
api_name='compute',
api_version='v1',
collection='instances',
request='insert',
params={
'body': webpagetest_server_config,
'project': project_id,
'zone': 'europe-west1-b',
},
).get('output_value')['result']
this.flow(
'google_operation_wait',
operation=operation,
)
try:
for _ in range(60):
instance_list = this.connect(
gcloud_connector,
name='find webpagetest-server-template',
api_name='compute',
api_version='v1',
collection='instances',
request='list',
params={
'filter': 'name=webpagetest-server-template',
'project': project_id,
'zone': 'europe-west1-b',
},
).get('output_value')
if len(instance_list['result'].get('items', [])) != 0:
break
this.sleep(1)
else:
return this.error(
'did not find webpagetest server template within 60 tries')
external_ip = instance_list['result']['items'][0]['networkInterfaces'][
0]['accessConfigs'][0]['natIP']
failed_tries = []
for _ in range(60):
this.sleep(1)
guest_attribute_task = this.connect(
gcloud_connector,
name='get webpagetest-server-template hostkey',
api_name='compute',
api_version='beta',
collection='instances',
request='getGuestAttributes',
params={
'instance': 'webpagetest-server-template',
'queryPath': 'hostkeys/ssh-rsa',
'project': project_id,
'zone': 'europe-west1-b',
},
wait=system.return_when.ALL_ENDED,
)
if guest_attribute_task.load('status') == 'ENDED_SUCCESS':
break
failed_tries.append(guest_attribute_task)
else:
return this.error('did not find hostkey within 60 tries')
for failed_try in failed_tries:
failed_try.delete()
hostkey = guest_attribute_task.get(
'output_value')['result']['queryValue']['items'][0]['value']
system.connector('webpagetestserver-setup').save(
connector_type='SSH',
value={
'key': setup_user_priv,
'username': '******',
'hostname': external_ip,
'hostkey': hostkey,
},
)
system.connector('webpagetestserver').save(
connector_type='SSH',
value={
'key': webpagetest_user_priv,
'username': '******',
'hostname': external_ip,
'hostkey': hostkey,
},
)
system.connector('webpagetestserver-copy').save(
connector_type='SCP',
value={
'key': webpagetest_user_priv,
'username': '******',
'hostname': external_ip,
'hostkey': hostkey,
},
)
webpagetestserver_template = {
'hostname': external_ip,
'hostkey': hostkey,
'setup-user': {
'username': '******',
'ssh-key': setup_user_priv,
'public-key': setup_user_pub,
},
'webpagetest-user': {
'username': '******',
'ssh-key': webpagetest_user_priv,
'public-key': webpagetest_user_pub,
},
}
for _ in range(10):
try:
this.connect(
'webpagetestserver-setup',
script="""
sudo apt-get update
sudo apt-get install -y --no-install-recommends python-pip python3-pip
sudo apt-get install -y --no-install-recommends build-essential python3-dev iftop iotop zip unzip
# DO NOT remove python3-cryptography! or instance will not be able to setup networking on boot
# sudo apt-get remove -y python3-cryptography
""",
name='install packages',
)
except Exception:
this.sleep(3)
else:
break
else:
return this.error(
'failed to ssh to webpagetestserver-template within 10 tries')
this.connect(
'webpagetestserver',
script="""
pip install wheel setuptools
pip3 install wheel setuptools
pip3 install alembic psycopg2-binary
""",
name='install pip packages',
)
this.connect(
'webpagetestserver-setup',
script="""
set +e
ssh -V
ret=$?
set -e
if [ "$ret" -ne "0" ]; then
sudo apt-get install -y --no-install-recommends openssh-client
fi
""",
name='install openssh client',
)
this.connect(
'webpagetestserver-setup',
script="""
set +e
jq -V
ret=$?
set -e
if [ "$ret" -ne "0" ]; then
sudo apt-get install -y --no-install-recommends jq
fi
set +e
jq -V
ret=$?
set -e
if [ "$ret" -ne "0" ]; then
sudo snap install jq
fi
""",
name='install jq',
)
this.connect(
'webpagetestserver-setup',
script="""
set +e
yq -V
ret=$?
set -e
if [ "$ret" -ne "0" ]; then
sudo snap install yq
fi
""",
name='install yq',
)
this.connect(
'webpagetestserver-setup',
script="""
set +e
docker --version
ret=$?
set -e
if [ "$ret" -ne "0" ]; then
curl -fsSL https://get.docker.com -o get-docker.sh
sh get-docker.sh
RELEASE=$(lsb_release -is | tr '[:upper:]' '[:lower:]')
sudo apt-get install -y --no-install-recommends \\
apt-transport-https \\
ca-certificates \\
curl \\
gnupg2 \\
software-properties-common
curl -fsSL https://download.docker.com/linux/${RELEASE}/gpg | sudo apt-key add -
sudo add-apt-repository \\
"deb [arch=amd64] https://download.docker.com/linux/${RELEASE} \\
$(lsb_release -cs) \\
stable"
sudo apt-get update
sudo apt-get install -y --no-install-recommends docker-ce
fi
sudo usermod -a -G docker webpagetest-user
""",
name='install docker',
)
this.connect(
'webpagetestserver',
name='create webpagetest folders',
script="""
mkdir -p webpagetest/server
mkdir -p webpagetest/agent
""",
)
this.connect(
'webpagetestserver-copy',
name='copy webpagetest server Dockerfile',
src='cloudomation:webpagetest-server-Dockerfile',
dst='webpagetest/server/Dockerfile',
)
this.connect(
'webpagetestserver-copy',
name='copy webpagetest server locations.ini',
src='cloudomation:webpagetest-server-locations.ini',
dst='webpagetest/server/locations.ini',
)
this.connect(
'webpagetestserver-copy',
name='copy webpagetest agent Dockerfile',
src='cloudomation:webpagetest-agent-Dockerfile',
dst='webpagetest/agent/Dockerfile',
)
this.connect(
'webpagetestserver-copy',
name='copy webpagetest agent script.sh',
src='cloudomation:webpagetest-agent-script.sh',
dst='webpagetest/agent/script.sh',
)
this.connect(
'webpagetestserver',
name='create webpagetest server and agent',
input_value={'script_timeout': 600},
script="""
pushd webpagetest/server
docker build -t wpt-server .
docker run -d -p 4000:80 --restart unless-stopped wpt-server
popd
pushd webpagetest/agent
chmod u+x script.sh
docker build -t wpt-agent .
docker run -d -p 4001:80 \\
--restart unless-stopped \\
--network="host" \\
-e "SERVER_URL=http://localhost:4000/work/" \\
-e "LOCATION=Test" \\
wpt-agent
""",
)
this.connect(
gcloud_connector,
name='stop webpagetest-server-template',
api_name='compute',
api_version='v1',
collection='instances',
request='stop',
params={
'instance': 'webpagetest-server-template',
'project': project_id,
'zone': 'europe-west1-b',
},
)
try:
operation = this.connect(
gcloud_connector,
name='delete old webpagetest-server-template snapshot',
api_name='compute',
api_version='v1',
collection='snapshots',
request='delete',
params={
'snapshot': 'webpagetest-server-template',
'project': project_id,
},
).get('output_value')['result']
this.flow(
'google_operation_wait',
operation=operation,
)
except Exception:
pass
today = date.today()
operation = this.connect(
gcloud_connector,
name='create webpagetest-server-template snapshot',
api_name='compute',
api_version='v1',
collection='disks',
request='createSnapshot',
params={
'disk': 'webpagetest-server-template',
'body': {
'name': 'webpagetest-server-template',
'description':
f'auto-generated at {today} by setup webpagetest server',
'storageLocations': ['europe-west1'],
},
'project': project_id,
'zone': 'europe-west1-b',
},
).get('output_value')['result']
this.flow(
'google_operation_wait',
operation=operation,
)
webpagetestserver_template.pop('hostname')
webpagetestserver_template.pop('hostkey')
webpagetestserver_template_setting.save(
value=webpagetestserver_template)
webpagetestserver_template_setting.release()
except Exception as ex:
this.save(message=repr(ex))
if this.get('input_value').get('interactive'):
this.sleep(120)
# this.pause()
raise
finally:
instance_list = this.connect(
gcloud_connector,
name='find webpagetest-server-template',
api_name='compute',
api_version='v1',
collection='instances',
request='list',
params={
'filter': f'name=webpagetest-server-template',
'project': project_id,
'zone': 'europe-west1-b'
},
).get('output_value')
if len(instance_list['result'].get('items', [])) > 0:
this.connect(
gcloud_connector,
name='delete webpagetest-server-template',
api_name='compute',
api_version='v1',
collection='instances',
request='delete',
params={
'instance': 'webpagetest-server-template',
'project': project_id,
'zone': 'europe-west1-b',
},
)
while True:
instance_list = this.connect(
gcloud_connector,
name='find webpagetest-server-template',
api_name='compute',
api_version='v1',
collection='instances',
request='list',
params={
'filter': 'name=webpagetest-server-template',
'project': project_id,
'zone': 'europe-west1-b',
},
).get('output_value')
if len(instance_list['result'].get('items', [])) == 0:
break
this.sleep(1)
return this.success('all done')
def handler(system: flow_api.System, this: flow_api.Execution):
"""
In this example we setup a Virtual Machine (VM) in the google-cloud.
Then, Ubuntu will be installed and a small bash script is executed.
Bash scripts can be used to install more software on the VM or
change settings, anything you could do on a local machine in a terminal.
This flow also requires the two files:
* create google cloud vm example.sh
* create google cloud vm example.json
In the future, you can take these as an example, adopt them and
upload them with a new name in Cloudomation > Files.
"""
# This is a pop-up message form, it will ask for some information
# The last element is an OK-button, after clicking it the
# values are passed further and the pop-up message closes.
form_response = system.message(
subject='Input information to setup a VM in google-cloud',
message_type='POPUP',
body={
'type':
'object',
'properties': {
'connector_name': {
'element': 'string',
'type': 'string',
'label': 'Enter name for the connector: \n\
(This name can be searched for and is used to connect with your VM)',
'example': 'my-vm-on-gcloud',
'order': 1,
},
'project_id': {
'element': 'string',
'type': 'string',
'label': 'Enter your google cloud project-ID:',
'default': 'my-first-project-id',
'order': 2,
},
'machine_type': {
'element': 'string',
'type': 'string',
'label':
'Enter the machine type you want to use on google-cloud:',
'default': 'n1-standard-1',
'order': 3,
},
'server_location': {
'element': 'string',
'type': 'string',
'label':
'Enter the desired location of the hosting server from google-cloud:',
'default': 'europe-west1-b',
'order': 4,
},
'Ok': {
'element': 'submit',
'label': 'OK',
'type': 'boolean',
'order': 5,
},
},
'required': [
'connector_name',
'project_id',
'machine_type',
'server_location',
],
},
).wait().get('response')
connector_name = form_response['connector_name']
project_id = form_response['project_id']
machine_type = form_response['machine_type']
server_location = form_response['server_location']
# generate a ssh key-pair for the setup-user:
setup_user_key = RSA.generate(2048)
setup_user_priv = setup_user_key.export_key().decode()
setup_user_pub = setup_user_key.publickey().export_key(
format='OpenSSH').decode()
setup_user_key = None
# load the startup script and pass the public ssh key to the script:
vm_startup_script = system.file(
'create google cloud vm example.sh').get_text_content()
vm_startup_script = vm_startup_script.format(
setup_user_pub=setup_user_pub, )
# load the configuration for the VM and setup last parameters:
vm_config_str = system.file(
'create google cloud vm example.json').get_text_content()
vm_config = json.loads(vm_config_str)
vm_config['name'] = connector_name
vm_config['machineType'] = \
f'projects/{project_id}/zones/{server_location}/machineTypes/{machine_type}'
vm_config['zone'] = f'projects/{project_id}/zones/{server_location}'
vm_config['metadata']['items'][0]['value'] = vm_startup_script
vm_config['disks'][0]['deviceName'] = connector_name
vm_config['disks'][0]['initializeParams']['diskType'] = \
f'projects/{project_id}/zones/{server_location}/diskTypes/pd-ssd'
vm_config['networkInterfaces'][0]['subnetwork'] = \
f'projects/{project_id}/regions/{server_location[:-2]}/subnetworks/default'
# NOTE: two sources can be used for initial system installation:
# a) from 'sourceSnapshot'
# b) from 'sourceImage'
# This flow script will install Ubuntu, using the parameter 'sourceImage'.
# For installing a system from snapshot use the respective parameter.
# This paramter is set under: disks / initializeParams
# Connect to the VM
operation = this.connect(
'gcloud',
name=f'launch {connector_name}',
api_name='compute',
api_version='v1',
collection='instances',
request='insert',
params={
'project': project_id,
'zone': server_location,
'body': vm_config,
},
).get('output_value')['result']
this.flow(
'google_operation_wait',
operation=operation,
)
# retrieve external IP and hostkey (required for creating a SSH connector):
for _ in range(60):
try:
instance_list = this.connect(
'gcloud',
name=f'find {connector_name}',
api_name='compute',
api_version='v1',
collection='instances',
request='list',
params={
'project': project_id,
'zone': server_location,
'filter': f'name={connector_name}',
},
).get('output_value')
except flow_api.DependencyFailedError:
this.sleep(1)
if len(instance_list['result'].get('items', [])) != 0:
break
this.sleep(1)
else:
return this.error('did not find VM within 60 tries')
external_ip = instance_list['result']['items'][0]['networkInterfaces'][0][
'accessConfigs'][0]['natIP']
last_try_guest_attribute = None
for _ in range(60):
if last_try_guest_attribute is not None:
last_try_guest_attribute.archive()
try:
last_try_guest_attribute = this.connect(
'gcloud',
name=f'get {connector_name} hostkey',
api_name='compute',
api_version='v1',
collection='instances',
request='getGuestAttributes',
params={
'project': project_id,
'zone': server_location,
'instance': connector_name,
'queryPath': 'hostkeys/ssh-rsa',
},
wait=system.return_when.ALL_ENDED,
)
except Exception:
this.sleep(1)
else:
if last_try_guest_attribute.load('status') == 'ENDED_SUCCESS':
break
else:
return this.error('did not find hostkey within 60 tries')
hostkey = last_try_guest_attribute.get(
'output_value')['result']['queryValue']['items'][0]['value']
# setup the SSH connector for the VM
system.connector(f'{connector_name}-ssh').save(
connector_type='SSH',
value={
'hostname': external_ip,
'hostkey': hostkey,
'username': '******',
'key': setup_user_priv,
'script_timeout': 600,
'interpreter': '/usr/bin/env bash -ex',
},
)
# Run a little bash script on the VM:
last_try_script_echo = None
for _ in range(20):
if last_try_script_echo is not None:
last_try_script_echo.archive()
try:
last_try_script_echo = this.connect(
f'{connector_name}-ssh',
script="""
echo "litte test content" >> ~/test_file.txt
cat ~/test_file.txt
rm ~/test_file.txt
""",
name='run test script on new vm',
)
except:
this.sleep(1)
else:
if last_try_script_echo.load('status') == 'ENDED_SUCCESS':
break
else:
return this.error('Failed to run the little test script.')
script_echo = last_try_script_echo.get('output_value')['report']
this.set_output(f'Echo from test-script:\n{script_echo}')
# Save flow state and pause, manually resume this script's execution to remove the VM
this.save(
message=
'The VM was successfully deployed. Resume this execution to remove the VM'
)
this.pause()
# Remove the VM
operation = this.connect(
'gcloud',
name=f'remove {connector_name}',
api_name='compute',
api_version='v1',
collection='instances',
request='delete',
params={
'project': project_id,
'zone': server_location,
'instance': connector_name,
},
).get('output_value')['result']
this.flow(
'google_operation_wait',
operation=operation,
)
return this.success('all done')