def handler(system: flow_api.System, this: flow_api.Execution): vault_config = 'my-vaultconfig' secret_path = 'my-git-secret' secret_key_user = '******' secret_key_password = '******' my_connector_name = 'git-example-connector' # Create connector of type GIT and link it with vault secrets system.connector(my_connector_name).save( connector_type='GIT', value={ 'username': f'vault.secret({secret_key_user})', 'password': f'vault.secret({secret_key_password})', }, vault_secrets=[{ 'vault_name': vault_config, 'engine_path': 'kv', 'secret_path': secret_path, }], ) # Use the connector: git_metadata = this.connect( my_connector_name, name='get git-metadata', repository_url='https://mygitrepo.com/repo/', command='metadata', ).get('output_value') this.log(git_metadata) return this.success('all done')
def handler(system: flow_api.System, this: flow_api.Execution): inputs = this.get('input_value') gcloud_connector = inputs.get('gcloud_connector') project_id = system.connector(gcloud_connector).get('value').get( 'key').get('project_id') if project_id is None or gcloud_connector is None: return this.error('Missing inputs') delete_task = this.connect( gcloud_connector, name='delete webpagetest-server', api_name='compute', api_version='v1', collection='instances', request='delete', params={ 'instance': 'webpagetest-server', 'project': project_id, 'zone': 'europe-west1-b', }, wait=system.return_when.ALL_ENDED, ) status = delete_task.get('status') message = delete_task.get('message') operation = delete_task.get('output_value').get('result') this.flow( 'google_operation_wait', operation=operation, ) system.setting('webpagetestserver').release() if status == 'ENDED_ERROR' and 'HttpError 404' not in message: return this.error('failed to delete webpagetest-server') webpagetestserver = system.setting('webpagetestserver').get('value') webpagetestserver['active'] = False system.setting('webpagetestserver').save(value=webpagetestserver) return this.success('all done')
def handler(system: flow_api.System, this: flow_api.Execution): inputs = this.get('input_value') gcloud_connector = inputs.get('gcloud_connector') project_id = system.connector(gcloud_connector).get('value').get( 'key').get('project_id') if project_id is None or gcloud_connector is None: return this.error('Missing inputs') try: webpagetestserver = system.setting('webpagetestserver').get('value') if webpagetestserver.get('active'): this.set_output(wait=False) return this.success('Server already active') except: webpagetestserver = system.setting('webpagetestserver') this.save(message=f'waiting for webpagetestserver lock') system.setting('webpagetestserver').acquire(timeout=None) this.save(message=f'creating webpagetest server') webpagetest_server_config_str = system.file( 'webpagetest-server.json').get_text_content() webpagetest_server_config = json.loads(webpagetest_server_config_str) webpagetest_server_config['name'] = 'webpagetest-server' webpagetest_server_config['disks'][0]['deviceName'] = 'webpagetest-server' webpagetest_server_config['disks'][0]['initializeParams'][ 'sourceSnapshot'] = f'projects/{project_id}/global/snapshots/webpagetest-server-template' launch_task = this.connect( gcloud_connector, name='launch webpagetest-server', api_name='compute', api_version='v1', collection='instances', request='insert', params={ 'body': webpagetest_server_config, 'project': project_id, 'zone': 'europe-west1-b', }, wait=system.return_when.ALL_ENDED, ) status = launch_task.get('status') message = launch_task.get('message') if status == 'ENDED_ERROR' and 'HttpError 409' in message: return this.success('using existing webpagetestserver') if status == 'ENDED_ERROR': return this.error('failed to launch webpagetest-server') operation = launch_task.get('output_value')['result'] this.flow( 'google_operation_wait', operation=operation, ) for _ in range(60): instance_list = this.connect( gcloud_connector, name='find webpagetest-server', api_name='compute', api_version='v1', collection='instances', request='list', params={ 'filter': 'name=webpagetest-server', 'project': project_id, 'zone': 'europe-west1-b', }, ).get('output_value') if len(instance_list['result'].get('items', [])) != 0: break this.sleep(1) else: return this.error('did not find webpagetest server within 60 tries') external_ip = instance_list['result']['items'][0]['networkInterfaces'][0][ 'accessConfigs'][0]['natIP'] webpagetestserver = system.setting('webpagetestserver').get('value') if webpagetestserver is None: webpagetestserver = {} webpagetestserver['hostname'] = external_ip webpagetestserver['created_at'] = datetime.timestamp(datetime.now()) webpagetestserver['active'] = True system.setting('webpagetestserver').save(value=webpagetestserver) this.set_output(wait=True) # wait until the docker containers are ready return this.success('all done')
def handler(system: flow_api.System, this: flow_api.Execution): inputs = this.get('input_value') gcloud_connector = inputs.get('gcloud_connector') project_id = system.connector(gcloud_connector).get('value').get( 'key').get('project_id') if project_id is None or gcloud_connector is None: return this.error('Missing inputs') webpagetestserver_template_setting = system.setting( 'webpagetest-server-template') webpagetestserver_template_setting.acquire(timeout=None) instance_list = this.connect( gcloud_connector, name='find webpagetest-server-template', api_name='compute', api_version='v1', collection='instances', request='list', params={ 'filter': 'name=webpagetest-server-template', 'project': project_id, 'zone': 'europe-west1-b', }, ).get('output_value') if len(instance_list['result'].get('items', [])) > 0: this.connect( gcloud_connector, name='delete webpagetest-template', api_name='compute', api_version='v1', collection='instances', request='delete', params={ 'instance': 'webpagetest-server-template', 'project': project_id, 'zone': 'europe-west1-b', }, ) for _ in range(60): instance_list = this.connect( gcloud_connector, name='find webpagetest-server-template', api_name='compute', api_version='v1', collection='instances', request='list', params={ 'filter': f'name=webpagetest-server-template', 'project': project_id, 'zone': 'europe-west1-b', }, ).get('output_value') if len(instance_list['result'].get('items', [])) == 0: break this.sleep(1) else: return this.error( 'old webpagetest-server-template still exists after 60 tries') images = this.connect(gcloud_connector, name='find ubuntu image name', api_name='compute', api_version='v1', collection='images', request='list', params={ 'project': 'ubuntu-os-cloud' }).load('output_value')['result']['items'] image_name = None for image in images: with contextlib.suppress(KeyError): if image['deprecated']['state'] == 'DEPRECATED': continue if image['name'].startswith('ubuntu-1804-bionic-'): image_name = image['name'] break setup_user_key = RSA.generate(2048) setup_user_priv = setup_user_key.export_key().decode() setup_user_pub = setup_user_key.publickey().export_key( format='OpenSSH').decode() setup_user_key = None webpagetest_user_key = RSA.generate(2048) webpagetest_user_priv = webpagetest_user_key.export_key().decode() webpagetest_user_pub = webpagetest_user_key.publickey().export_key( format='OpenSSH').decode() webpagetest_user_key = None webpagetest_server_startup_script = system.file( 'webpagetest-server-template-startup.sh').get_text_content() webpagetest_server_startup_script = webpagetest_server_startup_script.format( setup_user_pub=setup_user_pub, webpagetest_user_pub=webpagetest_user_pub, ) webpagetest_server_config_str = system.file( 'webpagetest-server-template.json').get_text_content() webpagetest_server_config_str.replace('{PROJECT_ID}', project_id) webpagetest_server_config = json.loads(webpagetest_server_config_str) webpagetest_server_config['metadata']['items'][0][ 'value'] = webpagetest_server_startup_script webpagetest_server_config['name'] = 'webpagetest-server-template' webpagetest_server_config['disks'][0][ 'deviceName'] = 'webpagetest-server-template' webpagetest_server_config['disks'][0]['initializeParams'][ 'sourceImage'] = f'projects/ubuntu-os-cloud/global/images/{image_name}' operation = this.connect( gcloud_connector, name='launch webpagetest-server-template', api_name='compute', api_version='v1', collection='instances', request='insert', params={ 'body': webpagetest_server_config, 'project': project_id, 'zone': 'europe-west1-b', }, ).get('output_value')['result'] this.flow( 'google_operation_wait', operation=operation, ) try: for _ in range(60): instance_list = this.connect( gcloud_connector, name='find webpagetest-server-template', api_name='compute', api_version='v1', collection='instances', request='list', params={ 'filter': 'name=webpagetest-server-template', 'project': project_id, 'zone': 'europe-west1-b', }, ).get('output_value') if len(instance_list['result'].get('items', [])) != 0: break this.sleep(1) else: return this.error( 'did not find webpagetest server template within 60 tries') external_ip = instance_list['result']['items'][0]['networkInterfaces'][ 0]['accessConfigs'][0]['natIP'] failed_tries = [] for _ in range(60): this.sleep(1) guest_attribute_task = this.connect( gcloud_connector, name='get webpagetest-server-template hostkey', api_name='compute', api_version='beta', collection='instances', request='getGuestAttributes', params={ 'instance': 'webpagetest-server-template', 'queryPath': 'hostkeys/ssh-rsa', 'project': project_id, 'zone': 'europe-west1-b', }, wait=system.return_when.ALL_ENDED, ) if guest_attribute_task.load('status') == 'ENDED_SUCCESS': break failed_tries.append(guest_attribute_task) else: return this.error('did not find hostkey within 60 tries') for failed_try in failed_tries: failed_try.delete() hostkey = guest_attribute_task.get( 'output_value')['result']['queryValue']['items'][0]['value'] system.connector('webpagetestserver-setup').save( connector_type='SSH', value={ 'key': setup_user_priv, 'username': '******', 'hostname': external_ip, 'hostkey': hostkey, }, ) system.connector('webpagetestserver').save( connector_type='SSH', value={ 'key': webpagetest_user_priv, 'username': '******', 'hostname': external_ip, 'hostkey': hostkey, }, ) system.connector('webpagetestserver-copy').save( connector_type='SCP', value={ 'key': webpagetest_user_priv, 'username': '******', 'hostname': external_ip, 'hostkey': hostkey, }, ) webpagetestserver_template = { 'hostname': external_ip, 'hostkey': hostkey, 'setup-user': { 'username': '******', 'ssh-key': setup_user_priv, 'public-key': setup_user_pub, }, 'webpagetest-user': { 'username': '******', 'ssh-key': webpagetest_user_priv, 'public-key': webpagetest_user_pub, }, } for _ in range(10): try: this.connect( 'webpagetestserver-setup', script=""" sudo apt-get update sudo apt-get install -y --no-install-recommends python-pip python3-pip sudo apt-get install -y --no-install-recommends build-essential python3-dev iftop iotop zip unzip # DO NOT remove python3-cryptography! or instance will not be able to setup networking on boot # sudo apt-get remove -y python3-cryptography """, name='install packages', ) except Exception: this.sleep(3) else: break else: return this.error( 'failed to ssh to webpagetestserver-template within 10 tries') this.connect( 'webpagetestserver', script=""" pip install wheel setuptools pip3 install wheel setuptools pip3 install alembic psycopg2-binary """, name='install pip packages', ) this.connect( 'webpagetestserver-setup', script=""" set +e ssh -V ret=$? set -e if [ "$ret" -ne "0" ]; then sudo apt-get install -y --no-install-recommends openssh-client fi """, name='install openssh client', ) this.connect( 'webpagetestserver-setup', script=""" set +e jq -V ret=$? set -e if [ "$ret" -ne "0" ]; then sudo apt-get install -y --no-install-recommends jq fi set +e jq -V ret=$? set -e if [ "$ret" -ne "0" ]; then sudo snap install jq fi """, name='install jq', ) this.connect( 'webpagetestserver-setup', script=""" set +e yq -V ret=$? set -e if [ "$ret" -ne "0" ]; then sudo snap install yq fi """, name='install yq', ) this.connect( 'webpagetestserver-setup', script=""" set +e docker --version ret=$? set -e if [ "$ret" -ne "0" ]; then curl -fsSL https://get.docker.com -o get-docker.sh sh get-docker.sh RELEASE=$(lsb_release -is | tr '[:upper:]' '[:lower:]') sudo apt-get install -y --no-install-recommends \\ apt-transport-https \\ ca-certificates \\ curl \\ gnupg2 \\ software-properties-common curl -fsSL https://download.docker.com/linux/${RELEASE}/gpg | sudo apt-key add - sudo add-apt-repository \\ "deb [arch=amd64] https://download.docker.com/linux/${RELEASE} \\ $(lsb_release -cs) \\ stable" sudo apt-get update sudo apt-get install -y --no-install-recommends docker-ce fi sudo usermod -a -G docker webpagetest-user """, name='install docker', ) this.connect( 'webpagetestserver', name='create webpagetest folders', script=""" mkdir -p webpagetest/server mkdir -p webpagetest/agent """, ) this.connect( 'webpagetestserver-copy', name='copy webpagetest server Dockerfile', src='cloudomation:webpagetest-server-Dockerfile', dst='webpagetest/server/Dockerfile', ) this.connect( 'webpagetestserver-copy', name='copy webpagetest server locations.ini', src='cloudomation:webpagetest-server-locations.ini', dst='webpagetest/server/locations.ini', ) this.connect( 'webpagetestserver-copy', name='copy webpagetest agent Dockerfile', src='cloudomation:webpagetest-agent-Dockerfile', dst='webpagetest/agent/Dockerfile', ) this.connect( 'webpagetestserver-copy', name='copy webpagetest agent script.sh', src='cloudomation:webpagetest-agent-script.sh', dst='webpagetest/agent/script.sh', ) this.connect( 'webpagetestserver', name='create webpagetest server and agent', input_value={'script_timeout': 600}, script=""" pushd webpagetest/server docker build -t wpt-server . docker run -d -p 4000:80 --restart unless-stopped wpt-server popd pushd webpagetest/agent chmod u+x script.sh docker build -t wpt-agent . docker run -d -p 4001:80 \\ --restart unless-stopped \\ --network="host" \\ -e "SERVER_URL=http://localhost:4000/work/" \\ -e "LOCATION=Test" \\ wpt-agent """, ) this.connect( gcloud_connector, name='stop webpagetest-server-template', api_name='compute', api_version='v1', collection='instances', request='stop', params={ 'instance': 'webpagetest-server-template', 'project': project_id, 'zone': 'europe-west1-b', }, ) try: operation = this.connect( gcloud_connector, name='delete old webpagetest-server-template snapshot', api_name='compute', api_version='v1', collection='snapshots', request='delete', params={ 'snapshot': 'webpagetest-server-template', 'project': project_id, }, ).get('output_value')['result'] this.flow( 'google_operation_wait', operation=operation, ) except Exception: pass today = date.today() operation = this.connect( gcloud_connector, name='create webpagetest-server-template snapshot', api_name='compute', api_version='v1', collection='disks', request='createSnapshot', params={ 'disk': 'webpagetest-server-template', 'body': { 'name': 'webpagetest-server-template', 'description': f'auto-generated at {today} by setup webpagetest server', 'storageLocations': ['europe-west1'], }, 'project': project_id, 'zone': 'europe-west1-b', }, ).get('output_value')['result'] this.flow( 'google_operation_wait', operation=operation, ) webpagetestserver_template.pop('hostname') webpagetestserver_template.pop('hostkey') webpagetestserver_template_setting.save( value=webpagetestserver_template) webpagetestserver_template_setting.release() except Exception as ex: this.save(message=repr(ex)) if this.get('input_value').get('interactive'): this.sleep(120) # this.pause() raise finally: instance_list = this.connect( gcloud_connector, name='find webpagetest-server-template', api_name='compute', api_version='v1', collection='instances', request='list', params={ 'filter': f'name=webpagetest-server-template', 'project': project_id, 'zone': 'europe-west1-b' }, ).get('output_value') if len(instance_list['result'].get('items', [])) > 0: this.connect( gcloud_connector, name='delete webpagetest-server-template', api_name='compute', api_version='v1', collection='instances', request='delete', params={ 'instance': 'webpagetest-server-template', 'project': project_id, 'zone': 'europe-west1-b', }, ) while True: instance_list = this.connect( gcloud_connector, name='find webpagetest-server-template', api_name='compute', api_version='v1', collection='instances', request='list', params={ 'filter': 'name=webpagetest-server-template', 'project': project_id, 'zone': 'europe-west1-b', }, ).get('output_value') if len(instance_list['result'].get('items', [])) == 0: break this.sleep(1) return this.success('all done')
def handler(system: flow_api.System, this: flow_api.Execution): """ In this example we setup a Virtual Machine (VM) in the google-cloud. Then, Ubuntu will be installed and a small bash script is executed. Bash scripts can be used to install more software on the VM or change settings, anything you could do on a local machine in a terminal. This flow also requires the two files: * create google cloud vm example.sh * create google cloud vm example.json In the future, you can take these as an example, adopt them and upload them with a new name in Cloudomation > Files. """ # This is a pop-up message form, it will ask for some information # The last element is an OK-button, after clicking it the # values are passed further and the pop-up message closes. form_response = system.message( subject='Input information to setup a VM in google-cloud', message_type='POPUP', body={ 'type': 'object', 'properties': { 'connector_name': { 'element': 'string', 'type': 'string', 'label': 'Enter name for the connector: \n\ (This name can be searched for and is used to connect with your VM)', 'example': 'my-vm-on-gcloud', 'order': 1, }, 'project_id': { 'element': 'string', 'type': 'string', 'label': 'Enter your google cloud project-ID:', 'default': 'my-first-project-id', 'order': 2, }, 'machine_type': { 'element': 'string', 'type': 'string', 'label': 'Enter the machine type you want to use on google-cloud:', 'default': 'n1-standard-1', 'order': 3, }, 'server_location': { 'element': 'string', 'type': 'string', 'label': 'Enter the desired location of the hosting server from google-cloud:', 'default': 'europe-west1-b', 'order': 4, }, 'Ok': { 'element': 'submit', 'label': 'OK', 'type': 'boolean', 'order': 5, }, }, 'required': [ 'connector_name', 'project_id', 'machine_type', 'server_location', ], }, ).wait().get('response') connector_name = form_response['connector_name'] project_id = form_response['project_id'] machine_type = form_response['machine_type'] server_location = form_response['server_location'] # generate a ssh key-pair for the setup-user: setup_user_key = RSA.generate(2048) setup_user_priv = setup_user_key.export_key().decode() setup_user_pub = setup_user_key.publickey().export_key( format='OpenSSH').decode() setup_user_key = None # load the startup script and pass the public ssh key to the script: vm_startup_script = system.file( 'create google cloud vm example.sh').get_text_content() vm_startup_script = vm_startup_script.format( setup_user_pub=setup_user_pub, ) # load the configuration for the VM and setup last parameters: vm_config_str = system.file( 'create google cloud vm example.json').get_text_content() vm_config = json.loads(vm_config_str) vm_config['name'] = connector_name vm_config['machineType'] = \ f'projects/{project_id}/zones/{server_location}/machineTypes/{machine_type}' vm_config['zone'] = f'projects/{project_id}/zones/{server_location}' vm_config['metadata']['items'][0]['value'] = vm_startup_script vm_config['disks'][0]['deviceName'] = connector_name vm_config['disks'][0]['initializeParams']['diskType'] = \ f'projects/{project_id}/zones/{server_location}/diskTypes/pd-ssd' vm_config['networkInterfaces'][0]['subnetwork'] = \ f'projects/{project_id}/regions/{server_location[:-2]}/subnetworks/default' # NOTE: two sources can be used for initial system installation: # a) from 'sourceSnapshot' # b) from 'sourceImage' # This flow script will install Ubuntu, using the parameter 'sourceImage'. # For installing a system from snapshot use the respective parameter. # This paramter is set under: disks / initializeParams # Connect to the VM operation = this.connect( 'gcloud', name=f'launch {connector_name}', api_name='compute', api_version='v1', collection='instances', request='insert', params={ 'project': project_id, 'zone': server_location, 'body': vm_config, }, ).get('output_value')['result'] this.flow( 'google_operation_wait', operation=operation, ) # retrieve external IP and hostkey (required for creating a SSH connector): for _ in range(60): try: instance_list = this.connect( 'gcloud', name=f'find {connector_name}', api_name='compute', api_version='v1', collection='instances', request='list', params={ 'project': project_id, 'zone': server_location, 'filter': f'name={connector_name}', }, ).get('output_value') except flow_api.DependencyFailedError: this.sleep(1) if len(instance_list['result'].get('items', [])) != 0: break this.sleep(1) else: return this.error('did not find VM within 60 tries') external_ip = instance_list['result']['items'][0]['networkInterfaces'][0][ 'accessConfigs'][0]['natIP'] last_try_guest_attribute = None for _ in range(60): if last_try_guest_attribute is not None: last_try_guest_attribute.archive() try: last_try_guest_attribute = this.connect( 'gcloud', name=f'get {connector_name} hostkey', api_name='compute', api_version='v1', collection='instances', request='getGuestAttributes', params={ 'project': project_id, 'zone': server_location, 'instance': connector_name, 'queryPath': 'hostkeys/ssh-rsa', }, wait=system.return_when.ALL_ENDED, ) except Exception: this.sleep(1) else: if last_try_guest_attribute.load('status') == 'ENDED_SUCCESS': break else: return this.error('did not find hostkey within 60 tries') hostkey = last_try_guest_attribute.get( 'output_value')['result']['queryValue']['items'][0]['value'] # setup the SSH connector for the VM system.connector(f'{connector_name}-ssh').save( connector_type='SSH', value={ 'hostname': external_ip, 'hostkey': hostkey, 'username': '******', 'key': setup_user_priv, 'script_timeout': 600, 'interpreter': '/usr/bin/env bash -ex', }, ) # Run a little bash script on the VM: last_try_script_echo = None for _ in range(20): if last_try_script_echo is not None: last_try_script_echo.archive() try: last_try_script_echo = this.connect( f'{connector_name}-ssh', script=""" echo "litte test content" >> ~/test_file.txt cat ~/test_file.txt rm ~/test_file.txt """, name='run test script on new vm', ) except: this.sleep(1) else: if last_try_script_echo.load('status') == 'ENDED_SUCCESS': break else: return this.error('Failed to run the little test script.') script_echo = last_try_script_echo.get('output_value')['report'] this.set_output(f'Echo from test-script:\n{script_echo}') # Save flow state and pause, manually resume this script's execution to remove the VM this.save( message= 'The VM was successfully deployed. Resume this execution to remove the VM' ) this.pause() # Remove the VM operation = this.connect( 'gcloud', name=f'remove {connector_name}', api_name='compute', api_version='v1', collection='instances', request='delete', params={ 'project': project_id, 'zone': server_location, 'instance': connector_name, }, ).get('output_value')['result'] this.flow( 'google_operation_wait', operation=operation, ) return this.success('all done')