def package(feg_host=None, vcs="hg", force=False): """ Create deploy package and push to S3. This defaults to running on local vagrant feg VM machines, but can also be pointed to an arbitrary host (e.g. amazon) by specifying a VM. feg_host: The ssh address string of the machine to run the package command. Formatted as "<user>@<host>:<port>". If not specified, defaults to the `feg` vagrant VM. vcs: version control system used, "hg" or "git". force: Bypass local commits or changes check if set to True. """ if not force and pkg.check_commit_changes(): abort("Local changes or commits not allowed") if feg_host: env.host_string = feg_host (env.user, _, _) = split_hoststring(feg_host) else: setup_env_vagrant("feg") target = "/tmp/magmadeploy_feg" run("rm -rf %s" % target) run("mkdir -p %s" % target) _package_python(target) _package_go(target) _package_scripts(target) return _push_archive_to_s3(vcs, target)
def _package_vagrant_zip(service, folder, cloud_host, commit_hash): if cloud_host != "": env.host_string = cloud_host (env.user, _, _) = split_hoststring(cloud_host) else: _vagrant() run("rm -rf %s" % folder) run("mkdir -p %s" % folder) with cd('magma/orc8r/cloud/deploy'): run('cp -pr aws/%s_appspec.yml %s/appspec.yml' % (service, folder)) run('cp -pr aws/scripts %s/.' % folder) run("mkdir -p %s/ansible/roles" % folder) run('cp -pr %s.yml %s/ansible/main.yml' % (service, folder)) run('cp -pr roles/aws_setup %s/ansible/roles/.' % folder) run('cp -pr roles/osquery %s/ansible/roles/.' % folder) run('cp -pr roles/service_registry %s/ansible/roles/.' % folder) if service == "metrics": run('cp -pr roles/prometheus %s/ansible/roles/.' % folder) else: run('cp -pr roles/%s %s/ansible/roles/.' % (service, folder)) if service == "controller": run('cp -pr /etc/magma %s/configs' % folder) run('cp -pr files/scripts/setup_swagger_ui %s/scripts/.' % folder) run('cp -pr files/static/apidocs %s/.' % folder) if service == "proxy": run('cp -pr /etc/magma %s/configs' % folder) run('cp -pr roles/disk_metrics %s/ansible/roles/.' % folder) run('cp -pr ../../../orc8r/tools/ansible/roles/pkgrepo ' '%s/ansible/roles/.' % folder) # Build Go binaries and plugins build() if service == "metrics": run("make -C magma/orc8r/cloud/go/services/metricsd/prometheus/prometheus-cache build" ) run('cp -pr go/plugins %s' % folder) _copy_go_binaries(service, folder) pkg_name = "magma_%s_%s" % (service, commit_hash) with cd(folder): run('zip -r %s *' % (pkg_name)) get('%s/%s.zip' % (folder, pkg_name), '%s/%s.zip' % (folder, pkg_name)) run('rm -rf %s' % folder) return "%s.zip" % pkg_name
def cloud_test(cloud=None, datastore=None, destroy_vm="True"): """ Run the cloud tests. This defaults to running on local vagrant machines, but can also be pointed to an arbitrary host (e.g. amazon) by passing "address:port" as arguments cloud: The ssh address string of the machine to run the cloud on. Formatted as "host:port". If not specified, defaults to the `cloud` vagrant box. datastore: The ssh address string of the machine to run the datastore on on. Formatted as "host:port". If not specified, defaults to the `datastore` vagrant box. """ destroy_vm = bool(strtobool(destroy_vm)) # Setup the datastore: use the provided test machine if given, else default # to the vagrant machine if not datastore: datastore = vagrant_setup("datastore", destroy_vm) else: ansible_setup(datastore, "datastore", "datastore.dev.yml") # Setup the cloud: use the provided address if given, else default to the # vagrant machine if not cloud: cloud = vagrant_setup("cloud", destroy_vm) else: ansible_setup(cloud, "magma-cloud-dev", "cloud.dev.yml") env.host_string = cloud (env.user, _, _) = split_hoststring(cloud) with cd('~/magma/orc8r/cloud'): # Retry golang dependency fetching 3 times to be resilient to network # flapping run('make download || make download || make download') run('make cover')
def get_test_logs(gateway_host=None, test_host=None, trf_host=None, dst_path="/tmp/build_logs.tar.gz"): ''' Downloads the relevant magma logs from the given gateway and test machines. Places the logs in a path specified in 'dst_path' or "/tmp/build_logs.tar.gz" by default gateway_host: The ssh address string of the gateway machine formatted as "host:port". If not specified, defaults to the `magma` vagrant box. test_host: The ssh address string of the test machine formatted as "host:port". If not specified, defaults to the `magma_test` vagrant box. trf_host: The ssh address string of the machine to run the TrafficServer on. Formatted as "host:port". If not specified, defaults to the `magma_trfserver` vagrant box. dst_path: The path where the tarred logs will be placed on the host ''' # Grab the build logs from the machines and bring them to the host local('rm -rf /tmp/build_logs') local('mkdir /tmp/build_logs') local('mkdir /tmp/build_logs/dev') local('mkdir /tmp/build_logs/test') local('mkdir /tmp/build_logs/trfserver') dev_files = [ '/var/log/mme.log', '/var/log/syslog', '/var/log/openvswitch/ovs*.log' ] test_files = ['/var/log/syslog', '/tmp/fw/'] trf_files = ['/home/admin/nohup.out'] # Set up to enter the gateway host env.host_string = gateway_host if not gateway_host: setup_env_vagrant("magma") gateway_host = env.hosts[0] (env.user, _, _) = split_hoststring(gateway_host) # Don't fail if the logs don't exists for p in dev_files: with settings(warn_only=True): get(remote_path=p, local_path='/tmp/build_logs/dev/', use_sudo=True) # Set up to enter the trfserver host env.host_string = trf_host if not trf_host: setup_env_vagrant("magma_trfserver") trf_host = env.hosts[0] (env.user, _, _) = split_hoststring(trf_host) # Don't fail if the logs don't exists for p in trf_files: with settings(warn_only=True): get(remote_path=p, local_path='/tmp/build_logs/trfserver/', use_sudo=True) # Set up to enter the test host env.host_string = test_host if not test_host: setup_env_vagrant("magma_test") test_host = env.hosts[0] (env.user, _, _) = split_hoststring(test_host) # Fix the permissions on the fw directory -- it has permissions 000 # otherwise with settings(warn_only=True): run('sudo chmod 755 /tmp/fw') # Don't fail if the logs don't exists for p in test_files: with settings(warn_only=True): get(remote_path=p, local_path='/tmp/build_logs/test/', use_sudo=True) local("tar -czvf /tmp/build_logs.tar.gz /tmp/build_logs/*") local(f'mv /tmp/build_logs.tar.gz {dst_path}') local('rm -rf /tmp/build_logs')