def cloud_test(cloud=None, datastore=None, destroy_vm="True"):
"""
Run the cloud tests. This defaults to running on local vagrant
machines, but can also be pointed to an arbitrary host (e.g. amazon) by
passing "address:port" as arguments
cloud: The ssh address string of the machine to run the cloud
on. Formatted as "host:port". If not specified, defaults to
the `cloud` vagrant box.
datastore: The ssh address string of the machine to run the datastore on
on. Formatted as "host:port". If not specified, defaults to the
`datastore` vagrant box.
"""
destroy_vm = bool(strtobool(destroy_vm))
# Setup the datastore: use the provided test machine if given, else default
# to the vagrant machine
if not datastore:
datastore = vagrant_setup("datastore", destroy_vm)
else:
ansible_setup(datastore, "datastore", "datastore.dev.yml")
# Setup the cloud: use the provided address if given, else default to the
# vagrant machine
if not cloud:
cloud = vagrant_setup("cloud", destroy_vm)
else:
ansible_setup(cloud, "magma-cloud-dev", "cloud.dev.yml")
env.host_string = cloud
(env.user, _, _) = split_hoststring(cloud)
with cd('~/magma/orc8r/cloud'):
run('make cover')
def package(feg_host=None, vcs="hg", force=False):
"""
Create deploy package and push to S3. This defaults to running on local
vagrant feg VM machines, but can also be pointed to an arbitrary host
(e.g. amazon) by specifying a VM.
feg_host: The ssh address string of the machine to run the package
command. Formatted as "<user>@<host>:<port>". If not specified,
defaults to the `feg` vagrant VM.
vcs: version control system used, "hg" or "git".
force: Bypass local commits or changes check if set to True.
"""
if not force and pkg.check_commit_changes():
abort("Local changes or commits not allowed")
if feg_host:
env.host_string = feg_host
(env.user, _, _) = split_hoststring(feg_host)
else:
setup_env_vagrant("feg")
target = "/tmp/magmadeploy_feg"
run("rm -rf %s" % target)
run("mkdir -p %s" % target)
_package_python(target)
_package_go(target)
_package_scripts(target)
return _push_archive_to_s3(vcs, target)
def _package_vagrant_zip(service, folder, cloud_host, commit_hash):
if cloud_host != "":
env.host_string = cloud_host
(env.user, _, _) = split_hoststring(cloud_host)
else:
_vagrant()
run("rm -rf %s" % folder)
run("mkdir -p %s" % folder)
with cd('magma/orc8r/cloud/deploy'):
run('cp -pr aws/%s_appspec.yml %s/appspec.yml' % (service, folder))
run('cp -pr aws/scripts %s/.' % folder)
run("mkdir -p %s/ansible/roles" % folder)
run('cp -pr %s.yml %s/ansible/main.yml' % (service, folder))
run('cp -pr roles/aws_setup %s/ansible/roles/.' % folder)
run('cp -pr roles/osquery %s/ansible/roles/.' % folder)
run('cp -pr roles/service_registry %s/ansible/roles/.' % folder)
if service == "metrics":
run('cp -pr roles/prometheus %s/ansible/roles/.' % folder)
run('cp -pr roles/third_party/graphite %s/ansible/roles.' % folder)
else:
run('cp -pr roles/%s %s/ansible/roles/.' % (service, folder))
if service == "controller":
run('cp -pr /etc/magma %s/configs' % folder)
run('cp -pr files/scripts/setup_swagger_ui %s/scripts/.' % folder)
run('cp -pr files/static/apidocs %s/.' % folder)
if service == "proxy":
run('cp -pr /etc/magma %s/configs' % folder)
run('cp -pr roles/disk_metrics %s/ansible/roles/.' % folder)
run('cp -pr ../../../orc8r/tools/ansible/roles/pkgrepo '
'%s/ansible/roles/.' % folder)
# Build Go binaries and plugins
build()
if service == "metrics":
run("make -C magma/orc8r/cloud/go/services/metricsd/prometheus/prometheus-cache build")
run('cp -pr go/plugins %s' % folder)
_copy_go_binaries(service, folder)
pkg_name = "magma_%s_%s" % (service, commit_hash)
with cd(folder):
run('zip -r %s *' % (pkg_name))
get('%s/%s.zip' % (folder, pkg_name), '%s/%s.zip' % (folder, pkg_name))
run('rm -rf %s' % folder)
return "%s.zip" % pkg_name
def package(service, cloud_host=None, vcs="hg", force=False):
"""
Create deploy package and push to S3. This defaults to running on local
vagrant cloud VM machines, but can also be pointed to an arbitrary host
(e.g. amazon) by specifying a VM.
cloud_host: The ssh address string of the machine to run the package
command. Formatted as "<user>@<host>:<port>". If not specified,
defaults to the `cloud` vagrant VM.
vcs: version control system used, "hg" or "git".
force: Bypass local commits or changes check if set to True.
"""
# Check that we have no local changes or commits at this point
if not force and pkg.check_commit_changes():
abort("Local changes or commits not allowed")
_validate_service(service)
if cloud_host:
env.host_string = cloud_host
(env.user, _, _) = split_hoststring(cloud_host)
else:
_vagrant()
# Use same temp folder name for local and VM operations
folder = "/tmp/magmadeploy_%s" % service
local("rm -rf %s" % folder)
local("mkdir -p %s" % folder)
run("rm -rf %s" % folder)
run("mkdir -p %s" % folder)
with cd('magma/orc8r/cloud/deploy'):
run('cp -pr aws/%s_appspec.yml %s/appspec.yml' % (service, folder))
run('cp -pr aws/scripts %s/.' % folder)
run("mkdir -p %s/ansible/roles" % folder)
run('cp -pr %s.yml %s/ansible/main.yml' % (service, folder))
run('cp -pr roles/%s %s/ansible/roles/.' % (service, folder))
run('cp -pr roles/aws_setup %s/ansible/roles/.' % folder)
run('cp -pr roles/osquery %s/ansible/roles/.' % folder)
run('cp -pr roles/service_registry %s/ansible/roles/.' % folder)
if service == "controller":
run('cp -pr /etc/magma %s/configs' % folder)
run('cp -pr files/scripts/setup_swagger_ui %s/scripts/.' % folder)
run('cp -pr files/static/apidocs %s/.' % folder)
if service == "proxy":
run('cp -pr /etc/magma %s/configs' % folder)
run('cp -pr roles/disk_metrics %s/ansible/roles/.' % folder)
run('cp -pr ../../../orc8r/tools/ansible/roles/pkgrepo '
'%s/ansible/roles/.' % folder)
if service == 'prometheus':
run('cp -pr roles/prometheus %s/ansible/roles/.' % folder)
run('mkdir -p %s/bin' % folder) # To make CodeDeploy happy
# Build Go binaries and plugins
build()
run('cp -pr go/plugins %s' % folder)
_copy_go_binaries(service, folder)
# Zip and push to s3
pkg_name = "magma_%s_%s" % (service, pkg.get_commit_hash(vcs))
_push_archive_to_s3(service, folder, pkg_name)
run('rm -rf %s' % folder)
local('rm -rf %s' % folder)
return "%s.zip" % pkg_name
def get_test_logs(gateway_host=None, test_host=None, trf_host=None):
'''
Downloads the relevant magma logs from the given gateway and test machines.
Places the logs in /tmp/build_logs.zip
gateway_host: The ssh address string of the gateway machine formatted as
"host:port". If not specified, defaults to the `magma` vagrant box.
test_host: The ssh address string of the test machine formatted as
"host:port". If not specified, defaults to the `magma_test` vagrant
box.
trf_host: The ssh address string of the machine to run the TrafficServer
on. Formatted as "host:port". If not specified, defaults to the
`magma_trfserver` vagrant box.
'''
# Grab the build logs from the machines and bring them to the host
local('rm -rf /tmp/build_logs')
local('mkdir /tmp/build_logs')
local('mkdir /tmp/build_logs/dev')
local('mkdir /tmp/build_logs/test')
local('mkdir /tmp/build_logs/trfserver')
dev_files = ['/var/log/mme.log', '/var/log/syslog']
test_files = ['/var/log/syslog', '/tmp/fw/']
trf_files = ['/home/admin/nohup.out']
# Set up to enter the gateway host
env.host_string = gateway_host
if not gateway_host:
setup_env_vagrant("magma")
gateway_host = env.hosts[0]
(env.user, _, _) = split_hoststring(gateway_host)
# Don't fail if the logs don't exists
for p in dev_files:
with settings(warn_only=True):
get(remote_path=p,
local_path='/tmp/build_logs/dev/',
use_sudo=True)
# Set up to enter the trfserver host
env.host_string = trf_host
if not trf_host:
setup_env_vagrant("magma_trfserver")
trf_host = env.hosts[0]
(env.user, _, _) = split_hoststring(trf_host)
# Don't fail if the logs don't exists
for p in trf_files:
with settings(warn_only=True):
get(remote_path=p,
local_path='/tmp/build_logs/trfserver/',
use_sudo=True)
# Set up to enter the test host
env.host_string = test_host
if not test_host:
setup_env_vagrant("magma_test")
test_host = env.hosts[0]
(env.user, _, _) = split_hoststring(test_host)
# Fix the permissions on the fw directory -- it has permissions 000
# otherwise
with settings(warn_only=True):
run('sudo chmod 755 /tmp/fw')
# Don't fail if the logs don't exists
for p in test_files:
with settings(warn_only=True):
get(remote_path=p,
local_path='/tmp/build_logs/test/',
use_sudo=True)
local('rm -rf /tmp/build_logs.zip')
local('zip -r /tmp/build_logs.zip /tmp/build_logs')
local('rm -rf /tmp/build_logs')