def package(feg_host=None, vcs="hg", force=False):
"""
Create deploy package and push to S3. This defaults to running on local
vagrant feg VM machines, but can also be pointed to an arbitrary host
(e.g. amazon) by specifying a VM.
feg_host: The ssh address string of the machine to run the package
command. Formatted as "<user>@<host>:<port>". If not specified,
defaults to the `feg` vagrant VM.
vcs: version control system used, "hg" or "git".
force: Bypass local commits or changes check if set to True.
"""
if not force and pkg.check_commit_changes():
abort("Local changes or commits not allowed")
if feg_host:
env.host_string = feg_host
(env.user, _, _) = split_hoststring(feg_host)
else:
setup_env_vagrant("feg")
target = "/tmp/magmadeploy_feg"
run("rm -rf %s" % target)
run("mkdir -p %s" % target)
_package_python(target)
_package_go(target)
_package_scripts(target)
return _push_archive_to_s3(vcs, target)
def connect_gateway_to_cloud(control_proxy_setting_path=None, cert_path=DEFAULT_CERT):
"""
Setup the feg gateway VM to connect to the cloud
Path to control_proxy.yml and rootCA.pem could be specified to use
non-default control proxy setting and certificates
"""
setup_env_vagrant("feg")
dev_utils.connect_gateway_to_cloud(control_proxy_setting_path, cert_path)
def vagrant_setup(host, destroy_vm, force_provision='false'):
"""
Setup the specified vagrant box
host: the Vagrant box to setup, e.g. "magma"
"""
if destroy_vm:
vagrant.teardown_vagrant(host)
vagrant.setup_env_vagrant(host, force_provision=force_provision)
return env.hosts[0]
def vagrant_setup(host, destroy_vm):
"""
Setup the specified vagrant box
host: the Vagrant box to setup, e.g. "magma"
"""
if destroy_vm:
vagrant.teardown_vagrant(host)
vagrant.setup_env_vagrant(host)
return env.hosts[0]
def delete_gateway_certs_from_vagrant(vm_name: str):
"""
Delete certificates and gw_challenge of a gateway running on Vagrant VM
Args:
vm_name: Name of the vagrant machine to use
"""
with settings(warn_only=True), hide('output', 'running', 'warnings'), \
cd('/var/opt/magma/certs'):
vagrant.setup_env_vagrant(vm_name)
sudo('rm gateway.*')
sudo('rm gw_challenge.key')
def package(vcs='hg',
all_deps="False",
cert_file=DEFAULT_CERT,
proxy_config=DEFAULT_PROXY):
""" Builds the magma package """
all_deps = False if all_deps == "False" else True
# If a host list isn't specified, default to the magma vagrant vm
if not env.hosts:
setup_env_vagrant()
if not hasattr(env, 'debug_mode'):
print("Error: The Deploy target isn't specified. Specify one with\n\n"
"\tfab [dev|test] package")
exit(1)
hash = pkg.get_commit_hash(vcs)
with cd('~/magma/lte/gateway'):
print("Building magma package, picking up commit %s..." % hash)
run('make clean')
build_type = "Debug" if env.debug_mode else "RelWithDebInfo"
run('./release/build-magma.sh -h "%s" -t %s --cert %s --proxy %s' %
(hash, build_type, cert_file, proxy_config))
# Generate magma dependency packages
run('mkdir -p ~/magma-deps')
print("Generating lte/setup.py magma dependency packages")
run('./release/pydep finddep -b --build-output ~/magma-deps python/setup.py'
)
print("Generating orc8r/setup.py magma dependency packages")
run('./release/pydep finddep -b --build-output ~/magma-deps %s/setup.py'
% ORC8R_AGW_PYTHON_ROOT)
run('rm -rf ~/magma-packages')
run('mkdir -p ~/magma-packages')
try:
run('cp -f ~/magma-deps/*.deb ~/magma-packages')
except Exception:
# might be a problem if no deps found, but don't handle here
pass
run('mv *.deb ~/magma-packages')
with cd('release'):
run('cat mirrored_packages | '
'xargs -I% sudo aptitude download -q2 %')
run('cp *.deb ~/magma-packages')
run('sudo rm -f *.deb')
if all_deps:
pkg.download_all_pkgs()
run('cp /var/cache/apt/archives/*.deb ~/magma-packages')
def get_hardware_id_from_vagrant(vm_name: str) -> str:
"""
Get the magmad hardware ID from vagrant
Args:
vm_name: Name of the vagrant machine to use
Returns:
Hardware snowflake from the VM
"""
with hide('output', 'running', 'warnings'):
vagrant.setup_env_vagrant(vm_name)
hardware_id = run('cat /etc/snowflake')
return str(hardware_id)
def package(vcs='hg',
all_deps="False",
cert_file=DEFAULT_CERT,
proxy_config=DEFAULT_PROXY):
""" Builds the magma package """
all_deps = False if all_deps == "False" else True
# If a host list isn't specified, default to the magma vagrant vm
if not env.hosts:
setup_env_vagrant()
if not hasattr(env, 'debug_mode'):
print("Error: The Deploy target isn't specified. Specify one with\n\n"
"\tfab [dev|test] package")
exit(1)
hash = pkg.get_commit_hash(vcs)
with cd('~/magma/lte/gateway'):
print("Building magma package, picking up commit %s..." % hash)
run('make clean')
build_type = "Debug" if env.debug_mode else "RelWithDebInfo"
run('./release/build-magma.sh -h "%s" -t %s --cert %s --proxy %s' %
(hash, build_type, cert_file, proxy_config))
# Generate magma dependency packages
print("Generating magma dependency packages")
with cd('python'):
run('../release/pydep finddep -b ../python/setup.py')
run('mv *.deb ../')
run('rm -rf ~/magma-packages')
run('mkdir -p ~/magma-packages')
run('mv *.deb ~/magma-packages')
if all_deps:
pkg.download_all_pkgs()
run('cp /var/cache/apt/archives/*.deb ~/magma-packages')
def get_test_logs(gateway_host=None,
test_host=None,
trf_host=None,
dst_path="/tmp/build_logs.tar.gz"):
'''
Downloads the relevant magma logs from the given gateway and test machines.
Places the logs in a path specified in 'dst_path' or
"/tmp/build_logs.tar.gz" by default
gateway_host: The ssh address string of the gateway machine formatted as
"host:port". If not specified, defaults to the `magma` vagrant box.
test_host: The ssh address string of the test machine formatted as
"host:port". If not specified, defaults to the `magma_test` vagrant
box.
trf_host: The ssh address string of the machine to run the TrafficServer
on. Formatted as "host:port". If not specified, defaults to the
`magma_trfserver` vagrant box.
dst_path: The path where the tarred logs will be placed on the host
'''
# Grab the build logs from the machines and bring them to the host
local('rm -rf /tmp/build_logs')
local('mkdir /tmp/build_logs')
local('mkdir /tmp/build_logs/dev')
local('mkdir /tmp/build_logs/test')
local('mkdir /tmp/build_logs/trfserver')
dev_files = [
'/var/log/mme.log', '/var/log/syslog', '/var/log/openvswitch/ovs*.log'
]
test_files = ['/var/log/syslog', '/tmp/fw/']
trf_files = ['/home/admin/nohup.out']
# Set up to enter the gateway host
env.host_string = gateway_host
if not gateway_host:
setup_env_vagrant("magma")
gateway_host = env.hosts[0]
(env.user, _, _) = split_hoststring(gateway_host)
# Don't fail if the logs don't exists
for p in dev_files:
with settings(warn_only=True):
get(remote_path=p,
local_path='/tmp/build_logs/dev/',
use_sudo=True)
# Set up to enter the trfserver host
env.host_string = trf_host
if not trf_host:
setup_env_vagrant("magma_trfserver")
trf_host = env.hosts[0]
(env.user, _, _) = split_hoststring(trf_host)
# Don't fail if the logs don't exists
for p in trf_files:
with settings(warn_only=True):
get(remote_path=p,
local_path='/tmp/build_logs/trfserver/',
use_sudo=True)
# Set up to enter the test host
env.host_string = test_host
if not test_host:
setup_env_vagrant("magma_test")
test_host = env.hosts[0]
(env.user, _, _) = split_hoststring(test_host)
# Fix the permissions on the fw directory -- it has permissions 000
# otherwise
with settings(warn_only=True):
run('sudo chmod 755 /tmp/fw')
# Don't fail if the logs don't exists
for p in test_files:
with settings(warn_only=True):
get(remote_path=p,
local_path='/tmp/build_logs/test/',
use_sudo=True)
local("tar -czvf /tmp/build_logs.tar.gz /tmp/build_logs/*")
local(f'mv /tmp/build_logs.tar.gz {dst_path}')
local('rm -rf /tmp/build_logs')
def integ_test(gateway_host=None,
test_host=None,
trf_host=None,
destroy_vm="True"):
"""
Run the integration tests. This defaults to running on local vagrant
machines, but can also be pointed to an arbitrary host (e.g. amazon) by
passing "address:port" as arguments
gateway_host: The ssh address string of the machine to run the gateway
services on. Formatted as "host:port". If not specified, defaults to
the `magma` vagrant box.
test_host: The ssh address string of the machine to run the tests on
on. Formatted as "host:port". If not specified, defaults to the
`magma_test` vagrant box.
trf_host: The ssh address string of the machine to run the TrafficServer
on. Formatted as "host:port". If not specified, defaults to the
`magma_trfserver` vagrant box.
"""
destroy_vm = bool(strtobool(destroy_vm))
# Setup the gateway: use the provided gateway if given, else default to the
# vagrant machine
gateway_ip = '192.168.60.142'
if not gateway_host:
gateway_host = vagrant_setup("magma", destroy_vm)
else:
ansible_setup(gateway_host, "dev", "magma_dev.yml")
gateway_ip = gateway_host.split('@')[1].split(':')[0]
execute(_dist_upgrade)
execute(_build_magma)
execute(_run_unit_tests)
execute(_python_coverage)
execute(_start_gateway)
# Run suite of integ tests that are required to be run on the access gateway
# instead of the test VM
# TODO: fix the integration test T38069907
# execute(_run_local_integ_tests)
# Setup the trfserver: use the provided trfserver if given, else default to the
# vagrant machine
if not trf_host:
trf_host = vagrant_setup("magma_trfserver", destroy_vm)
else:
ansible_setup(trf_host, "trfserver", "magma_trfserver.yml")
execute(_start_trfserver)
# Run the tests: use the provided test machine if given, else default to
# the vagrant machine
if not test_host:
test_host = vagrant_setup("magma_test", destroy_vm)
else:
ansible_setup(test_host, "test", "magma_test.yml")
execute(_make_integ_tests)
execute(_run_integ_tests, gateway_ip)
if not gateway_host:
setup_env_vagrant()
else:
env.hosts = [gateway_host]
execute(_oai_coverage)
def copy_packages():
if not env.hosts:
setup_env_vagrant()
pkg.copy_packages()
def upload_to_aws():
# If a host list isn't specified, default to the magma vagrant vm
if not env.hosts:
setup_env_vagrant()
pkg.upload_pkgs_to_aws()
def depclean():
'''Remove all generated packaged for dependencies'''
# If a host list isn't specified, default to the magma vagrant vm
if not env.hosts:
setup_env_vagrant()
run('rm -rf ~/magma-deps')
def bazel_integ_test_post_build(
gateway_host=None,
test_host=None,
trf_host=None,
destroy_vm='True',
provision_vm='True',
):
"""
Run the integration tests on the bazel build services. This defaults to
running on local vagrant machines, but can also be pointed to an
arbitrary host (e.g. amazon) by passing "address:port" as arguments
gateway_host: The ssh address string of the machine to run the gateway
services on. Formatted as "host:port". If not specified, defaults to
the `magma` vagrant box.
test_host: The ssh address string of the machine to run the tests on
on. Formatted as "host:port". If not specified, defaults to the
`magma_test` vagrant box.
trf_host: The ssh address string of the machine to run the TrafficServer
on. Formatted as "host:port". If not specified, defaults to the
`magma_trfserver` vagrant box.
"""
destroy_vm = bool(strtobool(destroy_vm))
provision_vm = bool(strtobool(provision_vm))
gateway_ip = '192.168.60.142'
if not gateway_host:
gateway_host = vagrant_setup(
'magma',
False,
force_provision=False,
)
else:
ansible_setup(gateway_host, "dev", "magma_dev.yml")
gateway_ip = gateway_host.split('@')[1].split(':')[0]
execute(_run_sudo_python_unit_tests)
execute(_restart_gateway)
# Setup the trfserver: use the provided trfserver if given, else default to the
# vagrant machine
if not trf_host:
trf_host = vagrant_setup(
'magma_trfserver',
destroy_vm,
force_provision=provision_vm,
)
else:
ansible_setup(trf_host, "trfserver", "magma_trfserver.yml")
execute(_start_trfserver)
# Run the tests: use the provided test machine if given, else default to
# the vagrant machine
if not test_host:
test_host = vagrant_setup(
'magma_test',
destroy_vm,
force_provision=provision_vm,
)
else:
ansible_setup(test_host, "test", "magma_test.yml")
execute(_make_integ_tests)
execute(_run_integ_tests, gateway_ip)
if not gateway_host:
setup_env_vagrant()
else:
env.hosts = [gateway_host]
