def series_upgrade_non_leaders_first(application, from_series="trusty",
to_series="xenial",
completed_machines=[],
post_upgrade_functions=None):
"""Series upgrade non leaders first.
Wrap all the functionality to handle series upgrade for charms
which must have non leaders upgraded first.
:param application: Name of application to upgrade series
:type application: str
:param from_series: The series from which to upgrade
:type from_series: str
:param to_series: The series to which to upgrade
:type to_series: str
:param completed_machines: List of completed machines which do no longer
require series upgrade.
:type completed_machines: list
:returns: None
:rtype: None
"""
status = model.get_status().applications[application]
leader = None
non_leaders = []
for unit in status["units"]:
if status["units"][unit].get("leader"):
leader = unit
else:
non_leaders.append(unit)
# Series upgrade the non-leaders first
for unit in non_leaders:
machine = status["units"][unit]["machine"]
if machine not in completed_machines:
logging.info("Series upgrade non-leader unit: {}"
.format(unit))
series_upgrade(unit, machine,
from_series=from_series, to_series=to_series,
origin=None,
post_upgrade_functions=post_upgrade_functions)
run_post_upgrade_functions(post_upgrade_functions)
completed_machines.append(machine)
else:
logging.info("Skipping unit: {}. Machine: {} already upgraded. "
.format(unit, machine, application))
model.block_until_all_units_idle()
# Series upgrade the leader
machine = status["units"][leader]["machine"]
logging.info("Series upgrade leader: {}".format(leader))
if machine not in completed_machines:
series_upgrade(leader, machine,
from_series=from_series, to_series=to_series,
origin=None,
post_upgrade_functions=post_upgrade_functions)
completed_machines.append(machine)
else:
logging.info("Skipping unit: {}. Machine: {} already upgraded."
.format(unit, machine, application))
model.block_until_all_units_idle()
def get_full_juju_status():
"""Return the full juju status output.
:returns: Full juju status output
:rtype: dict
"""
status = model.get_status()
return status
def get_full_juju_status():
"""Return the full juju status output
:returns: Full juju status output
:rtype: dict
"""
status = model.get_status(lifecycle_utils.get_juju_model())
return status
def test_200_run_series_upgrade(self):
"""Run series upgrade."""
# Set Feature Flag
os.environ["JUJU_DEV_FEATURE_FLAGS"] = "upgrade-series"
applications = model.get_status().applications
completed_machines = []
for application in applications:
# Defaults
origin = "openstack-origin"
pause_non_leader_subordinate = True
pause_non_leader_primary = True
# Skip subordinates
if applications[application]["subordinate-to"]:
continue
if "percona-cluster" in applications[application]["charm"]:
origin = "source"
pause_non_leader_primary = True
pause_non_leader_subordinate = True
if "rabbitmq-server" in applications[application]["charm"]:
origin = "source"
pause_non_leader_primary = True
pause_non_leader_subordinate = False
if "nova-compute" in applications[application]["charm"]:
pause_non_leader_primary = False
pause_non_leader_subordinate = False
if "ceph" in applications[application]["charm"]:
origin = "source"
pause_non_leader_primary = False
pause_non_leader_subordinate = False
if "memcached" in applications[application]["charm"]:
origin = None
pause_non_leader_primary = False
pause_non_leader_subordinate = False
if ("mongodb" in applications[application]["charm"]
or "vault" in applications[application]["charm"]):
# Mongodb and vault need to run series upgrade
# on its secondaries first.
generic_utils.series_upgrade_non_leaders_first(
application,
from_series=self.from_series,
to_series=self.to_series,
completed_machines=completed_machines)
continue
# The rest are likley APIs use defaults
generic_utils.series_upgrade_application(
application,
pause_non_leader_primary=pause_non_leader_primary,
pause_non_leader_subordinate=pause_non_leader_subordinate,
from_series=self.from_series,
to_series=self.to_series,
origin=origin,
completed_machines=completed_machines,
workaround_script=self.workaround_script,
files=self.files)
def get_full_juju_status(model_name=None):
"""Return the full juju status output.
:param model_name: Name of model to query.
:type model_name: str
:returns: Full juju status output
:rtype: dict
"""
status = model.get_status(model_name=model_name)
return status
def test_200_run_series_upgrade(self):
"""Run series upgrade."""
# Set Feature Flag
os.environ["JUJU_DEV_FEATURE_FLAGS"] = "upgrade-series"
applications = model.get_status().applications
completed_machines = []
for application, app_details in applications.items():
# Skip subordinates
if app_details["subordinate-to"]:
continue
if "easyrsa" in app_details["charm"]:
logging.warn(
"Skipping series upgrade of easyrsa Bug #1850121")
continue
if "etcd" in app_details["charm"]:
logging.warn(
"Skipping series upgrade of etcd Bug #1850124")
continue
charm_name = upgrade_utils.extract_charm_name_from_url(
app_details['charm'])
upgrade_config = series_upgrade_utils.app_config(
charm_name,
is_async=False)
upgrade_function = upgrade_config.pop('upgrade_function')
logging.warn("About to upgrade {}".format(application))
upgrade_function(
application,
**upgrade_config,
from_series=self.from_series,
to_series=self.to_series,
completed_machines=completed_machines,
workaround_script=self.workaround_script,
files=self.files,
)
if "rabbitmq-server" in app_details["charm"]:
logging.info(
"Running complete-cluster-series-upgrade action on leader")
model.run_action_on_leader(
application,
'complete-cluster-series-upgrade',
action_params={})
model.block_until_all_units_idle()
if "percona-cluster" in app_details["charm"]:
logging.info(
"Running complete-cluster-series-upgrade action on leader")
model.run_action_on_leader(
application,
'complete-cluster-series-upgrade',
action_params={})
model.block_until_all_units_idle()
def test_200_run_series_upgrade(self):
"""Run series upgrade."""
# Set Feature Flag
os.environ["JUJU_DEV_FEATURE_FLAGS"] = "upgrade-series"
upgrade_groups = upgrade_utils.get_series_upgrade_groups(
extra_filters=[_filter_etcd, _filter_easyrsa],
target_series=self.to_series)
from_series = self.from_series
to_series = self.to_series
completed_machines = []
workaround_script = None
files = []
applications = model.get_status().applications
for group_name, apps in upgrade_groups:
logging.info("About to upgrade {} from {} to {}".format(
group_name, from_series, to_series))
upgrade_functions = []
if group_name in [
"Database Services", "Stateful Services", "Data Plane",
"sweep_up"
]:
logging.info("Going to upgrade {} unit by unit".format(apps))
upgrade_function = \
parallel_series_upgrade.serial_series_upgrade
else:
logging.info("Going to upgrade {} all at once".format(apps))
upgrade_function = \
parallel_series_upgrade.parallel_series_upgrade
# allow up to 4 parallel upgrades at a time. This is to limit the
# amount of data/calls that asyncio is handling as it's gets
# unstable if all the applications are done at the same time.
sem = asyncio.Semaphore(4)
for charm_name in apps:
charm = applications[charm_name]['charm']
name = upgrade_utils.extract_charm_name_from_url(charm)
upgrade_config = parallel_series_upgrade.app_config(name)
upgrade_functions.append(
wrap_coroutine_with_sem(
sem,
upgrade_function(charm_name,
**upgrade_config,
from_series=from_series,
to_series=to_series,
completed_machines=completed_machines,
workaround_script=workaround_script,
files=files)))
asyncio.get_event_loop().run_until_complete(
asyncio.gather(*upgrade_functions))
model.block_until_all_units_idle()
logging.info("Finished {}".format(group_name))
logging.info("Done!")
def test5_upfgtp_connection(self):
""" ***** checking gtp connection in upf ***** """
upf_gtp_port = 2152
sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
for unit in model.get_units("upf1"):
upf1_ip = model.get_status().applications["upf1"]["units"][
unit.entity_id]["address"]
result = sock.connect_ex((upf1_ip, upf_gtp_port))
if result == 0:
logging.info("GTP Transport is Listening ...")
else:
logging.info("GTP Transport is not available")
self.assertEqual(result, 0)
def test4_sctp_connection_amf(self):
""" ***** checking sctp transport connection in amf ***** """
amf_sctp_port = 38412
sock = sctp.sctpsocket_tcp(socket.AF_INET)
for unit in model.get_units("amf"):
amf_ip = model.get_status().applications["amf"]["units"][
unit.entity_id]["address"]
result = sock.connect_ex((amf_ip, amf_sctp_port))
if result == 0:
logging.info("SCTP Transport is Listening ...")
else:
logging.info("SCTP Transport is not available")
self.assertEqual(result, 0)
def get_applications_with_substring_in_name(self, substring):
"""Get applications with substring in name.
:param substring: String to search for in application names
:type substring: str
:returns: List of matching applictions
:rtype: List
"""
status = model.get_status().applications
applications = []
for application in status.keys():
if substring in application:
applications.append(application)
return applications
def test6_tcp_scscf_connection(self):
""" ***** Checking scscf TCP connection ***** """
for unit in model.get_units("scscf"):
logging.info("Checking if the unit scscf is active: %s",
unit.entity_id)
scscf_ip = model.get_status().applications["scscf"]["units"][
unit.entity_id]["address"]
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
result = sock.connect_ex((scscf_ip, 6060))
if result == 0:
logging.info("scscf unit has TCP connection Established!!")
self.assertEqual(0, result)
else:
logging.info("Failed to connect")
def create_connection():
""" creating mysqldb connection """
try:
for unit in model.get_units("mysql"):
logging.info("Checking if the unit db is active: %s",
unit.entity_id)
logging.info("checking for mysql db connection ......")
db_ip = model.get_status().applications["mysql"]["units"][
unit.entity_id]["address"]
myclient = pymysql.connect(db_ip, "root", "root", "hss_db")
logging.info("Mysqldb connected successfully !!!")
except pymysql.Error:
logging.info("Could not connect to Mysqldb")
return myclient
def test_200_run_series_upgrade(self):
"""Run series upgrade."""
# Set Feature Flag
os.environ["JUJU_DEV_FEATURE_FLAGS"] = "upgrade-series"
upgrade_groups = upgrade_utils.get_series_upgrade_groups(
extra_filters=[
upgrade_utils._filter_etcd, upgrade_utils._filter_easyrsa
],
target_series=self.to_series)
applications = model.get_status().applications
completed_machines = []
for group_name, group in upgrade_groups:
logging.warn("About to upgrade {} ({})".format(group_name, group))
upgrade_group = []
for application, app_details in applications.items():
if application not in group:
continue
charm_name = upgrade_utils.extract_charm_name_from_url(
app_details['charm'])
upgrade_config = series_upgrade_utils.app_config(charm_name)
upgrade_function = upgrade_config.pop('upgrade_function')
logging.warn("About to upgrade {}".format(application))
upgrade_group.append(
upgrade_function(
application,
**upgrade_config,
from_series=self.from_series,
to_series=self.to_series,
completed_machines=completed_machines,
workaround_script=self.workaround_script,
files=self.files,
))
asyncio.get_event_loop().run_until_complete(
asyncio.gather(*upgrade_group))
if "rabbitmq-server" in group:
logging.info(
"Running complete-cluster-series-upgrade action on leader")
model.run_action_on_leader('rabbitmq-server',
'complete-cluster-series-upgrade',
action_params={})
model.block_until_all_units_idle()
if "percona-cluster" in group:
logging.info(
"Running complete-cluster-series-upgrade action on leader")
model.run_action_on_leader('mysql',
'complete-cluster-series-upgrade',
action_params={})
model.block_until_all_units_idle()
def create_connection():
""" create mongo mongodb connection """
try:
for unit in model.get_units("mongodb"):
logging.info("Checking if the unit mongodb is active: %s",
unit.entity_id)
logging.info("checking for mongodb connection....")
mongodb_ip = model.get_status().applications["mongodb"]["units"][
unit.entity_id]["address"]
myclient = pymongo.MongoClient("mongodb://" + mongodb_ip +
":27017/")
logging.info("Mongodb connected successfully !!!")
except pymongo.errors.ConnectionFailure:
logging.info("Could not connect to Mongomongodb")
return myclient
def get_leaders_and_non_leaders(application_name):
"""Get leader node and non-leader nodes.
:returns: leader, list of non-leader
:rtype: str, list of str
"""
status = model.get_status().applications[application_name]
leader = None
non_leaders = []
for unit in status["units"]:
if status["units"][unit].get("leader"):
leader = unit
else:
non_leaders.append(unit)
return leader, non_leaders
def test_rest_service_call(self):
""" ***** Testing the gnB config rest service *****"""
for unit in zaza.model.get_units("ran"):
ran_ip = model.get_status().applications["ran"]["units"][
unit.entity_id]["address"]
ENDPOINT = "http://" + ran_ip + ":8081/configread"
data = {
"Global": {
"mcc": "208",
"mnc": "93",
"gnbid": "454647"
},
"supportlist": [{
"tac":
"0123",
"broadplmnlist": [{
"mcc": "208",
"mnc": "93",
"slicesupport": [{
"sst": 1,
"sd": "010203"
}],
}],
}],
"paging":
"v34",
"gnbname":
"Tata",
"amfconfig": {
"amfip": "10.45.28.52",
"amfport": "38412",
"gnbport": "9487",
"ngapinterface": "eth0",
},
"upfconfig": {
"upfip": "10.45.28.51"
},
"pdnconfig": {
"pdninterface": "eth1"
},
}
result = requests.post(url=ENDPOINT, json=data)
if result.status_code == 200:
logging.info("The request was a Success !!")
self.assertEqual(200, result.status_code)
else:
logging.info("Request Failed..")
def get_subordinate_units(unit_list,
charm_name=None,
status=None,
model_name=None):
"""Get a list of all subordinate units associated with units in unit_list.
Get a list of all subordinate units associated with units in unit_list.
Subordinate can be filtered by using 'charm_name' which will only return
subordinate units which have 'charm_name' in the name of the charm e.g.
get_subordinate_units(
['cinder/1']) would return ['cinder-hacluster/1',
'cinder-ceph/2'])
where as
get_subordinate_units(
['cinder/1'], charm_name='hac') would return ['cinder-hacluster/1']
NOTE: The charm_name match is against the name of the charm not the
application name.
:param charm_name: List of unit names
:type unit_list: []
:param charm_name: charm_name to match against, can be a sub-string.
:type charm_name: str
:param status: Juju status to query against,
:type status: juju.client._definitions.FullStatus
:param model_name: Name of model to query.
:type model_name: str
:returns: List of matching unit names.
:rtype: []
"""
if not status:
status = model.get_status(model_name=model_name)
sub_units = []
for unit_name in unit_list:
app_name = unit_name.split('/')[0]
subs = status.applications[app_name]['units'][unit_name].get(
'subordinates') or {}
if charm_name:
for unit_name, unit_data in subs.items():
if charm_name in unit_data['charm']:
sub_units.append(unit_name)
else:
sub_units.extend([n for n in subs.keys()])
return sub_units
def encrypt_vip_endpoints():
"""Apply certs and keys to all charms that support them."""
(cakey, cacert) = create_ca()
ssl_vip_keys = ['vip', 'ssl_ca', 'ssl_cert', 'ssl_key']
status = model.get_status()
for application_name in status.applications.keys():
app_config = model.get_application_config(application_name)
if all(k in app_config for k in ssl_vip_keys):
cn = app_config['vip'].get('value')
# If there is no vip check if its a non-ha deploy.
if not cn:
units = model.get_units(application_name)
if len(units) == 1:
cn = units[0].public_address
if cn:
create_certs(application_name, cn, ISSUER_NAME, cakey)
apply_certs(application_name)
model.block_until_all_units_idle()
def destroy(model_name):
"""Run all steps to cleaup after a test run.
Note: on the OpenStack provider we also verify after the destroy model call
that the instances associated with the model really are gone. Reap any
instances that have the model name in them before returning.
Bug: https://bugs.launchpad.net/juju/+bug/1913418
:param model: Name of model to remove
:type bundle: str
"""
machines = model.get_status()["machines"]
zaza.controller.destroy_model(model_name)
if juju_utils.get_provider_type() == "openstack":
# only import openstack_provider if it's needed. This avoids forcing
# zaza to have dependencies for providers that the user isn't using.
import zaza.utilities.openstack_provider as op
op.clean_up_instances(model_name, machines)
def test_200_run_series_upgrade(self):
"""Run series upgrade."""
# Set Feature Flag
os.environ["JUJU_DEV_FEATURE_FLAGS"] = "upgrade-series"
upgrade_groups = upgrade_utils.get_series_upgrade_groups(
extra_filters=[_filter_etcd, _filter_easyrsa])
from_series = self.from_series
to_series = self.to_series
completed_machines = []
workaround_script = None
files = []
applications = model.get_status().applications
for group_name, apps in upgrade_groups.items():
logging.info("About to upgrade {} from {} to {}".format(
group_name, from_series, to_series))
upgrade_functions = []
if group_name in ["Stateful Services", "Data Plane", "sweep_up"]:
logging.info("Going to upgrade {} unit by unit".format(apps))
upgrade_function = \
parallel_series_upgrade.serial_series_upgrade
else:
logging.info("Going to upgrade {} all at once".format(apps))
upgrade_function = \
parallel_series_upgrade.parallel_series_upgrade
for charm_name in apps:
charm = applications[charm_name]['charm']
name = upgrade_utils.extract_charm_name_from_url(charm)
upgrade_config = parallel_series_upgrade.app_config(name)
upgrade_functions.append(
upgrade_function(charm_name,
**upgrade_config,
from_series=from_series,
to_series=to_series,
completed_machines=completed_machines,
workaround_script=workaround_script,
files=files))
asyncio.get_event_loop().run_until_complete(
asyncio.gather(*upgrade_functions))
model.block_until_all_units_idle()
logging.info("Finished {}".format(group_name))
logging.info("Done!")
def series_upgrade_application(application, pause_non_leader_primary=True,
pause_non_leader_subordinate=True,
from_series="trusty", to_series="xenial",
origin='openstack-origin',
completed_machines=[],
files=None, workaround_script=None,
post_upgrade_functions=None):
"""Series upgrade application.
Wrap all the functionality to handle series upgrade for a given
application. Including pausing non-leader units.
:param application: Name of application to upgrade series
:type application: str
:param pause_non_leader_primary: Whether the non-leader applications should
be paused
:type pause_non_leader_primary: bool
:param pause_non_leader_subordinate: Whether the non-leader subordinate
hacluster applications should be
paused
:type pause_non_leader_subordinate: bool
:param from_series: The series from which to upgrade
:type from_series: str
:param to_series: The series to which to upgrade
:type to_series: str
:param origin: The configuration setting variable name for changing origin
source. (openstack-origin or source)
:type origin: str
:param completed_machines: List of completed machines which do no longer
require series upgrade.
:type completed_machines: list
:param files: Workaround files to scp to unit under upgrade
:type files: list
:param workaround_script: Workaround script to run during series upgrade
:type workaround_script: str
:returns: None
:rtype: None
"""
status = model.get_status().applications[application]
# For some applications (percona-cluster) the leader unit must upgrade
# first. For API applications the non-leader haclusters must be paused
# before upgrade. Finally, for some applications this is arbitrary but
# generalized.
leader = None
non_leaders = []
for unit in status["units"]:
if status["units"][unit].get("leader"):
leader = unit
else:
non_leaders.append(unit)
# Pause the non-leaders
for unit in non_leaders:
if pause_non_leader_subordinate:
if status["units"][unit].get("subordinates"):
for subordinate in status["units"][unit]["subordinates"]:
_app = subordinate.split('/')[0]
if _app in SUBORDINATE_PAUSE_RESUME_BLACKLIST:
logging.info("Skipping pausing {} - blacklisted"
.format(subordinate))
else:
logging.info("Pausing {}".format(subordinate))
model.run_action(
subordinate, "pause", action_params={})
if pause_non_leader_primary:
logging.info("Pausing {}".format(unit))
model.run_action(unit, "pause", action_params={})
machine = status["units"][leader]["machine"]
# Series upgrade the leader
logging.info("Series upgrade leader: {}".format(leader))
if machine not in completed_machines:
series_upgrade(leader, machine,
from_series=from_series, to_series=to_series,
origin=origin, workaround_script=workaround_script,
files=files,
post_upgrade_functions=post_upgrade_functions)
completed_machines.append(machine)
else:
logging.info("Skipping unit: {}. Machine: {} already upgraded."
"But setting origin on the application {}"
.format(unit, machine, application))
logging.info("Set origin on {}".format(application))
set_origin(application, origin)
model.block_until_all_units_idle()
# Series upgrade the non-leaders
for unit in non_leaders:
machine = status["units"][unit]["machine"]
if machine not in completed_machines:
logging.info("Series upgrade non-leader unit: {}"
.format(unit))
series_upgrade(unit, machine,
from_series=from_series, to_series=to_series,
origin=origin, workaround_script=workaround_script,
files=files,
post_upgrade_functions=post_upgrade_functions)
completed_machines.append(machine)
else:
logging.info("Skipping unit: {}. Machine: {} already upgraded. "
"But setting origin on the application {}"
.format(unit, machine, application))
logging.info("Set origin on {}".format(application))
set_origin(application, origin)
model.block_until_all_units_idle()
def test_200_run_series_upgrade(self):
"""Run series upgrade."""
# Set Feature Flag
os.environ["JUJU_DEV_FEATURE_FLAGS"] = "upgrade-series"
applications = model.get_status().applications
completed_machines = []
for application in applications:
# Defaults
origin = "openstack-origin"
pause_non_leader_subordinate = True
pause_non_leader_primary = True
post_upgrade_functions = []
# Skip subordinates
if applications[application]["subordinate-to"]:
continue
if "percona-cluster" in applications[application]["charm"]:
origin = "source"
pause_non_leader_primary = True
pause_non_leader_subordinate = True
if "rabbitmq-server" in applications[application]["charm"]:
origin = "source"
pause_non_leader_primary = True
pause_non_leader_subordinate = False
if "nova-compute" in applications[application]["charm"]:
pause_non_leader_primary = False
pause_non_leader_subordinate = False
if "ceph" in applications[application]["charm"]:
origin = "source"
pause_non_leader_primary = False
pause_non_leader_subordinate = False
if "designate-bind" in applications[application]["charm"]:
origin = None
if "tempest" in applications[application]["charm"]:
origin = None
if "memcached" in applications[application]["charm"]:
origin = None
pause_non_leader_primary = False
pause_non_leader_subordinate = False
if "vault" in applications[application]["charm"]:
post_upgrade_functions = [
('zaza.openstack.charm_tests.vault.setup.'
'basic_setup_and_unseal')]
if ("mongodb" in applications[application]["charm"] or
"vault" in applications[application]["charm"]):
# Mongodb and vault need to run series upgrade
# on its secondaries first.
generic_utils.series_upgrade_non_leaders_first(
application,
from_series=self.from_series,
to_series=self.to_series,
completed_machines=completed_machines,
post_upgrade_functions=post_upgrade_functions)
continue
# The rest are likley APIs use defaults
generic_utils.series_upgrade_application(
application,
pause_non_leader_primary=pause_non_leader_primary,
pause_non_leader_subordinate=pause_non_leader_subordinate,
from_series=self.from_series,
to_series=self.to_series,
origin=origin,
completed_machines=completed_machines,
workaround_script=self.workaround_script,
files=self.files,
post_upgrade_functions=post_upgrade_functions)
if "rabbitmq-server" in applications[application]["charm"]:
logging.info(
"Running complete-cluster-series-upgrade action on leader")
model.run_action_on_leader(
'rabbitmq-server',
'complete-cluster-series-upgrade',
action_params={})
model.block_until_all_units_idle()