def setUp(self):
self.sys_path_save = sys.path
# Setting up the config module
from xosconfig import Config
config = os.path.join(test_path, "test_config.yaml")
Config.clear()
Config.init(config, "synchronizer-config-schema.yaml")
# END Setting up the config module
from xossynchronizer.mock_modelaccessor_build import build_mock_modelaccessor
# FIXME this is to get jenkins to pass the tests, somehow it is running tests in a different order
# and apparently it is not overriding the generated model accessor
build_mock_modelaccessor(sync_lib_dir, xos_dir, services_dir, [])
import xossynchronizer.modelaccessor
# import all class names to globals
for (
k,
v,
) in xossynchronizer.modelaccessor.model_accessor.all_model_classes.items():
globals()[k] = v
self.log = Mock()
def setUp(self):
global ComputeNodePolicy, MockObjectList
self.sys_path_save = sys.path
config = os.path.join(test_path, "../test_config.yaml")
from xosconfig import Config
Config.clear()
Config.init(config, 'synchronizer-config-schema.yaml')
from xossynchronizer.mock_modelaccessor_build import mock_modelaccessor_config
mock_modelaccessor_config(test_path, [("fabric", "fabric.xproto")])
import xossynchronizer.modelaccessor
import mock_modelaccessor
imp.reload(mock_modelaccessor) # in case nose2 loaded it in a previous test
imp.reload(xossynchronizer.modelaccessor) # in case nose2 loaded it in a previous test
from model_policy_compute_nodes import ComputeNodePolicy, model_accessor
self.model_accessor = model_accessor
from mock_modelaccessor import MockObjectList
# import all class names to globals
for (k, v) in model_accessor.all_model_classes.items():
globals()[k] = v
# Some of the functions we call have side-effects. For example, creating a VSGServiceInstance may lead to
# creation of tags. Ideally, this wouldn't happen, but it does. So make sure we reset the world.
model_accessor.reset_all_object_stores()
self.policy = ComputeNodePolicy
self.model = Mock()
def test_get_default_val_for_missing_param(self):
"""
[XOS-Config] Should get the default value if nothing is specified
"""
Config.init(basic_conf)
dir = Config.get("xos_dir")
self.assertEqual(dir, "/opt/xos")
def test_get_child_level(self):
"""
[XOS-Config] Should return a child level param
"""
Config.init(sample_conf)
res = Config.get("database.name")
self.assertEqual(res, "xos")
def _test_get_child_level(self):
"""
[XOS-Config] Should return a child level param
"""
Config.init(sample_conf)
res = Config.get("nested.parameter.for")
self.assertEqual(res, "testing")
def init():
global log
global kafka_producer
if not log:
log = create_logger(Config().get("logging"))
if kafka_producer:
raise Exception("XOSKafkaProducer already initialized")
else:
log.info(
"Connecting to Kafka with bootstrap servers: %s"
% Config.get("kafka_bootstrap_servers")
)
try:
producer_config = {
"bootstrap.servers": ",".join(Config.get("kafka_bootstrap_servers"))
}
kafka_producer = confluent_kafka.Producer(**producer_config)
log.info("Connected to Kafka: %s" % kafka_producer)
except confluent_kafka.KafkaError as e:
log.exception("Kafka Error: %s" % e)
def test_missing_file_exception(self):
"""
[XOS-Config] Raise if file not found
"""
with self.assertRaises(Exception) as e:
Config.init("missing_conf")
self.assertEqual(e.exception.message, "[XOS-Config] Config file not found at: missing_conf")
def config_accessor_grpcapi():
global orig_sigint
grpcapi_endpoint = Config.get("accessor.endpoint")
grpcapi_username = Config.get("accessor.username")
grpcapi_password = Config.get("accessor.password")
# if password starts with "@", then retreive the password from a file
if grpcapi_password.startswith("@"):
fn = grpcapi_password[1:]
if not os.path.exists(fn):
raise Exception("%s does not exist" % fn)
grpcapi_password = open(fn).readline().strip()
from xosapi.xos_grpc_client import SecureClient
from twisted.internet import reactor
grpcapi_client = SecureClient(endpoint=grpcapi_endpoint, username=grpcapi_username, password=grpcapi_password)
grpcapi_client.set_reconnect_callback(functools.partial(grpcapi_reconnect, grpcapi_client, reactor))
grpcapi_client.start()
# Start reactor. This will cause the client to connect and then execute
# grpcapi_callback().
# Reactor will take over SIGINT during reactor.run(), but does not return it when reactor.stop() is called.
orig_sigint = signal.getsignal(signal.SIGINT)
# Start reactor. This will cause the client to connect and then execute
# grpcapi_callback().
reactor.run()
def test_config_not_initialized(self):
"""
[XOS-Config] Raise if accessing properties without initialization
"""
with self.assertRaises(Exception) as e:
Config.get("database")
self.assertEqual(e.exception.message, "[XOS-Config] Module has not been initialized")
def test_get_config_file(self):
"""
[XOS-Config] Should return the config file in use
"""
Config.init(sample_conf)
res = Config.get_config_file()
self.assertEqual(res, sample_conf)
def setUp(self):
self.sys_path_save = sys.path
self.cwd_save = os.getcwd()
config = os.path.join(test_path, "test_config.yaml")
from xosconfig import Config
Config.clear()
Config.init(config, "synchronizer-config-schema.yaml")
from xossynchronizer.mock_modelaccessor_build import (
build_mock_modelaccessor,
)
build_mock_modelaccessor(sync_lib_dir, xos_dir, services_dir=None, service_xprotos=[])
# The test config.yaml references files in `xos-synchronizer-tests/` so make sure we're in the parent
# directory of the test directory.
os.chdir(os.path.join(test_path, ".."))
import xossynchronizer.event_loop
reload(xossynchronizer.event_loop)
import xossynchronizer.backend
reload(xossynchronizer.backend)
from xossynchronizer.modelaccessor import model_accessor
b = xossynchronizer.backend.Backend(model_accessor=model_accessor)
steps_dir = Config.get("steps_dir")
self.steps = b.load_sync_step_modules(steps_dir)
self.synchronizer = xossynchronizer.event_loop.XOSObserver(self.steps, model_accessor)
def run_playbook(ansible_hosts, ansible_config, fqp, opts):
args = {"ansible_hosts": ansible_hosts,
"ansible_config": ansible_config,
"fqp": fqp,
"opts": opts,
"config_file": Config.get_config_file()}
keep_temp_files = Config.get("keep_temp_files")
dir = tempfile.mkdtemp()
args_fn = None
result_fn = None
try:
log.info("creating args file",dir = dir)
args_fn = os.path.join(dir, "args")
result_fn = os.path.join(dir, "result")
open(args_fn, "w").write(pickle.dumps(args))
ansible_main_fn = os.path.join(os.path.dirname(__file__), "ansible_main.py")
os.system("python %s %s %s" % (ansible_main_fn, args_fn, result_fn))
result = pickle.loads(open(result_fn).read())
if hasattr(result, "exception"):
log.error("Exception in playbook",exception = result["exception"])
stats = result.get("stats", None)
aresults = result.get("aresults", None)
except Exception,e:
log.exception("Exception running ansible_main")
stats = None
aresults = None
def test_get_missing_param(self):
"""
[XOS-Config] Should return None reading a missing param
"""
Config.init(sample_conf)
res = Config.get("foo")
self.assertEqual(res, None)
def test_yaml_not_valid(self):
"""
[XOS-Config] Raise if yaml is not valid
"""
with self.assertRaises(Exception) as e:
Config.init(yaml_not_valid)
self.assertEqual(e.exception.message, "[XOS-Config] The config format is wrong: Unable to load any data from source yaml file")
def test_invalid_format(self):
"""
[XOS-Config] Raise if format is not valid (we expect a dictionary)
"""
with self.assertRaises(Exception) as e:
Config.init(invalid_format)
self.assertEqual(e.exception.message, "[XOS-Config] The config format is wrong: Schema validation failed:\n - Value '['I am', 'a yaml', 'but the', 'format is not', 'correct']' is not a dict. Value path: ''.")
def test_initialize_only_once(self):
"""
[XOS-Config] Raise if initialized twice
"""
with self.assertRaises(Exception) as e:
Config.init(sample_conf)
Config2.init(sample_conf)
self.assertEqual(str(e.exception), "[XOS-Config] Module already initialized")
def test_missing_get_service_info(self):
"""
[XOS-Config] Should query registrator and return an exception if service is not there
"""
with patch("xosconfig.config.requests.get") as mock_get:
mock_get.return_value.json.return_value = []
with self.assertRaises(Exception) as e:
Config.get_service_info("missing-service")
self.assertEqual(e.exception.message, "[XOS-Config] The service missing-service looking for does not exist")
def test_yaml_not_valid(self):
"""
[XOS-Config] Raise if yaml is not valid
"""
with self.assertRaises(Exception) as e:
Config.init(yaml_not_valid)
self.assertTrue(
str(e.exception).startswith("[XOS-Config] The config format is wrong:")
)
def test_get_first_level(self):
"""
[XOS-Config] Should return a first level param
"""
Config.init(sample_conf)
# NOTE we are using Config2 here to be sure that the configuration is readable from any import,
# not only from the one that has been used to initialize it
res = Config2.get("database")
self.assertEqual(res, {"name": "xos", "username": "******", "password": "******"})
def test_schema_override_usage(self):
"""
[XOS-Config] the XOS_CONFIG_SCHEMA should be used to validate a config
"""
os.environ["XOS_CONFIG_SCHEMA"] = small_schema
with self.assertRaises(Exception) as e:
Config.init(basic_conf)
self.assertEqual(e.exception.message, "[XOS-Config] The config format is wrong: Schema validation failed:\n - Key 'database' was not defined. Path: ''.")
del os.environ["XOS_CONFIG_SCHEMA"]
def test_fail_get_service_info(self):
"""
[XOS-Config] Should query registrator and return an exception if it"s down
"""
with patch("xosconfig.config.requests.get") as mock_get:
mock_get.return_value.ok = False
with self.assertRaises(Exception) as e:
Config.get_service_info("missing-service")
self.assertEqual(e.exception.message, "[XOS-Config] Registrator is down")
def test_config_override(self):
"""
[XOS-Config] If an override is provided for the config, it should return the overridden value
"""
Config.init(sample_conf, "xos-config-schema.yaml", override_conf)
res = Config.get("logging.level")
self.assertEqual(res, "info")
res = Config.get("database.password")
self.assertEqual(res, "overridden_password")
def test_env_override(self):
"""
[XOS-Config] the XOS_CONFIG_FILE environment variable should override the config_file
"""
os.environ["XOS_CONFIG_FILE"] = "env.yaml"
with self.assertRaises(Exception) as e:
Config.init("missing_conf")
self.assertEqual(e.exception.message, "[XOS-Config] Config file not found at: env.yaml")
del os.environ["XOS_CONFIG_FILE"]
def setUp(self):
from xosconfig import Config
test_path = os.path.abspath(os.path.dirname(os.path.realpath(__file__)))
config = os.path.join(test_path, "test_config.yaml")
Config.clear()
Config.init(config, "synchronizer-config-schema.yaml")
if USE_FAKE_STUB:
sys.path.append(PARENT_DIR)
def configure_logging(verbose):
global log
# INITIALIZING LOGGER
Config.init()
cfg = Config().get("logging")
if verbose:
cfg["handlers"]["console"]["level"] = "DEBUG"
log = create_logger(cfg)
def test_schema_override(self):
"""
[XOS-Config] the XOS_CONFIG_SCHEMA environment variable should override the config_schema
"""
os.environ["XOS_CONFIG_SCHEMA"] = "env-schema.yaml"
with self.assertRaises(Exception) as e:
Config.init(basic_conf)
self.assertRegexpMatches(e.exception.message, '\[XOS\-Config\] Config schema not found at: (.+)env-schema\.yaml')
# self.assertEqual(e.exception.message, "[XOS-Config] Config schema not found at: env-schema.yaml")
del os.environ["XOS_CONFIG_SCHEMA"]
def test_config_extend(self):
"""
[XOS-Config] If an override is provided for the config, it should
return the overridden value (also if not defined in the base one)
"""
Config.init(sample_conf, "xos-config-schema.yaml", extend_conf)
res = Config.get("xos_dir")
self.assertEqual(res, "/opt/xos")
res = Config.get("database.password")
self.assertEqual(res, "safe")
def setUp(self):
self.sys_path_save = sys.path
# Setting up the config module
from xosconfig import Config
config = os.path.join(test_path, "../test_config.yaml")
Config.clear()
Config.init(config, "synchronizer-config-schema.yaml")
# END Setting up the config module
from xossynchronizer.mock_modelaccessor_build import mock_modelaccessor_config
mock_modelaccessor_config(test_path, [("vrouter", "vrouter.xproto")])
import xossynchronizer.modelaccessor
import mock_modelaccessor
reload(mock_modelaccessor) # in case nose2 loaded it in a previous test
reload(xossynchronizer.modelaccessor) # in case nose2 loaded it in a previous test
from sync_routes import SyncRoutes, model_accessor
self.model_accessor = model_accessor
# import all class names to globals
for (k, v) in model_accessor.all_model_classes.items():
globals()[k] = v
self.sync_step = SyncRoutes
self.sync_step.log = Mock()
# mock onos-fabric
onos_fabric = Mock()
onos_fabric.name = "onos-fabric"
onos_fabric.rest_hostname = "onos-fabric"
onos_fabric.rest_port = "8181"
onos_fabric.rest_username = "******"
onos_fabric.rest_password = "******"
onos_fabric_base = Mock()
onos_fabric_base.leaf_model = onos_fabric
self.fabric = Mock()
self.fabric.name = "fabric"
self.fabric.provider_services = [onos_fabric_base]
self.vrouter = Mock()
self.vrouter.name = "vrouter"
self.vrouter.provider_services = [self.fabric]
# create a mock VRouterStaticRoute instance
self.o = Mock()
self.o.id = 1
self.o.vrouter.owner = self.vrouter
self.o.tologdict.return_value = {}
def test_get_default_val_for_missing_param(self):
"""
[XOS-Config] Should get the default value if nothing is specified
"""
Config.init(basic_conf)
log = Config.get("logging")
self.assertEqual(log, {
"level": "info",
"channels": ["file", "console"],
"logstash_hostport": "cordloghost:5617",
"file": "/var/log/xos.log",
})
def main():
base_config_file = os.path.abspath(os.path.dirname(
os.path.realpath(__file__)) + '/config.yaml')
mounted_config_file = os.path.abspath(os.path.dirname(
os.path.realpath(__file__)) + '/mounted_config.yaml')
if os.path.isfile(mounted_config_file):
Config.init(base_config_file, 'synchronizer-config-schema.yaml',
mounted_config_file)
else:
Config.init(base_config_file, 'synchronizer-config-schema.yaml')
Synchronizer().run()
def setUp(self):
global DeferredException
self.sys_path_save = sys.path
# Setting up the config module
from xosconfig import Config
config = os.path.join(test_path,
"../test_fabric_crossconnect_config.yaml")
Config.clear()
Config.init(config, "synchronizer-config-schema.yaml")
# END Setting up the config module
from xossynchronizer.mock_modelaccessor_build import mock_modelaccessor_config
mock_modelaccessor_config(test_path, [
("fabric-crossconnect", "fabric-crossconnect.xproto"),
])
import xossynchronizer.modelaccessor
import mock_modelaccessor
reload(
mock_modelaccessor) # in case nose2 loaded it in a previous test
reload(xossynchronizer.modelaccessor
) # in case nose2 loaded it in a previous test
from xossynchronizer.modelaccessor import model_accessor
self.model_accessor = model_accessor
from sync_fabric_crossconnect_service_instance import SyncFabricCrossconnectServiceInstance, model_accessor, \
DeferredException
from helpers import Helpers
self.helpers = Helpers
# import all class names to globals
for (k, v) in model_accessor.all_model_classes.items():
globals()[k] = v
self.sync_step = SyncFabricCrossconnectServiceInstance
self.sync_step.log = Mock()
# mock onos-fabric
self.onos_fabric = Service(name="onos-fabric",
rest_hostname="onos-fabric",
rest_port="8181",
rest_username="******",
rest_password="******")
self.service = FabricCrossconnectService(
name="fcservice", provider_services=[self.onos_fabric])
def setUpClass(cls):
global log
config = os.path.join(test_path, "test_config.yaml")
from xosconfig import Config
Config.clear()
Config.init(config, "synchronizer-config-schema.yaml")
if not log:
from multistructlog import create_logger
log = create_logger(Config().get("logging"))
def config_accessor():
accessor_kind = Config.get("accessor.kind")
if accessor_kind == "testframework":
config_accessor_mock()
elif accessor_kind == "grpcapi":
config_accessor_grpcapi()
else:
raise Exception("Unknown accessor kind %s" % accessor_kind)
# now import any wrappers that the synchronizer needs to add to the ORM
if Config.get("wrappers"):
for wrapper_name in Config.get("wrappers"):
importlib.import_module(wrapper_name)
def setUp(self):
config = os.path.abspath(
os.path.dirname(os.path.realpath(__file__)) + "/test_config.yaml")
Config.clear() # in case left unclean by a previous test case
Config.init(config)
import backupprocessor
self.backupprocessor = backupprocessor
self.setUpPyfakefs()
self.processor = backupprocessor.BackupProcessor()
self.mock_backuphandler = MagicMock(backup=MagicMock(),
restore=MagicMock())
def setUp(self):
self.sys_path_save = sys.path
self.cwd_save = os.getcwd()
sys.path.append(xos_dir)
sys.path.append(os.path.join(xos_dir, "synchronizers", "new_base"))
sys.path.append(
os.path.join(xos_dir, "synchronizers", "new_base", "tests",
"steps"))
config = os.path.join(test_path, "test_config.yaml")
from xosconfig import Config
Config.clear()
Config.init(config, "synchronizer-config-schema.yaml")
from synchronizers.new_base.mock_modelaccessor_build import (
build_mock_modelaccessor, )
build_mock_modelaccessor(xos_dir,
services_dir=None,
service_xprotos=[])
os.chdir(os.path.join(test_path,
"..")) # config references tests/model-deps
import event_loop
reload(event_loop)
import backend
reload(backend)
from modelaccessor import model_accessor
# import all class names to globals
for (k, v) in model_accessor.all_model_classes.items():
globals()[k] = v
b = backend.Backend()
steps_dir = Config.get("steps_dir")
self.steps = b.load_sync_step_modules(steps_dir)
self.synchronizer = event_loop.XOSObserver(self.steps)
try:
os.remove("/tmp/sync_ports")
except OSError:
pass
try:
os.remove("/tmp/delete_ports")
except OSError:
pass
def setUp(self):
global TenantWithContainerPolicy, LeastLoadedNodeScheduler, MockObjectList
self.sys_path_save = sys.path
self.cwd_save = os.getcwd()
sys.path.append(xos_dir)
sys.path.append(os.path.join(xos_dir, "synchronizers", "new_base"))
sys.path.append(
os.path.join(xos_dir, "synchronizers", "new_base",
"model_policies"))
config = basic_conf = os.path.abspath(
os.path.dirname(os.path.realpath(__file__)) + "/test_config.yaml")
Config.clear() # in case left unclean by a previous test case
Config.init(config, "synchronizer-config-schema.yaml")
from synchronizers.new_base.mock_modelaccessor_build import (
build_mock_modelaccessor, )
build_mock_modelaccessor(xos_dir,
services_dir=None,
service_xprotos=[])
import model_policy_tenantwithcontainer
from model_policy_tenantwithcontainer import (
TenantWithContainerPolicy,
LeastLoadedNodeScheduler,
)
from mock_modelaccessor import MockObjectList
# import all class names to globals
for (
k,
v,
) in model_policy_tenantwithcontainer.model_accessor.all_model_classes.items(
):
globals()[k] = v
# TODO: Mock_model_accessor lacks save or delete methods
# Instance.save = mock.Mock
# Instance.delete = mock.Mock
# TenantWithContainer.save = mock.Mock
self.policy = TenantWithContainerPolicy()
self.user = User(email="*****@*****.**")
self.tenant = TenantWithContainer(creator=self.user)
self.flavor = Flavor(name="m1.small")
def spawn_instance(self,
name,
key_name=None,
availability_zone=None,
hostname=None,
image_id=None,
security_group=None,
pubkeys=[],
nics=None,
metadata=None,
userdata=None,
flavor_name=None):
if not flavor_name:
flavor_name = Config.get("nova.default_flavor")
flavor = self.shell.nova.flavors.find(name=flavor_name)
if not security_group:
security_group = Config.get("nova.default_security_group")
files = {}
#if pubkeys:
# files["/root/.ssh/authorized_keys"] = "\n".join(pubkeys).encode('base64')
hints = {}
# determine availability zone and compute host
availability_zone_filter = None
if availability_zone is None or not availability_zone:
availability_zone_filter = 'nova'
else:
availability_zone_filter = availability_zone
if hostname:
availability_zone_filter += ':%s' % hostname
server = self.shell.nova.servers.create(
name=name,
key_name=key_name,
flavor=flavor.id,
image=image_id,
security_group=security_group,
#files = files,
scheduler_hints=hints,
availability_zone=availability_zone_filter,
nics=nics,
networks=nics,
meta=metadata,
userdata=userdata)
return server
def setUp(self):
global DeferredException
self.sys_path_save = sys.path
# Setting up the config module
from xosconfig import Config
config = os.path.join(test_path, "../test_config.yaml")
Config.clear()
Config.init(config, "synchronizer-config-schema.yaml")
# END Setting up the config module
from xossynchronizer.mock_modelaccessor_build import mock_modelaccessor_config
mock_modelaccessor_config(test_path, [("fabric", "fabric.xproto")])
import xossynchronizer.modelaccessor
import mock_modelaccessor
reload(
mock_modelaccessor) # in case nose2 loaded it in a previous test
reload(xossynchronizer.modelaccessor
) # in case nose2 loaded it in a previous test
from xossynchronizer.modelaccessor import model_accessor
self.model_accessor = model_accessor
from sync_fabric_port import SyncFabricPort
# import all class names to globals
for (k, v) in model_accessor.all_model_classes.items():
globals()[k] = v
self.sync_step = SyncFabricPort
self.sync_step.log = Mock()
# mock onos-fabric
onos_fabric = Mock()
onos_fabric.name = "onos-fabric"
onos_fabric.rest_hostname = "onos-fabric"
onos_fabric.rest_port = "8181"
onos_fabric.rest_username = "******"
onos_fabric.rest_password = "******"
onos_fabric_base = Mock()
onos_fabric_base.leaf_model = onos_fabric
self.fabric = Mock()
self.fabric.name = "fabric"
self.fabric.provider_services = [onos_fabric_base]
def setUp(self):
self.sys_path_save = sys.path
# Setting up the config module
from xosconfig import Config
config = os.path.join(test_path, "../test_config.yaml")
Config.clear()
Config.init(config, "synchronizer-config-schema.yaml")
from multistructlog import create_logger
log = create_logger(Config().get('logging'))
# END Setting up the config module
from xossynchronizer.mock_modelaccessor_build import mock_modelaccessor_config
mock_modelaccessor_config(
test_path, [("att-workflow-driver", "att-workflow-driver.xproto"),
("olt-service", "volt.xproto"),
("rcord", "rcord.xproto")])
import xossynchronizer.modelaccessor
import mock_modelaccessor
reload(
mock_modelaccessor) # in case nose2 loaded it in a previous test
reload(xossynchronizer.modelaccessor
) # in case nose2 loaded it in a previous test
from xossynchronizer.modelaccessor import model_accessor
from auth_event import SubscriberAuthEventStep
# import all class names to globals
for (k, v) in model_accessor.all_model_classes.items():
globals()[k] = v
self.model_accessor = model_accessor
self.log = log
self.event_step = SubscriberAuthEventStep(
model_accessor=self.model_accessor, log=self.log)
self.event = Mock()
self.volt = Mock()
self.volt.name = "vOLT"
self.volt.leaf_model = Mock()
self.hippie_si = AttWorkflowDriverServiceInstance()
self.hippie_si.serial_number = "BRCM1234"
self.hippie_si.save = Mock()
def load_dependency_graph(self):
try:
if Config.get("dependency_graph"):
self.log.debug(
"Loading model dependency graph",
path=Config.get("dependency_graph"),
)
dep_graph_str = open(Config.get("dependency_graph")).read()
else:
self.log.debug("Using default model dependency graph",
graph={})
dep_graph_str = "{}"
# joint_dependencies is of the form { Model1 -> [(Model2, src_port, dst_port), ...] }
# src_port is the field that accesses Model2 from Model1
# dst_port is the field that accesses Model1 from Model2
static_dependencies = json.loads(dep_graph_str)
dynamic_dependencies = [
] # Dropped Service and ServiceInstance dynamic dependencies
joint_dependencies = dict(static_dependencies.items() +
dynamic_dependencies)
model_dependency_graph = DiGraph()
for src_model, deps in joint_dependencies.items():
for dep in deps:
dst_model, src_accessor, dst_accessor = dep
if src_model != dst_model:
edge_label = {
"src_accessor": src_accessor,
"dst_accessor": dst_accessor,
}
model_dependency_graph.add_edge(
src_model, dst_model, edge_label)
model_dependency_graph_rev = model_dependency_graph.reverse(
copy=True)
self.model_dependency_graph = {
# deletion
True: model_dependency_graph_rev,
False: model_dependency_graph,
}
self.log.debug("Loaded dependencies",
edges=model_dependency_graph.edges())
except Exception as e:
self.log.exception("Error loading dependency graph", e=e)
raise e
def create_secure_client(self, username, password, arg):
"""
This method will check if this combination of username/password already
has stored orm classes in RESOURCES, otherwise create them
"""
deferred = defer.Deferred()
key = "%s~%s" % (username, password)
if key in RESOURCES:
reactor.callLater(0, deferred.callback, arg)
else:
local_cert = Config.get("local_cert")
client = SecureClient(
endpoint=self.grpc_secure_endpoint,
username=username,
password=password,
cacert=local_cert,
)
client.restart_on_disconnect = True
# SecureClient is preceeded by an insecure client, so treat all secure clients as previously connected
# See CORD-3152
client.was_connected = True
client.set_reconnect_callback(
functools.partial(self.setup_resources, client, key, deferred,
arg))
client.start()
return deferred
def test_get_cli_param(self):
"""
[XOS-Config] Should read CLI -C param
"""
args = ["-A", "Foo", "-c", "Bar", "-C", "config.yaml"]
res = Config.get_cli_param(args)
self.assertEqual(res, "config.yaml")
def setUp(self):
from xosconfig import Config
test_path = os.path.abspath(os.path.dirname(
os.path.realpath(__file__)))
config = os.path.join(test_path, "test_config.yaml")
Config.clear()
Config.init(config, "synchronizer-config-schema.yaml")
if USE_FAKE_STUB:
sys.path.append(PARENT_DIR)
# Import these after config, in case they depend on config
from xosapi.orm import ORMQuerySet, ORMLocalObjectManager
self.ORMQuerySet = ORMQuerySet
self.ORMLocalObjectManager = ORMLocalObjectManager
def unload_models(client, reactor, version):
# This function is called by a timer until it succeeds.
log.info("unload_models initiated by timer")
try:
result = ModelLoadClient(client).unload_models(
Config.get("name"),
version=version,
cleanup_behavior=ModelLoadClient.AUTOMATICALLY_CLEAN)
log.debug("Unload response", result=result)
if result.status in [result.SUCCESS, result.SUCCESS_NOTHING_CHANGED]:
log.info("Models successfully unloaded. Exiting with status",
code=0)
sys.exit(0)
if result.status == result.TRYAGAIN:
log.info("TRYAGAIN received. Expect to try again in 30 seconds.")
except Exception as e:
# If the synchronizer is operational, then assume the ORM's restart_on_disconnect will deal with the
# connection being lost.
log.exception(
"Error while unloading. Expect to try again in 30 seconds.")
Timer(30, functools.partial(unload_models, client, reactor,
version)).start()
def setUp(self):
b = backend.Backend()
steps_dir = Config.get("steps_dir")
self.steps = b.load_sync_step_modules(steps_dir)
self.synchronizer = event_loop.XOSObserver(self.steps)
os.remove('/tmp/sync_ports')
os.remove('/tmp/delete_ports')
def main():
input_fn = sys.argv[1]
result_fn = sys.argv[2]
args = pickle.loads(open(input_fn).read())
Config.init(args['config_file'], 'synchronizer-config-schema.yaml')
ansible_hosts = args["ansible_hosts"]
ansible_config = args["ansible_config"]
fqp = args["fqp"]
opts = args["opts"]
result = run_playbook(ansible_hosts, ansible_config, fqp, opts)
open(result_fn, "w").write(pickle.dumps(result))
def setUp(self):
self.sys_path_save = sys.path
# Setting up the config module
from xosconfig import Config
config = os.path.join(test_path, "../test_config.yaml")
Config.clear()
Config.init(config, "synchronizer-config-schema.yaml")
# END Setting up the config module
from xossynchronizer.mock_modelaccessor_build import build_mock_modelaccessor
# Can't use mock_modelaccessor_config because we're not in the xos-services directory, so do it
# the long way...
xos_dir = os.path.join(test_path, "../../../../xos")
services_dir = os.path.join(test_path, "../../../..")
service_xprotos = [os.path.join(test_path, "../models/testservice.xproto")]
build_mock_modelaccessor(None, xos_dir, services_dir, service_xprotos)
import xossynchronizer.modelaccessor
import mock_modelaccessor
imp.reload(mock_modelaccessor) # in case nose2 loaded it in a previous test
imp.reload(xossynchronizer.modelaccessor) # in case nose2 loaded it in a previous test
from sync_testservice_serviceinstance import SyncTestserviceServiceInstance
from xossynchronizer.modelaccessor import model_accessor
self.model_accessor = model_accessor
# import all class names to globals
for (k, v) in model_accessor.all_model_classes.items():
globals()[k] = v
self.sync_step = SyncTestserviceServiceInstance
# TODO: Use modelaccessor instead
# create a mock instance instance
self.model = Mock(name="Example",
some_integer=0,
sync_after_policy=False,
sync_during_policy=False,
policy_after_sync=False,
policy_during_sync=False,
update_during_sync=False,
update_during_policy=False,
create_duplicate=False)
def setUp(self):
self.sys_path_save = sys.path
# Setting up the config module
from xosconfig import Config
config = os.path.join(test_path, "../test_config.yaml")
Config.clear()
Config.init(config, "synchronizer-config-schema.yaml")
# END Setting up the config module
from xossynchronizer.mock_modelaccessor_build import mock_modelaccessor_config
mock_modelaccessor_config(
test_path, [("dt-workflow-driver", "dt-workflow-driver.xproto"),
("olt-service", "volt.xproto"),
("rcord", "rcord.xproto")])
import xossynchronizer.modelaccessor
import mock_modelaccessor
reload(
mock_modelaccessor) # in case nose2 loaded it in a previous test
reload(xossynchronizer.modelaccessor
) # in case nose2 loaded it in a previous test
from xossynchronizer.modelaccessor import model_accessor
from onu_event import ONUEventStep
# import all class names to globals
for (k, v) in model_accessor.all_model_classes.items():
globals()[k] = v
self.model_accessor = model_accessor
self.log = Mock()
self.event_step = ONUEventStep(model_accessor=self.model_accessor,
log=self.log)
self.event = Mock()
self.event_dict = {
'status': 'activated',
'serialNumber': 'BRCM1234',
'deviceId': 'of:109299321',
'portNumber': '16'
}
self.event.value = json.dumps(self.event_dict)
self.pppoe = DtWorkflowDriverService(name="dt-workflow-driver")
def run_playbook(ansible_hosts, ansible_config, fqp, opts):
args = {
"ansible_hosts": ansible_hosts,
"ansible_config": ansible_config,
"fqp": fqp,
"opts": opts,
"config_file": Config.get_config_file()
}
keep_temp_files = Config.get("keep_temp_files")
dir = tempfile.mkdtemp()
args_fn = None
result_fn = None
try:
logger.info("creating args file in %s" % dir)
args_fn = os.path.join(dir, "args")
result_fn = os.path.join(dir, "result")
open(args_fn, "w").write(pickle.dumps(args))
ansible_main_fn = os.path.join(os.path.dirname(__file__),
"ansible_main.py")
os.system("python %s %s %s" % (ansible_main_fn, args_fn, result_fn))
result = pickle.loads(open(result_fn).read())
if hasattr(result, "exception"):
logger.log_error("Exception in playbook: %s" % result["exception"])
stats = result.get("stats", None)
aresults = result.get("aresults", None)
except:
logger.log_exc("Exception running ansible_main")
stats = None
aresults = None
finally:
if not keep_temp_files:
if args_fn and os.path.exists(args_fn):
os.remove(args_fn)
if result_fn and os.path.exists(result_fn):
os.remove(result_fn)
os.rmdir(dir)
return (stats, aresults)
def setUp(self):
global DeferredException
self.sys_path_save = sys.path
sys.path.append(xos_dir)
sys.path.append(os.path.join(xos_dir, 'synchronizers', 'new_base'))
# Setting up the config module
from xosconfig import Config
config = os.path.join(test_path, "../test_config.yaml")
Config.clear()
Config.init(config, "synchronizer-config-schema.yaml")
# END Setting up the config module
from synchronizers.new_base.mock_modelaccessor_build import build_mock_modelaccessor
build_mock_modelaccessor(xos_dir, services_dir,
[get_models_fn("fabric", "fabric.xproto")])
import synchronizers.new_base.modelaccessor
from sync_fabric_switch import SyncFabricSwitch, model_accessor
# import all class names to globals
for (k, v) in model_accessor.all_model_classes.items():
globals()[k] = v
self.sync_step = SyncFabricSwitch
self.sync_step.log = Mock()
# mock onos-fabric
onos_fabric = Mock()
onos_fabric.name = "onos-fabric"
onos_fabric.rest_hostname = "onos-fabric"
onos_fabric.rest_port = "8181"
onos_fabric.rest_username = "******"
onos_fabric.rest_password = "******"
onos_fabric_base = Mock()
onos_fabric_base.leaf_model = onos_fabric
self.fabric = Mock()
self.fabric.name = "fabric"
self.fabric.provider_services = [onos_fabric_base]
# create a mock Switch instance
self.o = Mock()
self.o.name = "MockSwitch"
self.o.ofId = "of:1234"
def run(self):
observer_thread = None
watcher_thread = None
model_policy_thread = None
model_accessor.update_diag(sync_start=time.time(), backend_status="0 - Synchronizer Start")
steps_dir = Config.get("steps_dir")
if steps_dir:
sync_steps = self.load_sync_step_modules(steps_dir)
if sync_steps:
# start the observer
observer = XOSObserver(sync_steps)
observer_thread = threading.Thread(target=observer.run,name='synchronizer')
observer_thread.start()
# start the watcher thread
if (watchers_enabled):
watcher = XOSWatcher(sync_steps)
watcher_thread = threading.Thread(target=watcher.run,name='watcher')
watcher_thread.start()
else:
logger.info("Skipping observer and watcher threads due to no steps dir.")
# start model policies thread
policies_dir = Config.get("model_policies_dir")
if policies_dir:
policy_engine = XOSPolicyEngine(policies_dir=policies_dir)
model_policy_thread = threading.Thread(target=policy_engine.run, name="policy_engine")
model_policy_thread.start()
else:
logger.info("Skipping model policies thread due to no model_policies dir.")
while True:
try:
time.sleep(1000)
except KeyboardInterrupt:
print "exiting due to keyboard interrupt"
# TODO: See about setting the threads as daemons
if observer_thread:
observer_thread._Thread__stop()
if watcher_thread:
watcher_thread._Thread__stop()
if model_policy_thread:
model_policy_thread._Thread__stop()
sys.exit(1)
def setUp(self):
self.sys_path_save = sys.path
self.cwd_save = os.getcwd()
config = os.path.join(test_path, "test_config.yaml")
from xosconfig import Config
Config.clear()
Config.init(config, "synchronizer-config-schema.yaml")
from xossynchronizer import loadmodels
from xossynchronizer.loadmodels import ModelLoadClient
self.loadmodels = loadmodels
self.api = MagicMock()
self.api.dynamicload_pb2.LoadModelsRequest = MockLoadModelsRequest
self.loader = ModelLoadClient(self.api)
def setUp(self):
self.sys_path_save = sys.path
sys.path.append(xos_dir)
sys.path.append(os.path.join(xos_dir, 'synchronizers', 'new_base'))
# Setting up the config module
from xosconfig import Config
config = os.path.join(test_path, "../test_config.yaml")
Config.clear()
Config.init(config, "synchronizer-config-schema.yaml")
from multistructlog import create_logger
log = create_logger(Config().get('logging'))
# END Setting up the config module
from synchronizers.new_base.mock_modelaccessor_build import build_mock_modelaccessor
# build_mock_modelaccessor(xos_dir, services_dir, [get_models_fn("olt-service", "volt.xproto")])
build_mock_modelaccessor(xos_dir, services_dir, [
get_models_fn("hippie-oss", "hippie-oss.xproto"),
get_models_fn("olt-service", "volt.xproto"),
get_models_fn("rcord", "rcord.xproto")
])
import synchronizers.new_base.modelaccessor
from dhcp_event import SubscriberDhcpEventStep, model_accessor
# import all class names to globals
for (k, v) in model_accessor.all_model_classes.items():
globals()[k] = v
self.log = log
self.event_step = SubscriberDhcpEventStep(self.log)
self.event = Mock()
self.volt = Mock()
self.volt.name = "vOLT"
self.volt.leaf_model = Mock()
self.subscriber = RCORDSubscriber()
self.subscriber.onu_device = "BRCM1234"
self.subscriber.save = Mock()
self.mac_address = "aa:bb:cc:dd:ee"
self.ip_address = "192.168.3.5"
def test_get_service_endpoint(self):
"""
[XOS-Config] Should query registrator and return service endpoint
"""
with patch("xosconfig.config.requests.get") as mock_get:
mock_get.return_value.json.return_value = db_service
endpoint = Config.get_service_endpoint("xos-db")
self.assertEqual(endpoint, "http://172.18.0.4:5432")
def setUp(self):
self.mock_etcd = Mock(name="etcd-client")
etcd = Mock(name="etcd-mocked-lib")
etcd.client.return_value = self.mock_etcd
modules = {
'etcd3': etcd
}
self.module_patcher = patch.dict('sys.modules', modules)
self.module_patcher.start()
self.sys_path_save = sys.path
# Setting up the config module
from xosconfig import Config
config = os.path.join(test_path, "../test_config.yaml")
Config.clear()
Config.init(config, "synchronizer-config-schema.yaml")
# END Setting up the config module
from xossynchronizer.mock_modelaccessor_build import mock_modelaccessor_config
mock_modelaccessor_config(test_path, [("olt-service", "volt.xproto"),
("rcord", "rcord.xproto")])
import xossynchronizer.modelaccessor
reload(xossynchronizer.modelaccessor) # in case nose2 loaded it in a previous test
from xossynchronizer.modelaccessor import model_accessor
self.model_accessor = model_accessor
from xossynchronizer.modelaccessor import model_accessor
from sync_tech_profile import SyncTechnologyProfile
# import all class names to globals
for (k, v) in model_accessor.all_model_classes.items():
globals()[k] = v
self.sync_step = SyncTechnologyProfile
self.o = Mock()
self.o.technology = "test_technology"
self.o.profile_id = 64
self.o.profile_value = '{"test":"profile"}'
self.o.tologdict.return_value = {'name': "mock-tp"}
def run(self):
# This is our unique client id, to be used when firing and receiving events
# It needs to be generated once and placed in the config file
user = Config.get("feefie.client_user")
try:
clid = Config.get("feefie.client_id")
except:
clid = get_random_client_id()
print "EventListener: no feefie_client_id configured. Using random id %s" % clid
if fofum_enabled:
f = Fofum(user=user)
listener_thread = threading.Thread(target=f.listen_for_event,
args=(clid, self.handle_event))
listener_thread.start()
def create_kafka_consumer(self):
# use the service name as the group id
consumer_config = {
"group.id": Config().get("name"),
"bootstrap.servers": ",".join(self.bootstrap_servers),
"default.topic.config": {"auto.offset.reset": "smallest"},
}
return confluent_kafka.Consumer(**consumer_config)
def __init__(self, sync_steps):
# The Condition object that gets signalled by Feefie events
self.step_lookup = {}
self.sync_steps = sync_steps
self.load_sync_steps()
self.event_cond = threading.Condition()
self.driver = DRIVER
self.observer_name = Config.get("name")
def config_accessor():
accessor_kind = Config.get("accessor.kind")
if accessor_kind == "testframework":
pass
elif accessor_kind == "grpcapi":
config_accessor_grpcapi()
else:
raise Exception("Unknown accessor kind %s" % accessor_kind)
def extract_context(self, cur):
try:
observer_name = Config.get("name")
cur['synchronizer_name'] = observer_name
except:
pass
self.sanitize_extra_args(cur)
return cur
