def setUp(self): self.sys_path_save = sys.path # Setting up the config module from xosconfig import Config config = os.path.join(test_path, "test_config.yaml") Config.clear() Config.init(config, "synchronizer-config-schema.yaml") # END Setting up the config module from xossynchronizer.mock_modelaccessor_build import build_mock_modelaccessor # FIXME this is to get jenkins to pass the tests, somehow it is running tests in a different order # and apparently it is not overriding the generated model accessor build_mock_modelaccessor(sync_lib_dir, xos_dir, services_dir, []) import xossynchronizer.modelaccessor # import all class names to globals for ( k, v, ) in xossynchronizer.modelaccessor.model_accessor.all_model_classes.items(): globals()[k] = v self.log = Mock()
def setUp(self): global ComputeNodePolicy, MockObjectList self.sys_path_save = sys.path config = os.path.join(test_path, "../test_config.yaml") from xosconfig import Config Config.clear() Config.init(config, 'synchronizer-config-schema.yaml') from xossynchronizer.mock_modelaccessor_build import mock_modelaccessor_config mock_modelaccessor_config(test_path, [("fabric", "fabric.xproto")]) import xossynchronizer.modelaccessor import mock_modelaccessor imp.reload(mock_modelaccessor) # in case nose2 loaded it in a previous test imp.reload(xossynchronizer.modelaccessor) # in case nose2 loaded it in a previous test from model_policy_compute_nodes import ComputeNodePolicy, model_accessor self.model_accessor = model_accessor from mock_modelaccessor import MockObjectList # import all class names to globals for (k, v) in model_accessor.all_model_classes.items(): globals()[k] = v # Some of the functions we call have side-effects. For example, creating a VSGServiceInstance may lead to # creation of tags. Ideally, this wouldn't happen, but it does. So make sure we reset the world. model_accessor.reset_all_object_stores() self.policy = ComputeNodePolicy self.model = Mock()
def test_get_default_val_for_missing_param(self): """ [XOS-Config] Should get the default value if nothing is specified """ Config.init(basic_conf) dir = Config.get("xos_dir") self.assertEqual(dir, "/opt/xos")
def test_get_child_level(self): """ [XOS-Config] Should return a child level param """ Config.init(sample_conf) res = Config.get("database.name") self.assertEqual(res, "xos")
def _test_get_child_level(self): """ [XOS-Config] Should return a child level param """ Config.init(sample_conf) res = Config.get("nested.parameter.for") self.assertEqual(res, "testing")
def init(): global log global kafka_producer if not log: log = create_logger(Config().get("logging")) if kafka_producer: raise Exception("XOSKafkaProducer already initialized") else: log.info( "Connecting to Kafka with bootstrap servers: %s" % Config.get("kafka_bootstrap_servers") ) try: producer_config = { "bootstrap.servers": ",".join(Config.get("kafka_bootstrap_servers")) } kafka_producer = confluent_kafka.Producer(**producer_config) log.info("Connected to Kafka: %s" % kafka_producer) except confluent_kafka.KafkaError as e: log.exception("Kafka Error: %s" % e)
def test_missing_file_exception(self): """ [XOS-Config] Raise if file not found """ with self.assertRaises(Exception) as e: Config.init("missing_conf") self.assertEqual(e.exception.message, "[XOS-Config] Config file not found at: missing_conf")
def config_accessor_grpcapi(): global orig_sigint grpcapi_endpoint = Config.get("accessor.endpoint") grpcapi_username = Config.get("accessor.username") grpcapi_password = Config.get("accessor.password") # if password starts with "@", then retreive the password from a file if grpcapi_password.startswith("@"): fn = grpcapi_password[1:] if not os.path.exists(fn): raise Exception("%s does not exist" % fn) grpcapi_password = open(fn).readline().strip() from xosapi.xos_grpc_client import SecureClient from twisted.internet import reactor grpcapi_client = SecureClient(endpoint=grpcapi_endpoint, username=grpcapi_username, password=grpcapi_password) grpcapi_client.set_reconnect_callback(functools.partial(grpcapi_reconnect, grpcapi_client, reactor)) grpcapi_client.start() # Start reactor. This will cause the client to connect and then execute # grpcapi_callback(). # Reactor will take over SIGINT during reactor.run(), but does not return it when reactor.stop() is called. orig_sigint = signal.getsignal(signal.SIGINT) # Start reactor. This will cause the client to connect and then execute # grpcapi_callback(). reactor.run()
def test_config_not_initialized(self): """ [XOS-Config] Raise if accessing properties without initialization """ with self.assertRaises(Exception) as e: Config.get("database") self.assertEqual(e.exception.message, "[XOS-Config] Module has not been initialized")
def test_get_config_file(self): """ [XOS-Config] Should return the config file in use """ Config.init(sample_conf) res = Config.get_config_file() self.assertEqual(res, sample_conf)
def setUp(self): self.sys_path_save = sys.path self.cwd_save = os.getcwd() config = os.path.join(test_path, "test_config.yaml") from xosconfig import Config Config.clear() Config.init(config, "synchronizer-config-schema.yaml") from xossynchronizer.mock_modelaccessor_build import ( build_mock_modelaccessor, ) build_mock_modelaccessor(sync_lib_dir, xos_dir, services_dir=None, service_xprotos=[]) # The test config.yaml references files in `xos-synchronizer-tests/` so make sure we're in the parent # directory of the test directory. os.chdir(os.path.join(test_path, "..")) import xossynchronizer.event_loop reload(xossynchronizer.event_loop) import xossynchronizer.backend reload(xossynchronizer.backend) from xossynchronizer.modelaccessor import model_accessor b = xossynchronizer.backend.Backend(model_accessor=model_accessor) steps_dir = Config.get("steps_dir") self.steps = b.load_sync_step_modules(steps_dir) self.synchronizer = xossynchronizer.event_loop.XOSObserver(self.steps, model_accessor)
def run_playbook(ansible_hosts, ansible_config, fqp, opts): args = {"ansible_hosts": ansible_hosts, "ansible_config": ansible_config, "fqp": fqp, "opts": opts, "config_file": Config.get_config_file()} keep_temp_files = Config.get("keep_temp_files") dir = tempfile.mkdtemp() args_fn = None result_fn = None try: log.info("creating args file",dir = dir) args_fn = os.path.join(dir, "args") result_fn = os.path.join(dir, "result") open(args_fn, "w").write(pickle.dumps(args)) ansible_main_fn = os.path.join(os.path.dirname(__file__), "ansible_main.py") os.system("python %s %s %s" % (ansible_main_fn, args_fn, result_fn)) result = pickle.loads(open(result_fn).read()) if hasattr(result, "exception"): log.error("Exception in playbook",exception = result["exception"]) stats = result.get("stats", None) aresults = result.get("aresults", None) except Exception,e: log.exception("Exception running ansible_main") stats = None aresults = None
def test_get_missing_param(self): """ [XOS-Config] Should return None reading a missing param """ Config.init(sample_conf) res = Config.get("foo") self.assertEqual(res, None)
def test_yaml_not_valid(self): """ [XOS-Config] Raise if yaml is not valid """ with self.assertRaises(Exception) as e: Config.init(yaml_not_valid) self.assertEqual(e.exception.message, "[XOS-Config] The config format is wrong: Unable to load any data from source yaml file")
def test_invalid_format(self): """ [XOS-Config] Raise if format is not valid (we expect a dictionary) """ with self.assertRaises(Exception) as e: Config.init(invalid_format) self.assertEqual(e.exception.message, "[XOS-Config] The config format is wrong: Schema validation failed:\n - Value '['I am', 'a yaml', 'but the', 'format is not', 'correct']' is not a dict. Value path: ''.")
def test_initialize_only_once(self): """ [XOS-Config] Raise if initialized twice """ with self.assertRaises(Exception) as e: Config.init(sample_conf) Config2.init(sample_conf) self.assertEqual(str(e.exception), "[XOS-Config] Module already initialized")
def test_missing_get_service_info(self): """ [XOS-Config] Should query registrator and return an exception if service is not there """ with patch("xosconfig.config.requests.get") as mock_get: mock_get.return_value.json.return_value = [] with self.assertRaises(Exception) as e: Config.get_service_info("missing-service") self.assertEqual(e.exception.message, "[XOS-Config] The service missing-service looking for does not exist")
def test_yaml_not_valid(self): """ [XOS-Config] Raise if yaml is not valid """ with self.assertRaises(Exception) as e: Config.init(yaml_not_valid) self.assertTrue( str(e.exception).startswith("[XOS-Config] The config format is wrong:") )
def test_get_first_level(self): """ [XOS-Config] Should return a first level param """ Config.init(sample_conf) # NOTE we are using Config2 here to be sure that the configuration is readable from any import, # not only from the one that has been used to initialize it res = Config2.get("database") self.assertEqual(res, {"name": "xos", "username": "******", "password": "******"})
def test_schema_override_usage(self): """ [XOS-Config] the XOS_CONFIG_SCHEMA should be used to validate a config """ os.environ["XOS_CONFIG_SCHEMA"] = small_schema with self.assertRaises(Exception) as e: Config.init(basic_conf) self.assertEqual(e.exception.message, "[XOS-Config] The config format is wrong: Schema validation failed:\n - Key 'database' was not defined. Path: ''.") del os.environ["XOS_CONFIG_SCHEMA"]
def test_fail_get_service_info(self): """ [XOS-Config] Should query registrator and return an exception if it"s down """ with patch("xosconfig.config.requests.get") as mock_get: mock_get.return_value.ok = False with self.assertRaises(Exception) as e: Config.get_service_info("missing-service") self.assertEqual(e.exception.message, "[XOS-Config] Registrator is down")
def test_config_override(self): """ [XOS-Config] If an override is provided for the config, it should return the overridden value """ Config.init(sample_conf, "xos-config-schema.yaml", override_conf) res = Config.get("logging.level") self.assertEqual(res, "info") res = Config.get("database.password") self.assertEqual(res, "overridden_password")
def test_env_override(self): """ [XOS-Config] the XOS_CONFIG_FILE environment variable should override the config_file """ os.environ["XOS_CONFIG_FILE"] = "env.yaml" with self.assertRaises(Exception) as e: Config.init("missing_conf") self.assertEqual(e.exception.message, "[XOS-Config] Config file not found at: env.yaml") del os.environ["XOS_CONFIG_FILE"]
def setUp(self): from xosconfig import Config test_path = os.path.abspath(os.path.dirname(os.path.realpath(__file__))) config = os.path.join(test_path, "test_config.yaml") Config.clear() Config.init(config, "synchronizer-config-schema.yaml") if USE_FAKE_STUB: sys.path.append(PARENT_DIR)
def configure_logging(verbose): global log # INITIALIZING LOGGER Config.init() cfg = Config().get("logging") if verbose: cfg["handlers"]["console"]["level"] = "DEBUG" log = create_logger(cfg)
def test_schema_override(self): """ [XOS-Config] the XOS_CONFIG_SCHEMA environment variable should override the config_schema """ os.environ["XOS_CONFIG_SCHEMA"] = "env-schema.yaml" with self.assertRaises(Exception) as e: Config.init(basic_conf) self.assertRegexpMatches(e.exception.message, '\[XOS\-Config\] Config schema not found at: (.+)env-schema\.yaml') # self.assertEqual(e.exception.message, "[XOS-Config] Config schema not found at: env-schema.yaml") del os.environ["XOS_CONFIG_SCHEMA"]
def test_config_extend(self): """ [XOS-Config] If an override is provided for the config, it should return the overridden value (also if not defined in the base one) """ Config.init(sample_conf, "xos-config-schema.yaml", extend_conf) res = Config.get("xos_dir") self.assertEqual(res, "/opt/xos") res = Config.get("database.password") self.assertEqual(res, "safe")
def setUp(self): self.sys_path_save = sys.path # Setting up the config module from xosconfig import Config config = os.path.join(test_path, "../test_config.yaml") Config.clear() Config.init(config, "synchronizer-config-schema.yaml") # END Setting up the config module from xossynchronizer.mock_modelaccessor_build import mock_modelaccessor_config mock_modelaccessor_config(test_path, [("vrouter", "vrouter.xproto")]) import xossynchronizer.modelaccessor import mock_modelaccessor reload(mock_modelaccessor) # in case nose2 loaded it in a previous test reload(xossynchronizer.modelaccessor) # in case nose2 loaded it in a previous test from sync_routes import SyncRoutes, model_accessor self.model_accessor = model_accessor # import all class names to globals for (k, v) in model_accessor.all_model_classes.items(): globals()[k] = v self.sync_step = SyncRoutes self.sync_step.log = Mock() # mock onos-fabric onos_fabric = Mock() onos_fabric.name = "onos-fabric" onos_fabric.rest_hostname = "onos-fabric" onos_fabric.rest_port = "8181" onos_fabric.rest_username = "******" onos_fabric.rest_password = "******" onos_fabric_base = Mock() onos_fabric_base.leaf_model = onos_fabric self.fabric = Mock() self.fabric.name = "fabric" self.fabric.provider_services = [onos_fabric_base] self.vrouter = Mock() self.vrouter.name = "vrouter" self.vrouter.provider_services = [self.fabric] # create a mock VRouterStaticRoute instance self.o = Mock() self.o.id = 1 self.o.vrouter.owner = self.vrouter self.o.tologdict.return_value = {}
def test_get_default_val_for_missing_param(self): """ [XOS-Config] Should get the default value if nothing is specified """ Config.init(basic_conf) log = Config.get("logging") self.assertEqual(log, { "level": "info", "channels": ["file", "console"], "logstash_hostport": "cordloghost:5617", "file": "/var/log/xos.log", })
def main(): base_config_file = os.path.abspath(os.path.dirname( os.path.realpath(__file__)) + '/config.yaml') mounted_config_file = os.path.abspath(os.path.dirname( os.path.realpath(__file__)) + '/mounted_config.yaml') if os.path.isfile(mounted_config_file): Config.init(base_config_file, 'synchronizer-config-schema.yaml', mounted_config_file) else: Config.init(base_config_file, 'synchronizer-config-schema.yaml') Synchronizer().run()
def setUp(self): global DeferredException self.sys_path_save = sys.path # Setting up the config module from xosconfig import Config config = os.path.join(test_path, "../test_fabric_crossconnect_config.yaml") Config.clear() Config.init(config, "synchronizer-config-schema.yaml") # END Setting up the config module from xossynchronizer.mock_modelaccessor_build import mock_modelaccessor_config mock_modelaccessor_config(test_path, [ ("fabric-crossconnect", "fabric-crossconnect.xproto"), ]) import xossynchronizer.modelaccessor import mock_modelaccessor reload( mock_modelaccessor) # in case nose2 loaded it in a previous test reload(xossynchronizer.modelaccessor ) # in case nose2 loaded it in a previous test from xossynchronizer.modelaccessor import model_accessor self.model_accessor = model_accessor from sync_fabric_crossconnect_service_instance import SyncFabricCrossconnectServiceInstance, model_accessor, \ DeferredException from helpers import Helpers self.helpers = Helpers # import all class names to globals for (k, v) in model_accessor.all_model_classes.items(): globals()[k] = v self.sync_step = SyncFabricCrossconnectServiceInstance self.sync_step.log = Mock() # mock onos-fabric self.onos_fabric = Service(name="onos-fabric", rest_hostname="onos-fabric", rest_port="8181", rest_username="******", rest_password="******") self.service = FabricCrossconnectService( name="fcservice", provider_services=[self.onos_fabric])
def setUpClass(cls): global log config = os.path.join(test_path, "test_config.yaml") from xosconfig import Config Config.clear() Config.init(config, "synchronizer-config-schema.yaml") if not log: from multistructlog import create_logger log = create_logger(Config().get("logging"))
def config_accessor(): accessor_kind = Config.get("accessor.kind") if accessor_kind == "testframework": config_accessor_mock() elif accessor_kind == "grpcapi": config_accessor_grpcapi() else: raise Exception("Unknown accessor kind %s" % accessor_kind) # now import any wrappers that the synchronizer needs to add to the ORM if Config.get("wrappers"): for wrapper_name in Config.get("wrappers"): importlib.import_module(wrapper_name)
def setUp(self): config = os.path.abspath( os.path.dirname(os.path.realpath(__file__)) + "/test_config.yaml") Config.clear() # in case left unclean by a previous test case Config.init(config) import backupprocessor self.backupprocessor = backupprocessor self.setUpPyfakefs() self.processor = backupprocessor.BackupProcessor() self.mock_backuphandler = MagicMock(backup=MagicMock(), restore=MagicMock())
def setUp(self): self.sys_path_save = sys.path self.cwd_save = os.getcwd() sys.path.append(xos_dir) sys.path.append(os.path.join(xos_dir, "synchronizers", "new_base")) sys.path.append( os.path.join(xos_dir, "synchronizers", "new_base", "tests", "steps")) config = os.path.join(test_path, "test_config.yaml") from xosconfig import Config Config.clear() Config.init(config, "synchronizer-config-schema.yaml") from synchronizers.new_base.mock_modelaccessor_build import ( build_mock_modelaccessor, ) build_mock_modelaccessor(xos_dir, services_dir=None, service_xprotos=[]) os.chdir(os.path.join(test_path, "..")) # config references tests/model-deps import event_loop reload(event_loop) import backend reload(backend) from modelaccessor import model_accessor # import all class names to globals for (k, v) in model_accessor.all_model_classes.items(): globals()[k] = v b = backend.Backend() steps_dir = Config.get("steps_dir") self.steps = b.load_sync_step_modules(steps_dir) self.synchronizer = event_loop.XOSObserver(self.steps) try: os.remove("/tmp/sync_ports") except OSError: pass try: os.remove("/tmp/delete_ports") except OSError: pass
def setUp(self): global TenantWithContainerPolicy, LeastLoadedNodeScheduler, MockObjectList self.sys_path_save = sys.path self.cwd_save = os.getcwd() sys.path.append(xos_dir) sys.path.append(os.path.join(xos_dir, "synchronizers", "new_base")) sys.path.append( os.path.join(xos_dir, "synchronizers", "new_base", "model_policies")) config = basic_conf = os.path.abspath( os.path.dirname(os.path.realpath(__file__)) + "/test_config.yaml") Config.clear() # in case left unclean by a previous test case Config.init(config, "synchronizer-config-schema.yaml") from synchronizers.new_base.mock_modelaccessor_build import ( build_mock_modelaccessor, ) build_mock_modelaccessor(xos_dir, services_dir=None, service_xprotos=[]) import model_policy_tenantwithcontainer from model_policy_tenantwithcontainer import ( TenantWithContainerPolicy, LeastLoadedNodeScheduler, ) from mock_modelaccessor import MockObjectList # import all class names to globals for ( k, v, ) in model_policy_tenantwithcontainer.model_accessor.all_model_classes.items( ): globals()[k] = v # TODO: Mock_model_accessor lacks save or delete methods # Instance.save = mock.Mock # Instance.delete = mock.Mock # TenantWithContainer.save = mock.Mock self.policy = TenantWithContainerPolicy() self.user = User(email="*****@*****.**") self.tenant = TenantWithContainer(creator=self.user) self.flavor = Flavor(name="m1.small")
def spawn_instance(self, name, key_name=None, availability_zone=None, hostname=None, image_id=None, security_group=None, pubkeys=[], nics=None, metadata=None, userdata=None, flavor_name=None): if not flavor_name: flavor_name = Config.get("nova.default_flavor") flavor = self.shell.nova.flavors.find(name=flavor_name) if not security_group: security_group = Config.get("nova.default_security_group") files = {} #if pubkeys: # files["/root/.ssh/authorized_keys"] = "\n".join(pubkeys).encode('base64') hints = {} # determine availability zone and compute host availability_zone_filter = None if availability_zone is None or not availability_zone: availability_zone_filter = 'nova' else: availability_zone_filter = availability_zone if hostname: availability_zone_filter += ':%s' % hostname server = self.shell.nova.servers.create( name=name, key_name=key_name, flavor=flavor.id, image=image_id, security_group=security_group, #files = files, scheduler_hints=hints, availability_zone=availability_zone_filter, nics=nics, networks=nics, meta=metadata, userdata=userdata) return server
def setUp(self): global DeferredException self.sys_path_save = sys.path # Setting up the config module from xosconfig import Config config = os.path.join(test_path, "../test_config.yaml") Config.clear() Config.init(config, "synchronizer-config-schema.yaml") # END Setting up the config module from xossynchronizer.mock_modelaccessor_build import mock_modelaccessor_config mock_modelaccessor_config(test_path, [("fabric", "fabric.xproto")]) import xossynchronizer.modelaccessor import mock_modelaccessor reload( mock_modelaccessor) # in case nose2 loaded it in a previous test reload(xossynchronizer.modelaccessor ) # in case nose2 loaded it in a previous test from xossynchronizer.modelaccessor import model_accessor self.model_accessor = model_accessor from sync_fabric_port import SyncFabricPort # import all class names to globals for (k, v) in model_accessor.all_model_classes.items(): globals()[k] = v self.sync_step = SyncFabricPort self.sync_step.log = Mock() # mock onos-fabric onos_fabric = Mock() onos_fabric.name = "onos-fabric" onos_fabric.rest_hostname = "onos-fabric" onos_fabric.rest_port = "8181" onos_fabric.rest_username = "******" onos_fabric.rest_password = "******" onos_fabric_base = Mock() onos_fabric_base.leaf_model = onos_fabric self.fabric = Mock() self.fabric.name = "fabric" self.fabric.provider_services = [onos_fabric_base]
def setUp(self): self.sys_path_save = sys.path # Setting up the config module from xosconfig import Config config = os.path.join(test_path, "../test_config.yaml") Config.clear() Config.init(config, "synchronizer-config-schema.yaml") from multistructlog import create_logger log = create_logger(Config().get('logging')) # END Setting up the config module from xossynchronizer.mock_modelaccessor_build import mock_modelaccessor_config mock_modelaccessor_config( test_path, [("att-workflow-driver", "att-workflow-driver.xproto"), ("olt-service", "volt.xproto"), ("rcord", "rcord.xproto")]) import xossynchronizer.modelaccessor import mock_modelaccessor reload( mock_modelaccessor) # in case nose2 loaded it in a previous test reload(xossynchronizer.modelaccessor ) # in case nose2 loaded it in a previous test from xossynchronizer.modelaccessor import model_accessor from auth_event import SubscriberAuthEventStep # import all class names to globals for (k, v) in model_accessor.all_model_classes.items(): globals()[k] = v self.model_accessor = model_accessor self.log = log self.event_step = SubscriberAuthEventStep( model_accessor=self.model_accessor, log=self.log) self.event = Mock() self.volt = Mock() self.volt.name = "vOLT" self.volt.leaf_model = Mock() self.hippie_si = AttWorkflowDriverServiceInstance() self.hippie_si.serial_number = "BRCM1234" self.hippie_si.save = Mock()
def load_dependency_graph(self): try: if Config.get("dependency_graph"): self.log.debug( "Loading model dependency graph", path=Config.get("dependency_graph"), ) dep_graph_str = open(Config.get("dependency_graph")).read() else: self.log.debug("Using default model dependency graph", graph={}) dep_graph_str = "{}" # joint_dependencies is of the form { Model1 -> [(Model2, src_port, dst_port), ...] } # src_port is the field that accesses Model2 from Model1 # dst_port is the field that accesses Model1 from Model2 static_dependencies = json.loads(dep_graph_str) dynamic_dependencies = [ ] # Dropped Service and ServiceInstance dynamic dependencies joint_dependencies = dict(static_dependencies.items() + dynamic_dependencies) model_dependency_graph = DiGraph() for src_model, deps in joint_dependencies.items(): for dep in deps: dst_model, src_accessor, dst_accessor = dep if src_model != dst_model: edge_label = { "src_accessor": src_accessor, "dst_accessor": dst_accessor, } model_dependency_graph.add_edge( src_model, dst_model, edge_label) model_dependency_graph_rev = model_dependency_graph.reverse( copy=True) self.model_dependency_graph = { # deletion True: model_dependency_graph_rev, False: model_dependency_graph, } self.log.debug("Loaded dependencies", edges=model_dependency_graph.edges()) except Exception as e: self.log.exception("Error loading dependency graph", e=e) raise e
def create_secure_client(self, username, password, arg): """ This method will check if this combination of username/password already has stored orm classes in RESOURCES, otherwise create them """ deferred = defer.Deferred() key = "%s~%s" % (username, password) if key in RESOURCES: reactor.callLater(0, deferred.callback, arg) else: local_cert = Config.get("local_cert") client = SecureClient( endpoint=self.grpc_secure_endpoint, username=username, password=password, cacert=local_cert, ) client.restart_on_disconnect = True # SecureClient is preceeded by an insecure client, so treat all secure clients as previously connected # See CORD-3152 client.was_connected = True client.set_reconnect_callback( functools.partial(self.setup_resources, client, key, deferred, arg)) client.start() return deferred
def test_get_cli_param(self): """ [XOS-Config] Should read CLI -C param """ args = ["-A", "Foo", "-c", "Bar", "-C", "config.yaml"] res = Config.get_cli_param(args) self.assertEqual(res, "config.yaml")
def setUp(self): from xosconfig import Config test_path = os.path.abspath(os.path.dirname( os.path.realpath(__file__))) config = os.path.join(test_path, "test_config.yaml") Config.clear() Config.init(config, "synchronizer-config-schema.yaml") if USE_FAKE_STUB: sys.path.append(PARENT_DIR) # Import these after config, in case they depend on config from xosapi.orm import ORMQuerySet, ORMLocalObjectManager self.ORMQuerySet = ORMQuerySet self.ORMLocalObjectManager = ORMLocalObjectManager
def unload_models(client, reactor, version): # This function is called by a timer until it succeeds. log.info("unload_models initiated by timer") try: result = ModelLoadClient(client).unload_models( Config.get("name"), version=version, cleanup_behavior=ModelLoadClient.AUTOMATICALLY_CLEAN) log.debug("Unload response", result=result) if result.status in [result.SUCCESS, result.SUCCESS_NOTHING_CHANGED]: log.info("Models successfully unloaded. Exiting with status", code=0) sys.exit(0) if result.status == result.TRYAGAIN: log.info("TRYAGAIN received. Expect to try again in 30 seconds.") except Exception as e: # If the synchronizer is operational, then assume the ORM's restart_on_disconnect will deal with the # connection being lost. log.exception( "Error while unloading. Expect to try again in 30 seconds.") Timer(30, functools.partial(unload_models, client, reactor, version)).start()
def setUp(self): b = backend.Backend() steps_dir = Config.get("steps_dir") self.steps = b.load_sync_step_modules(steps_dir) self.synchronizer = event_loop.XOSObserver(self.steps) os.remove('/tmp/sync_ports') os.remove('/tmp/delete_ports')
def main(): input_fn = sys.argv[1] result_fn = sys.argv[2] args = pickle.loads(open(input_fn).read()) Config.init(args['config_file'], 'synchronizer-config-schema.yaml') ansible_hosts = args["ansible_hosts"] ansible_config = args["ansible_config"] fqp = args["fqp"] opts = args["opts"] result = run_playbook(ansible_hosts, ansible_config, fqp, opts) open(result_fn, "w").write(pickle.dumps(result))
def setUp(self): self.sys_path_save = sys.path # Setting up the config module from xosconfig import Config config = os.path.join(test_path, "../test_config.yaml") Config.clear() Config.init(config, "synchronizer-config-schema.yaml") # END Setting up the config module from xossynchronizer.mock_modelaccessor_build import build_mock_modelaccessor # Can't use mock_modelaccessor_config because we're not in the xos-services directory, so do it # the long way... xos_dir = os.path.join(test_path, "../../../../xos") services_dir = os.path.join(test_path, "../../../..") service_xprotos = [os.path.join(test_path, "../models/testservice.xproto")] build_mock_modelaccessor(None, xos_dir, services_dir, service_xprotos) import xossynchronizer.modelaccessor import mock_modelaccessor imp.reload(mock_modelaccessor) # in case nose2 loaded it in a previous test imp.reload(xossynchronizer.modelaccessor) # in case nose2 loaded it in a previous test from sync_testservice_serviceinstance import SyncTestserviceServiceInstance from xossynchronizer.modelaccessor import model_accessor self.model_accessor = model_accessor # import all class names to globals for (k, v) in model_accessor.all_model_classes.items(): globals()[k] = v self.sync_step = SyncTestserviceServiceInstance # TODO: Use modelaccessor instead # create a mock instance instance self.model = Mock(name="Example", some_integer=0, sync_after_policy=False, sync_during_policy=False, policy_after_sync=False, policy_during_sync=False, update_during_sync=False, update_during_policy=False, create_duplicate=False)
def setUp(self): self.sys_path_save = sys.path # Setting up the config module from xosconfig import Config config = os.path.join(test_path, "../test_config.yaml") Config.clear() Config.init(config, "synchronizer-config-schema.yaml") # END Setting up the config module from xossynchronizer.mock_modelaccessor_build import mock_modelaccessor_config mock_modelaccessor_config( test_path, [("dt-workflow-driver", "dt-workflow-driver.xproto"), ("olt-service", "volt.xproto"), ("rcord", "rcord.xproto")]) import xossynchronizer.modelaccessor import mock_modelaccessor reload( mock_modelaccessor) # in case nose2 loaded it in a previous test reload(xossynchronizer.modelaccessor ) # in case nose2 loaded it in a previous test from xossynchronizer.modelaccessor import model_accessor from onu_event import ONUEventStep # import all class names to globals for (k, v) in model_accessor.all_model_classes.items(): globals()[k] = v self.model_accessor = model_accessor self.log = Mock() self.event_step = ONUEventStep(model_accessor=self.model_accessor, log=self.log) self.event = Mock() self.event_dict = { 'status': 'activated', 'serialNumber': 'BRCM1234', 'deviceId': 'of:109299321', 'portNumber': '16' } self.event.value = json.dumps(self.event_dict) self.pppoe = DtWorkflowDriverService(name="dt-workflow-driver")
def run_playbook(ansible_hosts, ansible_config, fqp, opts): args = { "ansible_hosts": ansible_hosts, "ansible_config": ansible_config, "fqp": fqp, "opts": opts, "config_file": Config.get_config_file() } keep_temp_files = Config.get("keep_temp_files") dir = tempfile.mkdtemp() args_fn = None result_fn = None try: logger.info("creating args file in %s" % dir) args_fn = os.path.join(dir, "args") result_fn = os.path.join(dir, "result") open(args_fn, "w").write(pickle.dumps(args)) ansible_main_fn = os.path.join(os.path.dirname(__file__), "ansible_main.py") os.system("python %s %s %s" % (ansible_main_fn, args_fn, result_fn)) result = pickle.loads(open(result_fn).read()) if hasattr(result, "exception"): logger.log_error("Exception in playbook: %s" % result["exception"]) stats = result.get("stats", None) aresults = result.get("aresults", None) except: logger.log_exc("Exception running ansible_main") stats = None aresults = None finally: if not keep_temp_files: if args_fn and os.path.exists(args_fn): os.remove(args_fn) if result_fn and os.path.exists(result_fn): os.remove(result_fn) os.rmdir(dir) return (stats, aresults)
def setUp(self): global DeferredException self.sys_path_save = sys.path sys.path.append(xos_dir) sys.path.append(os.path.join(xos_dir, 'synchronizers', 'new_base')) # Setting up the config module from xosconfig import Config config = os.path.join(test_path, "../test_config.yaml") Config.clear() Config.init(config, "synchronizer-config-schema.yaml") # END Setting up the config module from synchronizers.new_base.mock_modelaccessor_build import build_mock_modelaccessor build_mock_modelaccessor(xos_dir, services_dir, [get_models_fn("fabric", "fabric.xproto")]) import synchronizers.new_base.modelaccessor from sync_fabric_switch import SyncFabricSwitch, model_accessor # import all class names to globals for (k, v) in model_accessor.all_model_classes.items(): globals()[k] = v self.sync_step = SyncFabricSwitch self.sync_step.log = Mock() # mock onos-fabric onos_fabric = Mock() onos_fabric.name = "onos-fabric" onos_fabric.rest_hostname = "onos-fabric" onos_fabric.rest_port = "8181" onos_fabric.rest_username = "******" onos_fabric.rest_password = "******" onos_fabric_base = Mock() onos_fabric_base.leaf_model = onos_fabric self.fabric = Mock() self.fabric.name = "fabric" self.fabric.provider_services = [onos_fabric_base] # create a mock Switch instance self.o = Mock() self.o.name = "MockSwitch" self.o.ofId = "of:1234"
def run(self): observer_thread = None watcher_thread = None model_policy_thread = None model_accessor.update_diag(sync_start=time.time(), backend_status="0 - Synchronizer Start") steps_dir = Config.get("steps_dir") if steps_dir: sync_steps = self.load_sync_step_modules(steps_dir) if sync_steps: # start the observer observer = XOSObserver(sync_steps) observer_thread = threading.Thread(target=observer.run,name='synchronizer') observer_thread.start() # start the watcher thread if (watchers_enabled): watcher = XOSWatcher(sync_steps) watcher_thread = threading.Thread(target=watcher.run,name='watcher') watcher_thread.start() else: logger.info("Skipping observer and watcher threads due to no steps dir.") # start model policies thread policies_dir = Config.get("model_policies_dir") if policies_dir: policy_engine = XOSPolicyEngine(policies_dir=policies_dir) model_policy_thread = threading.Thread(target=policy_engine.run, name="policy_engine") model_policy_thread.start() else: logger.info("Skipping model policies thread due to no model_policies dir.") while True: try: time.sleep(1000) except KeyboardInterrupt: print "exiting due to keyboard interrupt" # TODO: See about setting the threads as daemons if observer_thread: observer_thread._Thread__stop() if watcher_thread: watcher_thread._Thread__stop() if model_policy_thread: model_policy_thread._Thread__stop() sys.exit(1)
def setUp(self): self.sys_path_save = sys.path self.cwd_save = os.getcwd() config = os.path.join(test_path, "test_config.yaml") from xosconfig import Config Config.clear() Config.init(config, "synchronizer-config-schema.yaml") from xossynchronizer import loadmodels from xossynchronizer.loadmodels import ModelLoadClient self.loadmodels = loadmodels self.api = MagicMock() self.api.dynamicload_pb2.LoadModelsRequest = MockLoadModelsRequest self.loader = ModelLoadClient(self.api)
def setUp(self): self.sys_path_save = sys.path sys.path.append(xos_dir) sys.path.append(os.path.join(xos_dir, 'synchronizers', 'new_base')) # Setting up the config module from xosconfig import Config config = os.path.join(test_path, "../test_config.yaml") Config.clear() Config.init(config, "synchronizer-config-schema.yaml") from multistructlog import create_logger log = create_logger(Config().get('logging')) # END Setting up the config module from synchronizers.new_base.mock_modelaccessor_build import build_mock_modelaccessor # build_mock_modelaccessor(xos_dir, services_dir, [get_models_fn("olt-service", "volt.xproto")]) build_mock_modelaccessor(xos_dir, services_dir, [ get_models_fn("hippie-oss", "hippie-oss.xproto"), get_models_fn("olt-service", "volt.xproto"), get_models_fn("rcord", "rcord.xproto") ]) import synchronizers.new_base.modelaccessor from dhcp_event import SubscriberDhcpEventStep, model_accessor # import all class names to globals for (k, v) in model_accessor.all_model_classes.items(): globals()[k] = v self.log = log self.event_step = SubscriberDhcpEventStep(self.log) self.event = Mock() self.volt = Mock() self.volt.name = "vOLT" self.volt.leaf_model = Mock() self.subscriber = RCORDSubscriber() self.subscriber.onu_device = "BRCM1234" self.subscriber.save = Mock() self.mac_address = "aa:bb:cc:dd:ee" self.ip_address = "192.168.3.5"
def test_get_service_endpoint(self): """ [XOS-Config] Should query registrator and return service endpoint """ with patch("xosconfig.config.requests.get") as mock_get: mock_get.return_value.json.return_value = db_service endpoint = Config.get_service_endpoint("xos-db") self.assertEqual(endpoint, "http://172.18.0.4:5432")
def setUp(self): self.mock_etcd = Mock(name="etcd-client") etcd = Mock(name="etcd-mocked-lib") etcd.client.return_value = self.mock_etcd modules = { 'etcd3': etcd } self.module_patcher = patch.dict('sys.modules', modules) self.module_patcher.start() self.sys_path_save = sys.path # Setting up the config module from xosconfig import Config config = os.path.join(test_path, "../test_config.yaml") Config.clear() Config.init(config, "synchronizer-config-schema.yaml") # END Setting up the config module from xossynchronizer.mock_modelaccessor_build import mock_modelaccessor_config mock_modelaccessor_config(test_path, [("olt-service", "volt.xproto"), ("rcord", "rcord.xproto")]) import xossynchronizer.modelaccessor reload(xossynchronizer.modelaccessor) # in case nose2 loaded it in a previous test from xossynchronizer.modelaccessor import model_accessor self.model_accessor = model_accessor from xossynchronizer.modelaccessor import model_accessor from sync_tech_profile import SyncTechnologyProfile # import all class names to globals for (k, v) in model_accessor.all_model_classes.items(): globals()[k] = v self.sync_step = SyncTechnologyProfile self.o = Mock() self.o.technology = "test_technology" self.o.profile_id = 64 self.o.profile_value = '{"test":"profile"}' self.o.tologdict.return_value = {'name': "mock-tp"}
def run(self): # This is our unique client id, to be used when firing and receiving events # It needs to be generated once and placed in the config file user = Config.get("feefie.client_user") try: clid = Config.get("feefie.client_id") except: clid = get_random_client_id() print "EventListener: no feefie_client_id configured. Using random id %s" % clid if fofum_enabled: f = Fofum(user=user) listener_thread = threading.Thread(target=f.listen_for_event, args=(clid, self.handle_event)) listener_thread.start()
def create_kafka_consumer(self): # use the service name as the group id consumer_config = { "group.id": Config().get("name"), "bootstrap.servers": ",".join(self.bootstrap_servers), "default.topic.config": {"auto.offset.reset": "smallest"}, } return confluent_kafka.Consumer(**consumer_config)
def __init__(self, sync_steps): # The Condition object that gets signalled by Feefie events self.step_lookup = {} self.sync_steps = sync_steps self.load_sync_steps() self.event_cond = threading.Condition() self.driver = DRIVER self.observer_name = Config.get("name")
def config_accessor(): accessor_kind = Config.get("accessor.kind") if accessor_kind == "testframework": pass elif accessor_kind == "grpcapi": config_accessor_grpcapi() else: raise Exception("Unknown accessor kind %s" % accessor_kind)
def extract_context(self, cur): try: observer_name = Config.get("name") cur['synchronizer_name'] = observer_name except: pass self.sanitize_extra_args(cur) return cur