def __init__(self): self.db_name = Config().get("database.name") self.db_username = Config().get("database.username") self.db_password = Config().get("database.password") self.db_host = "xos-db" self.db_port = "5432" self.log = create_logger(Config().get("logging"))
def init(): global log global kafka_producer if not log: log = create_logger(Config().get("logging")) if kafka_producer: raise Exception("XOSKafkaProducer already initialized") else: log.info("Connecting to Kafka with bootstrap servers: %s" % Config.get("kafka_bootstrap_servers")) try: producer_config = { "bootstrap.servers": ",".join(Config.get("kafka_bootstrap_servers")) } kafka_producer = confluent_kafka.Producer(**producer_config) log.info("Connected to Kafka: %s" % kafka_producer) except confluent_kafka.KafkaError as e: log.exception("Kafka Error: %s" % e)
def main(): log = create_logger(Config().get("logging")) models_active = False wait = False while not models_active: try: _ = Instance.objects.first() _ = NetworkTemplate.objects.first() models_active = True except Exception as e: log.exception("Exception", e=e) log.info("Waiting for data model to come up before starting...") time.sleep(10) wait = True if wait: time.sleep( 60 ) # Safety factor, seeing that we stumbled waiting for the data model to come up. # start model policies thread policies_dir = Config.get("model_policies_dir") XOSPolicyEngine(policies_dir=policies_dir, log=log).run()
def create_kafka_consumer(self): # use the service name as the group id consumer_config = { "group.id": Config().get("name"), "bootstrap.servers": ",".join(self.bootstrap_servers), "default.topic.config": {"auto.offset.reset": "smallest"}, } return confluent_kafka.Consumer(**consumer_config)
def configure_logging(verbose): global log # INITIALIZING LOGGER Config.init() cfg = Config().get("logging") if verbose: cfg["handlers"]["console"]["level"] = "DEBUG" log = create_logger(cfg)
def setUp(self): self.sys_path_save = sys.path # Setting up the config module from xosconfig import Config config = os.path.join(test_path, "../test_config.yaml") Config.clear() Config.init(config, "synchronizer-config-schema.yaml") from multistructlog import create_logger log = create_logger(Config().get('logging')) # END Setting up the config module from xossynchronizer.mock_modelaccessor_build import mock_modelaccessor_config mock_modelaccessor_config(test_path, [("dt-workflow-driver", "dt-workflow-driver.xproto"), ("olt-service", "volt.xproto"), ("rcord", "rcord.xproto")]) import xossynchronizer.modelaccessor import mock_modelaccessor reload(mock_modelaccessor) # in case nose2 loaded it in a previous test reload(xossynchronizer.modelaccessor) # in case nose2 loaded it in a previous test from xossynchronizer.modelaccessor import model_accessor from pppoe_event import SubscriberPppoeEventStep # import all class names to globals for (k, v) in model_accessor.all_model_classes.items(): globals()[k] = v self.model_accessor = model_accessor self.log = log self.event_step = SubscriberPppoeEventStep(model_accessor=self.model_accessor, log=self.log) self.event = Mock() self.volt = Mock() self.volt.name = "vOLT" self.volt.leaf_model = Mock() # self.subscriber = RCORDSubscriber() # self.subscriber.onu_device = "BRCM1234" # self.subscriber.save = Mock() self.mac_address = "00:AA:00:00:00:01" self.ip_address = "192.168.3.5" self.pppoe_session_id = "12" self.si = DtWorkflowDriverServiceInstance() self.si.serial_number = "BRCM1234" self.si.save = Mock()
def run(self): self.create_model_accessor() self.wait_for_ready() # Don't import backend until after the model accessor has been initialized. This is to support sync steps that # use `from xossynchronizer.modelaccessor import ...` and require the model accessor to be initialized before # their code can be imported. from .backend import Backend log_closure = self.log.bind(synchronizer_name=Config().get("name")) backend = Backend(log=log_closure, model_accessor=self.model_accessor) backend.run()
def load_model_policies(policies_dir=None): global model_policies if policies_dir is None: policies_dir = Config().observer_model_policies_dir for fn in os.listdir(policies_dir): pathname = os.path.join(policies_dir, fn) if (os.path.isfile(pathname) and fn.startswith("model_policy_") and fn.endswith(".py") and (fn != "__init__.py")): model_policies[fn[:-3]] = imp.load_source(fn[:-3], pathname) log.debug("Loaded model polices %s from %s" % (",".join(model_policies.keys()), policies_dir))
def setUpClass(cls): global log config = os.path.join(test_path, "test_config.yaml") from xosconfig import Config Config.clear() Config.init(config, "synchronizer-config-schema.yaml") if not log: from multistructlog import create_logger log = create_logger(Config().get("logging"))
def setUp(self): self.sys_path_save = sys.path sys.path.append(xos_dir) sys.path.append(os.path.join(xos_dir, 'synchronizers', 'new_base')) # Setting up the config module from xosconfig import Config config = os.path.join(test_path, "../test_config.yaml") Config.clear() Config.init(config, "synchronizer-config-schema.yaml") from multistructlog import create_logger log = create_logger(Config().get('logging')) # END Setting up the config module from synchronizers.new_base.mock_modelaccessor_build import build_mock_modelaccessor # build_mock_modelaccessor(xos_dir, services_dir, [get_models_fn("olt-service", "volt.xproto")]) build_mock_modelaccessor(xos_dir, services_dir, [ get_models_fn("hippie-oss", "hippie-oss.xproto"), get_models_fn("olt-service", "volt.xproto"), get_models_fn("rcord", "rcord.xproto") ]) import synchronizers.new_base.modelaccessor from dhcp_event import SubscriberDhcpEventStep, model_accessor # import all class names to globals for (k, v) in model_accessor.all_model_classes.items(): globals()[k] = v self.log = log self.event_step = SubscriberDhcpEventStep(self.log) self.event = Mock() self.volt = Mock() self.volt.name = "vOLT" self.volt.leaf_model = Mock() self.subscriber = RCORDSubscriber() self.subscriber.onu_device = "BRCM1234" self.subscriber.save = Mock() self.mac_address = "aa:bb:cc:dd:ee" self.ip_address = "192.168.3.5"
def setUp(self): self.sys_path_save = sys.path # Setting up the config module from xosconfig import Config config = os.path.join(test_path, "../test_config.yaml") Config.clear() Config.init(config, "synchronizer-config-schema.yaml") from multistructlog import create_logger log = create_logger(Config().get('logging')) # END Setting up the config module from xossynchronizer.mock_modelaccessor_build import mock_modelaccessor_config mock_modelaccessor_config( test_path, [("att-workflow-driver", "att-workflow-driver.xproto"), ("olt-service", "volt.xproto"), ("rcord", "rcord.xproto")]) import xossynchronizer.modelaccessor import mock_modelaccessor reload( mock_modelaccessor) # in case nose2 loaded it in a previous test reload(xossynchronizer.modelaccessor ) # in case nose2 loaded it in a previous test from xossynchronizer.modelaccessor import model_accessor from auth_event import SubscriberAuthEventStep # import all class names to globals for (k, v) in model_accessor.all_model_classes.items(): globals()[k] = v self.model_accessor = model_accessor self.log = log self.event_step = SubscriberAuthEventStep( model_accessor=self.model_accessor, log=self.log) self.event = Mock() self.att_si = AttWorkflowDriverServiceInstance() self.att_si.serial_number = "BRCM1234" self.att_si.save = Mock()
def run(self): # start model policies thread policies_dir = Config("model_policies_dir") if policies_dir: from synchronizers.model_policy import run_policy model_policy_thread = threading.Thread(target=run_policy) model_policy_thread.start() else: model_policy_thread = None log.info("Skipping model policies thread due to no model_policies dir.") while True: try: time.sleep(1000) except KeyboardInterrupt: print "exiting due to keyboard interrupt" if model_policy_thread: model_policy_thread._Thread__stop() sys.exit(1)
def main(): models_active = False wait = False while not models_active: try: _i = Instance.objects.first() _n = NetworkTemplate.objects.first() models_active = True except Exception as e: log.info("Exception", e=e) log.info("Waiting for data model to come up before starting...") time.sleep(10) wait = True if wait: time.sleep( 60 ) # Safety factor, seeing that we stumbled waiting for the data model to come up. log_closure = log.bind(synchronizer_name=Config().get("name")) backend = Backend(log=log_closure) backend.run()
from collections import defaultdict from networkx import ( DiGraph, weakly_connected_component_subgraphs, all_shortest_paths, NetworkXNoPath, ) from networkx.algorithms.dag import topological_sort from xossynchronizer.steps.syncstep import InnocuousException, DeferredException, SyncStep from xosconfig import Config from multistructlog import create_logger log = create_logger(Config().get("logging")) class StepNotReady(Exception): pass class ExternalDependencyFailed(Exception): pass # FIXME: Move drivers into a context shared across sync steps. class NoOpDriver: def __init__(self):
# limitations under the License. import os import sys from xossynchronizer.steps.syncstep import SyncStep from xossynchronizer.modelaccessor import MCordSubscriberInstance, ServiceInstanceLink, ProgranServiceInstance from xosconfig import Config from multistructlog import create_logger import json import requests from requests.auth import HTTPBasicAuth log = create_logger(Config().get('logging')) parentdir = os.path.join(os.path.dirname(__file__), "..") sys.path.insert(0, parentdir) sys.path.insert(0, os.path.dirname(__file__)) from helpers import ProgranHelpers class SyncProgranIMSILink(SyncStep): provides = [ServiceInstanceLink] observes = ServiceInstanceLink # NOTE Override the default fetch_pending method to receive on links between MCordSubscriberInstances and ProgranServiceInstances def fetch_pending(self, deleted): objs = super(SyncProgranIMSILink, self).fetch_pending(deleted)
# See the License for the specific language governing permissions and # limitations under the License. #!/usr/bin/env python import os import argparse import sys sys.path.append('/opt/xos') os.environ.setdefault("DJANGO_SETTINGS_MODULE", "xos.settings") from xosconfig import Config from multistructlog import create_logger log = create_logger(Config().get('logging')) import time from synchronizers.new_base.modelaccessor import * from synchronizers.new_base.backend import Backend def main(): models_active = False wait = False while not models_active: try: _ = Instance.objects.first() _ = NetworkTemplate.objects.first() models_active = True except Exception,e:
def __init__(self): self.log = create_logger(Config().get("logging"))
def __init__(self): self.backup_request_dir = "/var/run/xos/backup/requests" self.backup_response_dir = "/var/run/xos/backup/responses" self.backup_file_dir = "/var/run/xos/backup/local" self.log = create_logger(Config().get("logging"))
def setUp(self): self.sys_path_save = sys.path # Setting up the config module from xosconfig import Config config = os.path.join(test_path, "test_config.yaml") Config.clear() Config.init(config, "synchronizer-config-schema.yaml") # END Setting up the config module from multistructlog import create_logger self.log = create_logger(Config().get('logging')) from xossynchronizer.mock_modelaccessor_build import mock_modelaccessor_config mock_modelaccessor_config( test_path, [("ntt-workflow-driver", "ntt-workflow-driver.xproto"), ("olt-service", "volt.xproto"), ("rcord", "rcord.xproto")]) import xossynchronizer.modelaccessor import mock_modelaccessor reload( mock_modelaccessor) # in case nose2 loaded it in a previous test reload(xossynchronizer.modelaccessor ) # in case nose2 loaded it in a previous test from xossynchronizer.modelaccessor import model_accessor from helpers import NttHelpers # import all class names to globals for (k, v) in model_accessor.all_model_classes.items(): globals()[k] = v self.helpers = NttHelpers self.model_accessor = model_accessor self._volt = VOLTService() self._volt.id = 1 self.volt = Service() self.volt.id = 1 self.volt.name = "vOLT" self.volt.leaf_model = self._volt self.pon_port = PONPort() self.pon_port.port_no = 1234 self.onu = ONUDevice() self.onu.pon_port = self.pon_port self.onu.serial_number = "BRCM1234" self.technologyProfile = TechnologyProfile() self.technologyProfile.profile_id = 64 self.technologyProfile.profile_value = '{"profile_type": "EPON","epon_attribute": {"package_type": "A"}}' self.ntt_si = NttWorkflowDriverServiceInstance( serial_number="BRCM1234", owner=self.volt, owner_id=self.volt.id, mac_address="0a0a0a", of_dpid="of:1234") self.whitelist_entry = NttWorkflowDriverWhiteListEntry( mac_address="0a0a0a", owner=self.volt, owner_id=self.volt.id, pon_port_from=1234, pon_port_to=1235, )
# http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from synchronizers.new_base.modelaccessor import * from synchronizers.new_base.model_policies.model_policy_tenantwithcontainer import Policy from synchronizers.new_base.exceptions import * from xosconfig import Config from multistructlog import create_logger log = create_logger(Config().get('logging')) blueprints = Config().get('blueprints') class VEPCServiceInstancePolicy(Policy): model_name = "VEPCServiceInstance" def get_service_object(self, name): """ return: First Service object """ if any(map(lambda x: x in name, ["ServiceInstance", "Tenant"])): name = name.replace("ServiceInstance", "Service") name = name.replace("Tenant", "Service") service_obj = getattr(Slice().stub, name).objects.first() if not service_obj:
class SyncFabricCrossconnectServiceInstance(SyncStep): provides = [FabricCrossconnectServiceInstance] log = create_logger(Config().get('logging')) observes = FabricCrossconnectServiceInstance @staticmethod def format_url(url): if 'http' in url: return url else: return 'http://%s' % url @staticmethod def get_fabric_onos_info(si): # get the fabric-crossconnect service fabric_crossconnect = si.owner # get the onos_fabric service fabric_onos = [ s.leaf_model for s in fabric_crossconnect.provider_services if "onos" in s.name.lower() ] if len(fabric_onos) == 0: raise Exception( 'Cannot find ONOS service in provider_services of Fabric-Crossconnect' ) fabric_onos = fabric_onos[0] return { 'url': SyncFabricCrossconnectServiceInstance.format_url( "%s:%s" % (fabric_onos.rest_hostname, fabric_onos.rest_port)), 'user': fabric_onos.rest_username, 'pass': fabric_onos.rest_password } def make_handle(self, s_tag, switch_datapath_id): # Generate a backend_handle that uniquely identifies the cross connect. ONOS doesn't provide us a handle, so # we make up our own. This helps us to detect other FabricCrossconnectServiceInstance using the same # entry, as well as to be able to extract the necessary information to delete the entry later. return "%d/%s" % (s_tag, switch_datapath_id) def extract_handle(self, backend_handle): (s_tag, switch_datapath_id) = backend_handle.split("/", 1) s_tag = int(s_tag) return (s_tag, switch_datapath_id) def range_matches(self, value, pattern): value = int(value) for this_range in pattern.split(","): this_range = this_range.strip() if "-" in this_range: (first, last) = this_range.split("-") first = int(first.strip()) last = int(last.strip()) if (value >= first) and (value <= last): return True elif this_range.lower() == "any": return True else: if (value == int(this_range)): return True return False def find_bng(self, s_tag): # See if there's a mapping for our s-tag directly bng_mappings = BNGPortMapping.objects.filter(s_tag=str(s_tag)) if bng_mappings: return bng_mappings[0] # TODO(smbaker): Examine miss performance, and if necessary set a flag in the save method to allow filtering # of mappings based on whether they are ranges or any. # See if there are any ranges or "any" that match for bng_mapping in BNGPortMapping.objects.all(): if self.range_matches(s_tag, bng_mapping.s_tag): return bng_mapping return None def sync_record(self, o): self.log.info("Sync'ing Fabric Crossconnect Service Instance", service_instance=o) if (o.policed is None) or (o.policed < o.updated): raise DeferredException( "Waiting for model_policy to run on fcsi %s" % o.id) onos = self.get_fabric_onos_info(o) si = ServiceInstance.objects.get(id=o.id) if (o.s_tag is None): raise Exception( "Cannot sync FabricCrossconnectServiceInstance if s_tag is None on fcsi %s" % o.id) if (o.source_port is None): raise Exception( "Cannot sync FabricCrossconnectServiceInstance if source_port is None on fcsi %s" % o.id) if (not o.switch_datapath_id): raise Exception( "Cannot sync FabricCrossconnectServiceInstance if switch_datapath_id is unset on fcsi %s" % o.id) bng_mapping = self.find_bng(s_tag=o.s_tag) if not bng_mapping: raise Exception("Unable to determine BNG port for s_tag %s" % o.s_tag) east_port = bng_mapping.switch_port data = { "deviceId": o.switch_datapath_id, "vlanId": o.s_tag, "ports": [int(o.source_port), int(east_port)] } url = onos['url'] + '/onos/segmentrouting/xconnect' self.log.info("Sending request to ONOS", url=url, body=data) r = requests.post(url, json=data, auth=HTTPBasicAuth(onos['user'], onos['pass'])) if r.status_code != 200: raise Exception( "Failed to create fabric crossconnect in ONOS: %s" % r.text) # TODO(smbaker): If the o.backend_handle changed, then someone must have changed the # FabricCrossconnectServiceInstance. If so, then we potentially need to clean up the old # entry in ONOS. Furthermore, we might want to also save the two port numbers that we used, # to detect someone changing those. o.backend_handle = self.make_handle(o.s_tag, o.switch_datapath_id) o.save(update_fields=["backend_handle"]) self.log.info("ONOS response", res=r.text) def delete_record(self, o): self.log.info("Deleting Fabric Crossconnect Service Instance", service_instance=o) if o.backend_handle: onos = self.get_fabric_onos_info(o) # backend_handle has everything we need in it to delete this entry. (s_tag, switch_datapath_id) = self.extract_handle(o.backend_handle) data = {"deviceId": switch_datapath_id, "vlanId": s_tag} url = onos['url'] + '/onos/segmentrouting/xconnect' r = requests.delete(url, json=data, auth=HTTPBasicAuth(onos['user'], onos['pass'])) if r.status_code != 204: raise Exception( "Failed to remove fabric crossconnect in ONOS: %s" % r.text) self.log.info("ONOS response", res=r.text)
# See the License for the specific language governing permissions and # limitations under the License. import json import requests from requests.auth import HTTPBasicAuth from xossynchronizer.steps.syncstep import SyncStep, DeferredException from xossynchronizer.modelaccessor import model_accessor from xossynchronizer.modelaccessor import ONOSApp, ServiceInstance, ServiceInstanceAttribute from xosconfig import Config from multistructlog import create_logger from helpers import Helpers log = create_logger(Config().get('logging')) log.info("config file", file=Config().get_config_file()) class SyncONOSApp(SyncStep): provides = [ONOSApp] observes = [ONOSApp, ServiceInstanceAttribute] def get_service_instance_attribute(self, o): # NOTE this method is defined in the core convenience methods for service_instances svc = ServiceInstance.objects.get(id=o.id) return svc.serviceinstanceattribute_dict def check_app_dependencies(self, deps): """ Check if all the dependencies required by this application are installed