def __init__(self):
self.db_name = Config().get("database.name")
self.db_username = Config().get("database.username")
self.db_password = Config().get("database.password")
self.db_host = "xos-db"
self.db_port = "5432"
self.log = create_logger(Config().get("logging"))
def init():
global log
global kafka_producer
if not log:
log = create_logger(Config().get("logging"))
if kafka_producer:
raise Exception("XOSKafkaProducer already initialized")
else:
log.info("Connecting to Kafka with bootstrap servers: %s" %
Config.get("kafka_bootstrap_servers"))
try:
producer_config = {
"bootstrap.servers":
",".join(Config.get("kafka_bootstrap_servers"))
}
kafka_producer = confluent_kafka.Producer(**producer_config)
log.info("Connected to Kafka: %s" % kafka_producer)
except confluent_kafka.KafkaError as e:
log.exception("Kafka Error: %s" % e)
def main():
log = create_logger(Config().get("logging"))
models_active = False
wait = False
while not models_active:
try:
_ = Instance.objects.first()
_ = NetworkTemplate.objects.first()
models_active = True
except Exception as e:
log.exception("Exception", e=e)
log.info("Waiting for data model to come up before starting...")
time.sleep(10)
wait = True
if wait:
time.sleep(
60
) # Safety factor, seeing that we stumbled waiting for the data model to come up.
# start model policies thread
policies_dir = Config.get("model_policies_dir")
XOSPolicyEngine(policies_dir=policies_dir, log=log).run()
def create_kafka_consumer(self):
# use the service name as the group id
consumer_config = {
"group.id": Config().get("name"),
"bootstrap.servers": ",".join(self.bootstrap_servers),
"default.topic.config": {"auto.offset.reset": "smallest"},
}
return confluent_kafka.Consumer(**consumer_config)
def configure_logging(verbose):
global log
# INITIALIZING LOGGER
Config.init()
cfg = Config().get("logging")
if verbose:
cfg["handlers"]["console"]["level"] = "DEBUG"
log = create_logger(cfg)
def setUp(self):
self.sys_path_save = sys.path
# Setting up the config module
from xosconfig import Config
config = os.path.join(test_path, "../test_config.yaml")
Config.clear()
Config.init(config, "synchronizer-config-schema.yaml")
from multistructlog import create_logger
log = create_logger(Config().get('logging'))
# END Setting up the config module
from xossynchronizer.mock_modelaccessor_build import mock_modelaccessor_config
mock_modelaccessor_config(test_path, [("dt-workflow-driver", "dt-workflow-driver.xproto"),
("olt-service", "volt.xproto"),
("rcord", "rcord.xproto")])
import xossynchronizer.modelaccessor
import mock_modelaccessor
reload(mock_modelaccessor) # in case nose2 loaded it in a previous test
reload(xossynchronizer.modelaccessor) # in case nose2 loaded it in a previous test
from xossynchronizer.modelaccessor import model_accessor
from pppoe_event import SubscriberPppoeEventStep
# import all class names to globals
for (k, v) in model_accessor.all_model_classes.items():
globals()[k] = v
self.model_accessor = model_accessor
self.log = log
self.event_step = SubscriberPppoeEventStep(model_accessor=self.model_accessor, log=self.log)
self.event = Mock()
self.volt = Mock()
self.volt.name = "vOLT"
self.volt.leaf_model = Mock()
# self.subscriber = RCORDSubscriber()
# self.subscriber.onu_device = "BRCM1234"
# self.subscriber.save = Mock()
self.mac_address = "00:AA:00:00:00:01"
self.ip_address = "192.168.3.5"
self.pppoe_session_id = "12"
self.si = DtWorkflowDriverServiceInstance()
self.si.serial_number = "BRCM1234"
self.si.save = Mock()
def run(self):
self.create_model_accessor()
self.wait_for_ready()
# Don't import backend until after the model accessor has been initialized. This is to support sync steps that
# use `from xossynchronizer.modelaccessor import ...` and require the model accessor to be initialized before
# their code can be imported.
from .backend import Backend
log_closure = self.log.bind(synchronizer_name=Config().get("name"))
backend = Backend(log=log_closure, model_accessor=self.model_accessor)
backend.run()
def load_model_policies(policies_dir=None):
global model_policies
if policies_dir is None:
policies_dir = Config().observer_model_policies_dir
for fn in os.listdir(policies_dir):
pathname = os.path.join(policies_dir, fn)
if (os.path.isfile(pathname) and fn.startswith("model_policy_")
and fn.endswith(".py") and (fn != "__init__.py")):
model_policies[fn[:-3]] = imp.load_source(fn[:-3], pathname)
log.debug("Loaded model polices %s from %s" %
(",".join(model_policies.keys()), policies_dir))
def setUpClass(cls):
global log
config = os.path.join(test_path, "test_config.yaml")
from xosconfig import Config
Config.clear()
Config.init(config, "synchronizer-config-schema.yaml")
if not log:
from multistructlog import create_logger
log = create_logger(Config().get("logging"))
def setUp(self):
self.sys_path_save = sys.path
sys.path.append(xos_dir)
sys.path.append(os.path.join(xos_dir, 'synchronizers', 'new_base'))
# Setting up the config module
from xosconfig import Config
config = os.path.join(test_path, "../test_config.yaml")
Config.clear()
Config.init(config, "synchronizer-config-schema.yaml")
from multistructlog import create_logger
log = create_logger(Config().get('logging'))
# END Setting up the config module
from synchronizers.new_base.mock_modelaccessor_build import build_mock_modelaccessor
# build_mock_modelaccessor(xos_dir, services_dir, [get_models_fn("olt-service", "volt.xproto")])
build_mock_modelaccessor(xos_dir, services_dir, [
get_models_fn("hippie-oss", "hippie-oss.xproto"),
get_models_fn("olt-service", "volt.xproto"),
get_models_fn("rcord", "rcord.xproto")
])
import synchronizers.new_base.modelaccessor
from dhcp_event import SubscriberDhcpEventStep, model_accessor
# import all class names to globals
for (k, v) in model_accessor.all_model_classes.items():
globals()[k] = v
self.log = log
self.event_step = SubscriberDhcpEventStep(self.log)
self.event = Mock()
self.volt = Mock()
self.volt.name = "vOLT"
self.volt.leaf_model = Mock()
self.subscriber = RCORDSubscriber()
self.subscriber.onu_device = "BRCM1234"
self.subscriber.save = Mock()
self.mac_address = "aa:bb:cc:dd:ee"
self.ip_address = "192.168.3.5"
def setUp(self):
self.sys_path_save = sys.path
# Setting up the config module
from xosconfig import Config
config = os.path.join(test_path, "../test_config.yaml")
Config.clear()
Config.init(config, "synchronizer-config-schema.yaml")
from multistructlog import create_logger
log = create_logger(Config().get('logging'))
# END Setting up the config module
from xossynchronizer.mock_modelaccessor_build import mock_modelaccessor_config
mock_modelaccessor_config(
test_path, [("att-workflow-driver", "att-workflow-driver.xproto"),
("olt-service", "volt.xproto"),
("rcord", "rcord.xproto")])
import xossynchronizer.modelaccessor
import mock_modelaccessor
reload(
mock_modelaccessor) # in case nose2 loaded it in a previous test
reload(xossynchronizer.modelaccessor
) # in case nose2 loaded it in a previous test
from xossynchronizer.modelaccessor import model_accessor
from auth_event import SubscriberAuthEventStep
# import all class names to globals
for (k, v) in model_accessor.all_model_classes.items():
globals()[k] = v
self.model_accessor = model_accessor
self.log = log
self.event_step = SubscriberAuthEventStep(
model_accessor=self.model_accessor, log=self.log)
self.event = Mock()
self.att_si = AttWorkflowDriverServiceInstance()
self.att_si.serial_number = "BRCM1234"
self.att_si.save = Mock()
def run(self):
# start model policies thread
policies_dir = Config("model_policies_dir")
if policies_dir:
from synchronizers.model_policy import run_policy
model_policy_thread = threading.Thread(target=run_policy)
model_policy_thread.start()
else:
model_policy_thread = None
log.info("Skipping model policies thread due to no model_policies dir.")
while True:
try:
time.sleep(1000)
except KeyboardInterrupt:
print "exiting due to keyboard interrupt"
if model_policy_thread:
model_policy_thread._Thread__stop()
sys.exit(1)
def main():
models_active = False
wait = False
while not models_active:
try:
_i = Instance.objects.first()
_n = NetworkTemplate.objects.first()
models_active = True
except Exception as e:
log.info("Exception", e=e)
log.info("Waiting for data model to come up before starting...")
time.sleep(10)
wait = True
if wait:
time.sleep(
60
) # Safety factor, seeing that we stumbled waiting for the data model to come up.
log_closure = log.bind(synchronizer_name=Config().get("name"))
backend = Backend(log=log_closure)
backend.run()
from collections import defaultdict
from networkx import (
DiGraph,
weakly_connected_component_subgraphs,
all_shortest_paths,
NetworkXNoPath,
)
from networkx.algorithms.dag import topological_sort
from xossynchronizer.steps.syncstep import InnocuousException, DeferredException, SyncStep
from xosconfig import Config
from multistructlog import create_logger
log = create_logger(Config().get("logging"))
class StepNotReady(Exception):
pass
class ExternalDependencyFailed(Exception):
pass
# FIXME: Move drivers into a context shared across sync steps.
class NoOpDriver:
def __init__(self):
# limitations under the License.
import os
import sys
from xossynchronizer.steps.syncstep import SyncStep
from xossynchronizer.modelaccessor import MCordSubscriberInstance, ServiceInstanceLink, ProgranServiceInstance
from xosconfig import Config
from multistructlog import create_logger
import json
import requests
from requests.auth import HTTPBasicAuth
log = create_logger(Config().get('logging'))
parentdir = os.path.join(os.path.dirname(__file__), "..")
sys.path.insert(0, parentdir)
sys.path.insert(0, os.path.dirname(__file__))
from helpers import ProgranHelpers
class SyncProgranIMSILink(SyncStep):
provides = [ServiceInstanceLink]
observes = ServiceInstanceLink
# NOTE Override the default fetch_pending method to receive on links between MCordSubscriberInstances and ProgranServiceInstances
def fetch_pending(self, deleted):
objs = super(SyncProgranIMSILink, self).fetch_pending(deleted)
# See the License for the specific language governing permissions and
# limitations under the License.
#!/usr/bin/env python
import os
import argparse
import sys
sys.path.append('/opt/xos')
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "xos.settings")
from xosconfig import Config
from multistructlog import create_logger
log = create_logger(Config().get('logging'))
import time
from synchronizers.new_base.modelaccessor import *
from synchronizers.new_base.backend import Backend
def main():
models_active = False
wait = False
while not models_active:
try:
_ = Instance.objects.first()
_ = NetworkTemplate.objects.first()
models_active = True
except Exception,e:
def __init__(self):
self.log = create_logger(Config().get("logging"))
def __init__(self):
self.backup_request_dir = "/var/run/xos/backup/requests"
self.backup_response_dir = "/var/run/xos/backup/responses"
self.backup_file_dir = "/var/run/xos/backup/local"
self.log = create_logger(Config().get("logging"))
def setUp(self):
self.sys_path_save = sys.path
# Setting up the config module
from xosconfig import Config
config = os.path.join(test_path, "test_config.yaml")
Config.clear()
Config.init(config, "synchronizer-config-schema.yaml")
# END Setting up the config module
from multistructlog import create_logger
self.log = create_logger(Config().get('logging'))
from xossynchronizer.mock_modelaccessor_build import mock_modelaccessor_config
mock_modelaccessor_config(
test_path, [("ntt-workflow-driver", "ntt-workflow-driver.xproto"),
("olt-service", "volt.xproto"),
("rcord", "rcord.xproto")])
import xossynchronizer.modelaccessor
import mock_modelaccessor
reload(
mock_modelaccessor) # in case nose2 loaded it in a previous test
reload(xossynchronizer.modelaccessor
) # in case nose2 loaded it in a previous test
from xossynchronizer.modelaccessor import model_accessor
from helpers import NttHelpers
# import all class names to globals
for (k, v) in model_accessor.all_model_classes.items():
globals()[k] = v
self.helpers = NttHelpers
self.model_accessor = model_accessor
self._volt = VOLTService()
self._volt.id = 1
self.volt = Service()
self.volt.id = 1
self.volt.name = "vOLT"
self.volt.leaf_model = self._volt
self.pon_port = PONPort()
self.pon_port.port_no = 1234
self.onu = ONUDevice()
self.onu.pon_port = self.pon_port
self.onu.serial_number = "BRCM1234"
self.technologyProfile = TechnologyProfile()
self.technologyProfile.profile_id = 64
self.technologyProfile.profile_value = '{"profile_type": "EPON","epon_attribute": {"package_type": "A"}}'
self.ntt_si = NttWorkflowDriverServiceInstance(
serial_number="BRCM1234",
owner=self.volt,
owner_id=self.volt.id,
mac_address="0a0a0a",
of_dpid="of:1234")
self.whitelist_entry = NttWorkflowDriverWhiteListEntry(
mac_address="0a0a0a",
owner=self.volt,
owner_id=self.volt.id,
pon_port_from=1234,
pon_port_to=1235,
)
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from synchronizers.new_base.modelaccessor import *
from synchronizers.new_base.model_policies.model_policy_tenantwithcontainer import Policy
from synchronizers.new_base.exceptions import *
from xosconfig import Config
from multistructlog import create_logger
log = create_logger(Config().get('logging'))
blueprints = Config().get('blueprints')
class VEPCServiceInstancePolicy(Policy):
model_name = "VEPCServiceInstance"
def get_service_object(self, name):
""" return: First Service object """
if any(map(lambda x: x in name, ["ServiceInstance", "Tenant"])):
name = name.replace("ServiceInstance", "Service")
name = name.replace("Tenant", "Service")
service_obj = getattr(Slice().stub, name).objects.first()
if not service_obj:
class SyncFabricCrossconnectServiceInstance(SyncStep):
provides = [FabricCrossconnectServiceInstance]
log = create_logger(Config().get('logging'))
observes = FabricCrossconnectServiceInstance
@staticmethod
def format_url(url):
if 'http' in url:
return url
else:
return 'http://%s' % url
@staticmethod
def get_fabric_onos_info(si):
# get the fabric-crossconnect service
fabric_crossconnect = si.owner
# get the onos_fabric service
fabric_onos = [
s.leaf_model for s in fabric_crossconnect.provider_services
if "onos" in s.name.lower()
]
if len(fabric_onos) == 0:
raise Exception(
'Cannot find ONOS service in provider_services of Fabric-Crossconnect'
)
fabric_onos = fabric_onos[0]
return {
'url':
SyncFabricCrossconnectServiceInstance.format_url(
"%s:%s" % (fabric_onos.rest_hostname, fabric_onos.rest_port)),
'user':
fabric_onos.rest_username,
'pass':
fabric_onos.rest_password
}
def make_handle(self, s_tag, switch_datapath_id):
# Generate a backend_handle that uniquely identifies the cross connect. ONOS doesn't provide us a handle, so
# we make up our own. This helps us to detect other FabricCrossconnectServiceInstance using the same
# entry, as well as to be able to extract the necessary information to delete the entry later.
return "%d/%s" % (s_tag, switch_datapath_id)
def extract_handle(self, backend_handle):
(s_tag, switch_datapath_id) = backend_handle.split("/", 1)
s_tag = int(s_tag)
return (s_tag, switch_datapath_id)
def range_matches(self, value, pattern):
value = int(value)
for this_range in pattern.split(","):
this_range = this_range.strip()
if "-" in this_range:
(first, last) = this_range.split("-")
first = int(first.strip())
last = int(last.strip())
if (value >= first) and (value <= last):
return True
elif this_range.lower() == "any":
return True
else:
if (value == int(this_range)):
return True
return False
def find_bng(self, s_tag):
# See if there's a mapping for our s-tag directly
bng_mappings = BNGPortMapping.objects.filter(s_tag=str(s_tag))
if bng_mappings:
return bng_mappings[0]
# TODO(smbaker): Examine miss performance, and if necessary set a flag in the save method to allow filtering
# of mappings based on whether they are ranges or any.
# See if there are any ranges or "any" that match
for bng_mapping in BNGPortMapping.objects.all():
if self.range_matches(s_tag, bng_mapping.s_tag):
return bng_mapping
return None
def sync_record(self, o):
self.log.info("Sync'ing Fabric Crossconnect Service Instance",
service_instance=o)
if (o.policed is None) or (o.policed < o.updated):
raise DeferredException(
"Waiting for model_policy to run on fcsi %s" % o.id)
onos = self.get_fabric_onos_info(o)
si = ServiceInstance.objects.get(id=o.id)
if (o.s_tag is None):
raise Exception(
"Cannot sync FabricCrossconnectServiceInstance if s_tag is None on fcsi %s"
% o.id)
if (o.source_port is None):
raise Exception(
"Cannot sync FabricCrossconnectServiceInstance if source_port is None on fcsi %s"
% o.id)
if (not o.switch_datapath_id):
raise Exception(
"Cannot sync FabricCrossconnectServiceInstance if switch_datapath_id is unset on fcsi %s"
% o.id)
bng_mapping = self.find_bng(s_tag=o.s_tag)
if not bng_mapping:
raise Exception("Unable to determine BNG port for s_tag %s" %
o.s_tag)
east_port = bng_mapping.switch_port
data = {
"deviceId": o.switch_datapath_id,
"vlanId": o.s_tag,
"ports": [int(o.source_port), int(east_port)]
}
url = onos['url'] + '/onos/segmentrouting/xconnect'
self.log.info("Sending request to ONOS", url=url, body=data)
r = requests.post(url,
json=data,
auth=HTTPBasicAuth(onos['user'], onos['pass']))
if r.status_code != 200:
raise Exception(
"Failed to create fabric crossconnect in ONOS: %s" % r.text)
# TODO(smbaker): If the o.backend_handle changed, then someone must have changed the
# FabricCrossconnectServiceInstance. If so, then we potentially need to clean up the old
# entry in ONOS. Furthermore, we might want to also save the two port numbers that we used,
# to detect someone changing those.
o.backend_handle = self.make_handle(o.s_tag, o.switch_datapath_id)
o.save(update_fields=["backend_handle"])
self.log.info("ONOS response", res=r.text)
def delete_record(self, o):
self.log.info("Deleting Fabric Crossconnect Service Instance",
service_instance=o)
if o.backend_handle:
onos = self.get_fabric_onos_info(o)
# backend_handle has everything we need in it to delete this entry.
(s_tag, switch_datapath_id) = self.extract_handle(o.backend_handle)
data = {"deviceId": switch_datapath_id, "vlanId": s_tag}
url = onos['url'] + '/onos/segmentrouting/xconnect'
r = requests.delete(url,
json=data,
auth=HTTPBasicAuth(onos['user'], onos['pass']))
if r.status_code != 204:
raise Exception(
"Failed to remove fabric crossconnect in ONOS: %s" %
r.text)
self.log.info("ONOS response", res=r.text)
# See the License for the specific language governing permissions and
# limitations under the License.
import json
import requests
from requests.auth import HTTPBasicAuth
from xossynchronizer.steps.syncstep import SyncStep, DeferredException
from xossynchronizer.modelaccessor import model_accessor
from xossynchronizer.modelaccessor import ONOSApp, ServiceInstance, ServiceInstanceAttribute
from xosconfig import Config
from multistructlog import create_logger
from helpers import Helpers
log = create_logger(Config().get('logging'))
log.info("config file", file=Config().get_config_file())
class SyncONOSApp(SyncStep):
provides = [ONOSApp]
observes = [ONOSApp, ServiceInstanceAttribute]
def get_service_instance_attribute(self, o):
# NOTE this method is defined in the core convenience methods for service_instances
svc = ServiceInstance.objects.get(id=o.id)
return svc.serviceinstanceattribute_dict
def check_app_dependencies(self, deps):
"""
Check if all the dependencies required by this application are installed
