def __init__(self, netflow_ip='127.0.0.1', netflow_port=23456):
self._create_time = time.time()
self.app_repository = repository.get("app")
self.device_repository = repository.get("device")
self.used_flow_id = repository.get("used_flow_id")
self._netflow_worker = NetflowWorker(
netflow_ip,
netflow_port
)
# Tasks
self._ssh_worker = SSHWorker(
SNMPFetch,
ClearInactiveFlowTask,
ClearPolicyTask,
TrafficMonitorTask,
PolicyMonitorTask,
ClearDeviceTask
)
# Thread for SSH Worker
self._ssh_worker_t = None
self.init()
logging.info("Create topology")
def __init__(self, version):
super(CLIController, self).__init__()
self.version = version
self.config_command = ConfigCommand(version)
self.prompt = 'SDN Handmade ({:s})# '.format(version)
self.logbug.prompt = self.prompt
self.device_repository = repository.get('device')
self.link_utilization_repository = repository.get('link_utilization')
self.copied_route_repository = repository.get('copied_route')
async def process_cdp(device_id, host, community, port):
device_neighbor_repository = repository.get('device_neighbor')
device_repository = repository.get('device')
cdp = await snmp_async.get_cdp(host, community, port)
if cdp is None:
device_repository.set_cdp_by_mgmt_ip(host, False)
else:
# Insert CDP
device_neighbor_repository.update_neighbor(device_id, host, cdp)
device_repository.set_cdp_by_mgmt_ip(host, True)
return True
def __init__(self, auto_update_graph=True):
# If true.
# It automatically update graph when get a path
self.auto_update_graph = auto_update_graph
# Cache Simple path
self._all_simple_paths = {}
# Cache link information
self.link_cache = {}
self.device_repository = repository.get("device")
self.copied_route_repository = repository.get('copied_route')
# Create NetworkX graph
self.graph = None
self.update_graph()
async def get_and_store(device):
""" Get snmp information and add to database
"""
device_id = device['_id']
host = device['management_ip']
community = device['snmp_info']['community']
port = device['snmp_info']['port']
results = await asyncio.gather(
asyncio.ensure_future(
snmp_process.process_system(device_id, host, community, port)),
asyncio.ensure_future(
snmp_process.process_route(device_id, host, community, port)),
asyncio.ensure_future(
snmp_process.process_cdp(device_id, host, community, port)),
)
device_repository = repository.get("device")
l = [r for r in results]
# logging.info(l)
if not all(l):
device_repository.set_snmp_is_connect_by_mgmt_ip(host, False)
logging.info("SNMP Worker: Device ip %s is gone down", host)
else:
device_repository.set_snmp_is_connect_by_mgmt_ip(host, True)
def __init__(self, version):
super(ConfigCommand, self).__init__()
self.version = version
self.prompt = 'SDN Handmade ({:s})(config)# '.format(version)
self.add_device_cmd = AddDeviceCommand()
self.device_repository = repository.get("device")
self.logbug = logbug.get()
def __init__(self, version, name):
super(FlowCommand, self).__init__()
self.name = name
self.prompt = 'SDN Handmade ({:s})(config-flow)# '.format(version)
self.device_repository = repository.get('device')
self.flow_routing_repository = repository.get('flow_routing')
# self.used_flow_id_repository = repository.get('')
self.new_flow = {'name': name, 'actions': []}
self.flow = {
'info': {
'submit_from': {
"type": 1,
"user": "******"
},
'status': 0
},
'new_flow': {}
}
def __init__(self):
self.inactive_time = 70 # In seconds
# self.low_utilize = 60
# self.normal_utilize = 70
# self.high_utilize = 80
# self.device_service = get('device')
# self.policy_service = get('policy')
# self.policy_seq_service = get('policy_seq')
self.flow_stat_repository = get('flow_stat')
def __init__(self):
self.device_repository = get('device')
self.flow_stat_repository = get('flow_stat')
self.copied_route_repository = get('copied_route')
self.link_utilization_repository = get('link_utilization')
self.flow_routing_repository = get('flow_routing')
self.last_run = 0
self.delay = 1
self.path_finder = PathFinder(auto_update_graph=False)
self.use_port = False
self.active_paths = []
self.utilize = 85
self.reverse_path = []
self.reverse_path_link = []
self.current_flow = None
self.explorer_neighbor = []
def new(new_url):
short = str()
while True:
for x in range(6):
short += base64[randrange(2 ** 6)]
if not repository.get(short):
break
short = str()
repository.create(short, new_url)
return short
def main():
device_repository = repository.get('device')
devices = device_repository.get_all()
for device in devices:
device_repository.set_snmp_running(device['management_ip'], False)
worker = snmp_worker.SNMPWorker()
a = mp.Process(target=worker.run)
try:
a.start()
except KeyboardInterrupt:
worker.shutdown()
a.join()
def __init__(self, *tasks):
# self.max_worker = 2 # Maximum update flow worker
self.set_worker = []
self.stop_signal = False
self.device_repository = repository.get("device")
self.results_q = Queue()
self.ssh_connection = SSHConnection()
self.manager = Manager()
self.tasks = [task() for task in tasks]
self.initialTasks = []
self.pool = None
self.stop_control_q = Queue(maxsize=1)
def get_all_subnet():
device_neighbor_repository = repository.get('device_neighbor')
device_repository = repository.get('device')
devices = device_repository.get_active()
subnet_list = {}
for device in devices:
cdp_info = device_neighbor_repository.get_by_device_ip(
device.get('management_ip'))
if not cdp_info:
raise NotImplementedError(
'Please enable CDP to use this function.')
for interface in device.get('interfaces'):
ip_addr = interface.get('ipv4_address')
if ip_addr:
subnet = interface.get('subnet')
network = netaddr.IPNetwork("{}/{}".format(ip_addr, subnet))
subnet_list[str(network)] = {
'node': device.get('management_ip'),
'ip': ip_addr,
'mask': subnet
}
return subnet_list
def run_loop(device):
""" Run loop
"""
management_ip = device['management_ip']
device_repository = repository.get('device')
# logging.info("SNMP Worker: Start loop device IP %s", management_ip)
loop = asyncio.new_event_loop()
asyncio.set_event_loop(loop)
if can_uvloop:
asyncio.set_event_loop_policy(uvloop.EventLoopPolicy())
loop.run_until_complete(SNMPWorker.get_and_store(device))
device_repository.set_snmp_finish_running(management_ip)
loop.close()
def __init__(self, ssh_info, work_q, result_q, is_connect_value,
stop_signal_value, **kwargs):
super(SSHQueueWorker, self).__init__()
self.ssh_info = ssh_info
self._work_queue = work_q
self._result_q = result_q
self._is_connect = is_connect_value
self._stop_signal = stop_signal_value
if kwargs.get('max_retry_timeout'):
self._max_retry_timeout = kwargs.get('max_retry_timeout')
else:
self._max_retry_timeout = 3
self._lock = threading.Lock()
self.setName('SSH-Q-WORKER' + ssh_info['ip'])
self.device_repository = repository.get("device")
async def process_route(device_id, host, community, port):
copied_route_repository = repository.get('copied_route')
routes = await snmp_async.get_routes(host, community, port)
if routes is None:
logging.info("SNMP (Process route): host {} is down".format(host))
return None
for route in routes:
ip = IPNetwork("{}/{}".format(route['dst'], route['mask']))
# /32
if ip.size == 1:
start_ip = ip.first
end_ip = ip.first
# /31
elif ip.size == 2:
start_ip = ip.first
end_ip = ip.last
# Other
else:
start_ip = ip.first + 1
end_ip = ip.last - 1
route['start_ip'] = start_ip
route['end_ip'] = end_ip
# route['device_ip'] = route['device_ip']
del route['device_ip']
route['device_id'] = device_id
# Set update time
route['updated_at'] = time.time()
# Clear old routes
# copied_route_repository.delete_all_by_device_ip(host)
copied_route_repository.delete_all_by_device_id(device_id)
# Insert net routes
copied_route_repository.model.insert_many(routes)
return True
def get(id):
"""
Gets a trace.
@param id: Trace id.
@return Trace.
@code 404: Trace doesn't exist.
@code 400: The id is not valid.
"""
trace = None
try:
trace = repository.get(int(id))
except repository.TraceNotFound as e:
raise bottle.HTTPError(status=404)
except ValueError as ve:
raise bottle.HTTPError(status=400)
resp = bottle.response
resp.body = str(trace)
return resp
def add_handle(self):
"""
:return:
"""
print("Add device to topology")
print("If you want to cancel this task. Please enter `exit`")
device_info = {}
ssh_info = {}
snmp_info = {}
try:
device_info['type'] = self._get_device_type()
device_info['ip'] = self._get_ip()
ssh_info['username'] = self._get_ssh_username()
ssh_info['password'] = self._get_ssh_password()
ssh_info['secret'] = self._get_ssh_secret()
ssh_info['port'] = self._get_ssh_port()
snmp_info['version'] = self._get_snmp_version()
snmp_info['community'] = self._get_snmp_community()
snmp_info['port'] = self._get_snmp_port()
logging.debug("Device info %s", device_info)
logging.debug("SSH info %s", ssh_info)
logging.debug("SNMP info %s", snmp_info)
device_repository = repository.get('device')
device_repository.add_device({
'management_ip': device_info['ip'],
'status': DeviceRepository.STATUS_WAIT_UPDATE,
'type': device_info['type'],
'ssh_info': ssh_info,
'snmp_info': snmp_info,
})
# Clear prompt
except KeyboardInterrupt:
print("Interrupt add device.")
def run(self):
num_worker = sdn_utils.get_snmp_num_worker()
executor = ProcessPoolExecutor(num_worker)
device_repository = repository.get('device')
while not self.stop_signal:
self.running = list(
filter(lambda x: x.done() is False, self.running))
active_device = device_repository.get_by_snmp_can_run(
self.loop_time)
for device in active_device:
# device_service = services.get_service('device')
management_ip = device['management_ip']
if device_repository.snmp_is_running(management_ip):
return
# Mark device SNMP is running
device_repository.set_snmp_running(device['management_ip'],
True)
# Submit
self.running.append(
executor.submit(SNMPWorker.run_loop, device))
time.sleep(1)
def create_networkx_graph(devices, add_link=True):
""" Create graph than using NetworkX library
"""
networkx = nx.Graph()
# networkx = nx.MultiGraph()
# networkx.add
device_neighbor_repository = repository.get('device_neighbor')
device_repository = repository.get('device')
link_utilization_repository = repository.get('link_utilization')
link_list = []
for src_device in devices:
cdp = device_neighbor_repository.get_by_device_id(src_device['_id'])
if cdp is not None:
cdp_neighbor = cdp.get('neighbor')
# current_device = src_device
# current_device = device_service.get_device(src_device['management_ip'])
for neighbor in cdp_neighbor:
# check device is exist in topology
# If not continue to next cdp device
neighbor_device = device_repository.find_by_if_ip(
neighbor.get('ip_addr'))
if neighbor_device is None:
continue
if_index = -1
neighbor_if_speed = 0
neighbor_in_use = 0
neighbor_out_use = 0
for neighbor_interface in neighbor_device.get('interfaces'):
# Default is -1, -2 because if no IP is not match
if neighbor_interface.get('ipv4_address',
-1) == neighbor.get(
'ip_addr', -2):
if_index = neighbor_interface['index']
neighbor_if_speed = neighbor_interface['speed']
neighbor_in_use = neighbor_interface[
'bw_in_usage_persec']
neighbor_out_use = neighbor_interface[
'bw_out_usage_persec']
break
# not have a neighbor
if if_index == -1:
continue
if_speed = 0
current_if_speed = 0
can_find_interface = False
src_in_use = 0
src_out_use = 0
for interface in src_device['interfaces']:
if interface.get('index') == neighbor['local_ifindex']:
current = interface
if_speed = min(neighbor_if_speed,
interface.get('speed'))
current_if_speed = interface.get('speed')
src_in_use = interface.get('bw_in_usage_persec')
src_out_use = interface.get('bw_out_usage_persec')
can_find_interface = True
break
if not can_find_interface:
raise ValueError(
"Can't find current device interface ip %s" %
src_device['management_ip'])
if netaddr.IPAddress(
current['ipv4_address']) < netaddr.IPAddress(
neighbor['ip_addr']):
src_if_ip = current['ipv4_address']
src_port = current['description']
src_if_index = current['index']
# src_usage = current['bw_in_usage_octets']
dst_if_ip = neighbor['ip_addr']
dst_port = neighbor['port']
dst_if_index = neighbor_interface['index']
else:
dst_if_ip = current['ipv4_address']
dst_port = current['description']
dst_if_index = current['index']
src_if_ip = neighbor['ip_addr']
src_port = neighbor['port']
src_if_index = neighbor_interface['index']
src_in_use, neighbor_in_use = neighbor_in_use, src_in_use
src_out_use, neighbor_out_use = neighbor_out_use, src_out_use
# src_usage = current['bw_in_usage_octets']
logging.info("Added edge: " + src_device['management_ip'] +
" - " + neighbor_device['management_ip'] +
" nb speed: " + str(neighbor_if_speed) +
" my speed: " + str(current_if_speed) +
" src_ip: " + src_if_ip + " src_port: " +
src_port)
# TODO implement multiple edges between nodes
try:
edge = networkx.edges[(src_device['management_ip'],
neighbor_device['management_ip'])]
except KeyError:
edge = None
if edge:
links = edge['links']
else:
links = {}
# if netaddr.IPAddress(src_device['management_ip']) < netaddr.IPAddress(neighbor_device['management_ip']):
if netaddr.IPAddress(
current['ipv4_address']) < netaddr.IPAddress(
neighbor['ip_addr']):
src_node_ip = src_device['management_ip']
src_node_id = src_device['_id']
src_node_hostname = src_device[
'name'] # Todo change name to hostname
dst_node_ip = neighbor_device['management_ip']
dst_node_id = neighbor_device['_id']
dst_node_hostname = neighbor_device[
'name'] # Todo change name to hostname
else:
dst_node_hostname = src_device[
'name'] # Todo change name to hostname
dst_node_ip = src_device['management_ip']
dst_node_id = src_device['_id']
src_node_ip = neighbor_device['management_ip']
src_node_id = neighbor_device['_id']
src_node_hostname = neighbor_device[
'name'] # Todo change name to hostname
link_info = {
'src_node_id': src_node_id,
'src_node_ip': src_node_ip,
'src_ip': src_if_ip,
'src_if_ip': src_if_ip,
'src_node_hostname': src_node_hostname,
'src_port': src_port,
'src_if_index': src_if_index,
'src_in_use': src_in_use,
'src_out_use': src_out_use,
# 'src_usage': src_usage,
'dst_node_id': dst_node_id,
'dst_node_ip': dst_node_ip,
'dst_ip': dst_if_ip,
'dst_if_ip': dst_if_ip,
'dst_node_hostname': dst_node_hostname,
'dst_port': dst_port,
'dst_if_index': dst_if_index,
'dst_in_use': neighbor_in_use,
'dst_out_use': neighbor_out_use,
'link_min_speed': if_speed
}
link_id = sdn_utils.generate_link_id(src_if_ip, dst_if_ip)
links[link_id] = link_info
networkx.add_edge(src_device['management_ip'],
neighbor_device['management_ip'],
links=links)
# logging.info(networkx.edges)
if add_link:
link_list.append(link_info)
else:
# TODO
# raise NotImplementedError("SNMP currently is not support")
return networkx
if add_link:
link_utilization_repository.add_links(link_list)
return networkx
async def process_system(device_id, host, community, port):
device_repository = repository.get('device')
system_info = await snmp_async.get_system_info(host, community, port)
interfaces = await snmp_async.get_interfaces(host, community, port)
interface_update_time = time.time()
ip_addrs = await snmp_async.get_ip_addr(host, community, port)
if not system_info:
logging.info("SNMP (Process system [sys info]): host {} is down".format(host))
return None
if not ip_addrs:
logging.info("SNMP (Process system [ip addr]): host {} is down".format(host))
return None
if not interfaces:
logging.info("SNMP (Process system [interface]): host {} is down".format(host))
return None
# Todo optimize this
for if_index, interface in enumerate(interfaces):
for ip_index, ip_addr in enumerate(ip_addrs):
if interface['index'] == ip_addr['if_index']:
interface['ipv4_address'] = ip_addr['ipv4_address']
interface['subnet'] = ip_addr['subnet']
break
my_device = device_repository.get_device_by_mgmt_ip(host)
diff_interface_update_time = interface_update_time - my_device.get('interfaces_update_time', 0)
diff_interface_update_time *= 100
try:
if my_device.get('interfaces'):
for if_index, interface in enumerate(interfaces):
for my_interface in my_device['interfaces']:
if interface['description'] == my_interface['description']:
# In
in_octets = interface['in_octets'] - my_interface['in_octets']
# in_in_time = system_info['uptime'] - my_device['uptime']
bw_in_usage_percent = sdn_utils.cal_bw_usage_percent(
in_octets,
interface['speed'],
diff_interface_update_time)
# Out
out_octets = interface['out_octets'] - my_interface['out_octets']
# out_in_time = system_info['uptime'] - my_device['uptime']
bw_out_usage_percent = sdn_utils.cal_bw_usage_percent(
out_octets,
interface['speed'],
diff_interface_update_time)
# Add information
interface['bw_in_usage_octets'] = in_octets
interface['bw_in_usage_persec'] = sdn_utils.bandwidth_usage_percent_to_bit(interface['speed'],
bw_in_usage_percent)
interface['bw_in_usage_percent'] = bw_in_usage_percent
interface['bw_out_usage_octets'] = out_octets
# interface['bw_out_usage_persec'] = (out_octets / out_in_time) * 8
interface['bw_out_usage_persec'] = sdn_utils.bandwidth_usage_percent_to_bit(interface['speed'],
bw_out_usage_percent)
interface['bw_out_usage_percent'] = bw_out_usage_percent
interface['bw_usage_update'] = time.time()
if interface.get('ipv4_address'):
ip = IPNetwork("{}/{}".format(interface['ipv4_address'], interface['subnet']))
if ip.size == 1:
start_ip = ip.first
end_ip = ip.first
elif ip.size == 2:
start_ip = ip.first
end_ip = ip.last
else:
start_ip = ip.first
end_ip = ip.last
interface['start_ip'] = start_ip
interface['end_ip'] = end_ip
break
except Exception as e:
logging.info("Except: %s", e)
system_info['interfaces'] = interfaces
system_info['interfaces_update_time'] = interface_update_time
# Set update time
system_info['updated_at'] = time.time()
# Update device info
device_repository.set(host, system_info)
return True
def redirect(key):
return repository.get(key)
def __init__(self):
self.device_repository = get('device')
self.flow_routing_repository = get('flow_routing')
self.used_flow_id_repository = get('used_flow_id')
self.snmp_fetch = SNMPFetch()