def is_created(self):
"""Attempt to ensure network has been created successfully."""
subst = self.subst_dict()
if not util.check_for_interface(subst['vx_bridge'], up=True):
LOG.withObj(self).warning('%s is not up' % subst['vx_bridge'])
return False
return True
def delete(self):
subst = self.subst_dict()
# Cleanup local node
with lockutils.lock('sf_net_%s' % self.uuid,
external=True,
lock_path='/tmp/'):
if util.check_for_interface(subst['vx_bridge']):
with util.RecordedOperation('delete vxlan bridge', self) as _:
processutils.execute('ip link delete %(vx_bridge)s' %
subst,
shell=True)
if util.check_for_interface(subst['vx_interface']):
with util.RecordedOperation('delete vxlan interface',
self) as _:
processutils.execute('ip link delete %(vx_interface)s' %
subst,
shell=True)
# If this is the network node do additional cleanup
if config.parsed.get('NODE_IP') == config.parsed.get(
'NETWORK_NODE_IP'):
if util.check_for_interface(subst['vx_veth_outer']):
with util.RecordedOperation('delete router veth', self) as _:
processutils.execute('ip link delete %(vx_veth_outer)s' %
subst,
shell=True)
if util.check_for_interface(subst['physical_veth_outer']):
with util.RecordedOperation('delete physical veth', self) as _:
processutils.execute(
'ip link delete %(physical_veth_outer)s' % subst,
shell=True)
if os.path.exists('/var/run/netns/%(namespace)s' % subst):
with util.RecordedOperation('delete netns', self) as _:
processutils.execute('ip netns del %(namespace)s' % subst,
shell=True)
if self.floating_gateway:
floatnet = from_db('floating')
floatnet.ipmanager.release(self.floating_gateway)
floatnet.persist_ipmanager()
def delete(self):
subst = self.subst_dict()
LOG.withFields(subst).debug('net.delete()')
# Cleanup local node
with db.get_object_lock(self, ttl=120, op='Network delete'):
if util.check_for_interface(subst['vx_bridge']):
with util.RecordedOperation('delete vxlan bridge', self):
util.execute(None, 'ip link delete %(vx_bridge)s' % subst)
if util.check_for_interface(subst['vx_interface']):
with util.RecordedOperation('delete vxlan interface', self):
util.execute(None,
'ip link delete %(vx_interface)s' % subst)
# If this is the network node do additional cleanup
if util.is_network_node():
if util.check_for_interface(subst['vx_veth_outer']):
with util.RecordedOperation('delete router veth', self):
util.execute(
None, 'ip link delete %(vx_veth_outer)s' % subst)
if util.check_for_interface(subst['physical_veth_outer']):
with util.RecordedOperation('delete physical veth', self):
util.execute(
None,
'ip link delete %(physical_veth_outer)s' % subst)
if os.path.exists('/var/run/netns/%(netns)s' % subst):
with util.RecordedOperation('delete netns', self):
util.execute(None, 'ip netns del %(netns)s' % subst)
if self.db_entry['floating_gateway']:
with db.get_lock('ipmanager',
None,
'floating',
ttl=120,
op='Network delete'):
ipm = db.get_ipmanager('floating')
ipm.release(self.db_entry['floating_gateway'])
db.persist_ipmanager('floating', ipm.save())
def delete(self):
subst = self.subst_dict()
# Cleanup local node
with db.get_lock('sf/net/%s' % self.uuid, ttl=120) as _:
if util.check_for_interface(subst['vx_bridge']):
with util.RecordedOperation('delete vxlan bridge', self) as _:
processutils.execute('ip link delete %(vx_bridge)s' % subst,
shell=True)
if util.check_for_interface(subst['vx_interface']):
with util.RecordedOperation('delete vxlan interface', self) as _:
processutils.execute('ip link delete %(vx_interface)s' % subst,
shell=True)
# If this is the network node do additional cleanup
if config.parsed.get('NODE_IP') == config.parsed.get('NETWORK_NODE_IP'):
if util.check_for_interface(subst['vx_veth_outer']):
with util.RecordedOperation('delete router veth', self) as _:
processutils.execute('ip link delete %(vx_veth_outer)s' % subst,
shell=True)
if util.check_for_interface(subst['physical_veth_outer']):
with util.RecordedOperation('delete physical veth', self) as _:
processutils.execute('ip link delete %(physical_veth_outer)s' % subst,
shell=True)
if os.path.exists('/var/run/netns/%(netns)s' % subst):
with util.RecordedOperation('delete netns', self) as _:
processutils.execute('ip netns del %(netns)s' % subst,
shell=True)
if self.floating_gateway:
with db.get_lock('sf/ipmanager/floating', ttl=120) as _:
ipm = db.get_ipmanager('floating')
ipm.release(self.floating_gateway)
db.persist_ipmanager('floating', ipm.save())
def create(self):
subst = self.subst_dict()
with db.get_lock('sf/net/%s' % self.uuid, ttl=120) as _:
if not util.check_for_interface(subst['vx_interface']):
with util.RecordedOperation('create vxlan interface', self) as _:
processutils.execute(
'ip link add %(vx_interface)s type vxlan id %(vx_id)s '
'dev %(physical_interface)s dstport 0'
% subst, shell=True)
processutils.execute(
'sysctl -w net.ipv4.conf.%(vx_interface)s.arp_notify=1' % subst,
shell=True)
if not util.check_for_interface(subst['vx_bridge']):
with util.RecordedOperation('create vxlan bridge', self) as _:
processutils.execute(
'ip link add %(vx_bridge)s type bridge' % subst, shell=True)
processutils.execute(
'ip link set %(vx_interface)s master %(vx_bridge)s' % subst,
shell=True)
processutils.execute(
'ip link set %(vx_interface)s up' % subst, shell=True)
processutils.execute(
'ip link set %(vx_bridge)s up' % subst, shell=True)
processutils.execute(
'sysctl -w net.ipv4.conf.%(vx_bridge)s.arp_notify=1' % subst,
shell=True)
processutils.execute(
'brctl setfd %(vx_bridge)s 0' % subst, shell=True)
processutils.execute(
'brctl stp %(vx_bridge)s off' % subst, shell=True)
processutils.execute(
'brctl setageing %(vx_bridge)s 0' % subst, shell=True)
if config.parsed.get('NODE_IP') == config.parsed.get('NETWORK_NODE_IP'):
if not os.path.exists('/var/run/netns/%(netns)s' % subst):
with util.RecordedOperation('create netns', self) as _:
processutils.execute(
'ip netns add %(netns)s' % subst, shell=True)
if not util.check_for_interface(subst['vx_veth_outer']):
with util.RecordedOperation('create router veth', self) as _:
processutils.execute(
'ip link add %(vx_veth_outer)s type veth peer name %(vx_veth_inner)s' % subst,
shell=True)
processutils.execute(
'ip link set %(vx_veth_inner)s netns %(netns)s' % subst, shell=True)
processutils.execute(
'brctl addif %(vx_bridge)s %(vx_veth_outer)s' % subst, shell=True)
processutils.execute(
'ip link set %(vx_veth_outer)s up' % subst, shell=True)
processutils.execute(
'%(in_netns)s ip link set %(vx_veth_inner)s up' % subst, shell=True)
processutils.execute(
'%(in_netns)s ip addr add %(router)s/%(netmask)s dev %(vx_veth_inner)s' % subst,
shell=True)
if not util.check_for_interface(subst['physical_veth_outer']):
with util.RecordedOperation('create physical veth', self) as _:
processutils.execute(
'ip link add %(physical_veth_outer)s type veth peer name '
'%(physical_veth_inner)s' % subst,
shell=True)
processutils.execute(
'brctl addif %(physical_bridge)s %(physical_veth_outer)s' % subst,
shell=True)
processutils.execute(
'ip link set %(physical_veth_outer)s up' % subst, shell=True)
processutils.execute(
'ip link set %(physical_veth_inner)s netns %(netns)s' % subst,
shell=True)
self.deploy_nat()
self.update_dhcp()
else:
admin_token = util.get_api_token(
'http://%s:%d' % (config.parsed.get('NETWORK_NODE_IP'),
config.parsed.get('API_PORT')),
namespace='system')
requests.request(
'put',
('http://%s:%d/deploy_network_node'
% (config.parsed.get('NETWORK_NODE_IP'),
config.parsed.get('API_PORT'))),
data=json.dumps({'uuid': self.uuid}),
headers={'Authorization': admin_token,
'User-Agent': util.get_user_agent()})
def create(self):
subst = self.subst_dict()
with db.get_object_lock(self, ttl=120, op='Network create'):
# Ensure network was not deleted whilst waiting for the lock.
if self.is_dead():
raise DeadNetwork('network=%s' % self)
if not util.check_for_interface(subst['vx_interface']):
with util.RecordedOperation('create vxlan interface', self):
util.create_interface(
subst['vx_interface'], 'vxlan',
'id %(vx_id)s dev %(physical_interface)s dstport 0' %
subst)
util.execute(
None, 'sysctl -w net.ipv4.conf.'
'%(vx_interface)s.arp_notify=1' % subst)
if not util.check_for_interface(subst['vx_bridge']):
with util.RecordedOperation('create vxlan bridge', self):
util.create_interface(subst['vx_bridge'], 'bridge', '')
util.execute(
None, 'ip link set %(vx_interface)s '
'master %(vx_bridge)s' % subst)
util.execute(None,
'ip link set %(vx_interface)s up' % subst)
util.execute(None, 'ip link set %(vx_bridge)s up' % subst)
util.execute(
None, 'sysctl -w net.ipv4.conf.'
'%(vx_bridge)s.arp_notify=1' % subst)
util.execute(None, 'brctl setfd %(vx_bridge)s 0' % subst)
util.execute(None, 'brctl stp %(vx_bridge)s off' % subst)
util.execute(None,
'brctl setageing %(vx_bridge)s 0' % subst)
if util.is_network_node():
if not os.path.exists('/var/run/netns/%(netns)s' % subst):
with util.RecordedOperation('create netns', self):
util.execute(None, 'ip netns add %(netns)s' % subst)
if not util.check_for_interface(subst['vx_veth_outer']):
with util.RecordedOperation('create router veth', self):
util.create_interface(
subst['vx_veth_outer'], 'veth',
'peer name %(vx_veth_inner)s' % subst)
util.execute(
None, 'ip link set %(vx_veth_inner)s netns %(netns)s' %
subst)
util.execute(
None,
'brctl addif %(vx_bridge)s %(vx_veth_outer)s' % subst)
util.execute(None,
'ip link set %(vx_veth_outer)s up' % subst)
util.execute(
None, '%(in_netns)s ip link set %(vx_veth_inner)s up' %
subst)
util.execute(
None,
'%(in_netns)s ip addr add %(router)s/%(netmask)s '
'dev %(vx_veth_inner)s' % subst)
if not util.check_for_interface(subst['physical_veth_outer']):
with util.RecordedOperation('create physical veth', self):
util.create_interface(
subst['physical_veth_outer'], 'veth',
'peer name %(physical_veth_inner)s' % subst)
util.execute(
None, 'brctl addif %(physical_bridge)s '
'%(physical_veth_outer)s' % subst)
util.execute(
None, 'ip link set %(physical_veth_outer)s up' % subst)
util.execute(
None, 'ip link set %(physical_veth_inner)s '
'netns %(netns)s' % subst)
self.deploy_nat()
self.update_dhcp()
else:
db.enqueue('networknode', DeployNetworkTask(self.db_entry['uuid']))
db.add_event('network', self.db_entry['uuid'], 'deploy',
'enqueued', None, None)
def main():
global DAEMON_IMPLEMENTATIONS
global DAEMON_PIDS
setproctitle.setproctitle(daemon.process_name('main'))
# Log configuration on startup
for key, value in config.dict().items():
LOG.info('Configuration item %s = %s' % (key, value))
daemon.set_log_level(LOG, 'main')
# Check in early and often, also reset processing queue items
db.clear_stale_locks()
db.see_this_node()
db.restart_queues()
def _start_daemon(d):
pid = os.fork()
if pid == 0:
DAEMON_IMPLEMENTATIONS[d].Monitor(d).run()
DAEMON_PIDS[pid] = d
LOG.withField('pid', pid).info('Started %s' % d)
# Resource usage publisher, we need this early because scheduling decisions
# might happen quite early on.
_start_daemon('resources')
# If I am the network node, I need some setup
if util.is_network_node():
# Bootstrap the floating network in the Networks table
floating_network = db.get_network('floating')
if not floating_network:
db.create_floating_network(config.get('FLOATING_NETWORK'))
floating_network = net.from_db('floating')
subst = {
'physical_bridge':
util.get_safe_interface_name('phy-br-%s' %
config.get('NODE_EGRESS_NIC')),
'physical_nic':
config.get('NODE_EGRESS_NIC')
}
if not util.check_for_interface(subst['physical_bridge']):
# NOTE(mikal): Adding the physical interface to the physical bridge
# is considered outside the scope of the orchestration software as
# it will cause the node to lose network connectivity. So instead
# all we do is create a bridge if it doesn't exist and the wire
# everything up to it. We can do egress NAT in that state, even if
# floating IPs don't work.
with util.RecordedOperation('create physical bridge', None):
# No locking as read only
ipm = db.get_ipmanager('floating')
subst['master_float'] = ipm.get_address_at_index(1)
subst['netmask'] = ipm.netmask
util.create_interface(subst['physical_bridge'], 'bridge', '')
util.execute(None,
'ip link set %(physical_bridge)s up' % subst)
util.execute(
None, 'ip addr add %(master_float)s/%(netmask)s '
'dev %(physical_bridge)s' % subst)
util.execute(
None, 'iptables -A FORWARD -o %(physical_nic)s '
'-i %(physical_bridge)s -j ACCEPT' % subst)
util.execute(
None, 'iptables -A FORWARD -i %(physical_nic)s '
'-o %(physical_bridge)s -j ACCEPT' % subst)
util.execute(
None, 'iptables -t nat -A POSTROUTING '
'-o %(physical_nic)s -j MASQUERADE' % subst)
def _audit_daemons():
running_daemons = []
for pid in DAEMON_PIDS:
running_daemons.append(DAEMON_PIDS[pid])
for d in DAEMON_IMPLEMENTATIONS:
if d not in running_daemons:
_start_daemon(d)
for d in DAEMON_PIDS:
if not psutil.pid_exists(d):
LOG.warning('%s pid is missing, restarting' % DAEMON_PIDS[d])
_start_daemon(DAEMON_PIDS[d])
_audit_daemons()
restore_instances()
while True:
time.sleep(10)
wpid, _ = os.waitpid(-1, os.WNOHANG)
while wpid != 0:
LOG.warning('%s died (pid %d)' %
(DAEMON_PIDS.get(wpid, 'unknown'), wpid))
del DAEMON_PIDS[wpid]
wpid, _ = os.waitpid(-1, os.WNOHANG)
_audit_daemons()
db.see_this_node()
def main():
# Log configuration on startup
for key in config.parsed.config:
LOG.info('Configuration item %s = %s' % (key, config.parsed.get(key)))
# Check in early and often
db.see_this_node()
# Resource usage publisher, we need this early because scheduling decisions
# might happen quite early on.
resource_pid = os.fork()
if resource_pid == 0:
resource_daemon.monitor().run()
# If I am the network node, I need some setup
if config.parsed.get('NODE_IP') == config.parsed.get('NETWORK_NODE_IP'):
# Bootstrap the floating network in the Networks table
floating_network = db.get_network('floating')
if not floating_network:
db.create_floating_network(config.parsed.get('FLOATING_NETWORK'))
floating_network = net.from_db('floating')
subst = {
'physical_bridge':
'phy-br-%s' % config.parsed.get('NODE_EGRESS_NIC'),
'physical_nic': config.parsed.get('NODE_EGRESS_NIC')
}
if not util.check_for_interface(subst['physical_bridge']):
# NOTE(mikal): Adding the physical interface to the physical bridge
# is considered outside the scope of the orchestration software as it
# will cause the node to lose network connectivity. So instead all we
# do is create a bridge if it doesn't exist and the wire everything up
# to it. We can do egress NAT in that state, even if floating IPs
# don't work.
with util.RecordedOperation('create physical bridge',
'startup') as _:
# No locking as read only
ipm = db.get_ipmanager('floating')
subst['master_float'] = ipm.get_address_at_index(1)
subst['netmask'] = ipm.netmask
processutils.execute(
'ip link add %(physical_bridge)s type bridge' % subst,
shell=True)
processutils.execute('ip link set %(physical_bridge)s up' %
subst,
shell=True)
processutils.execute(
'ip addr add %(master_float)s/%(netmask)s dev %(physical_bridge)s'
% subst,
shell=True)
processutils.execute(
'iptables -A FORWARD -o %(physical_nic)s -i %(physical_bridge)s -j ACCEPT'
% subst,
shell=True)
processutils.execute(
'iptables -A FORWARD -i %(physical_nic)s -o %(physical_bridge)s -j ACCEPT'
% subst,
shell=True)
processutils.execute(
'iptables -t nat -A POSTROUTING -o %(physical_nic)s -j MASQUERADE'
% subst,
shell=True)
# Network mesh maintenance
net_pid = os.fork()
if net_pid == 0:
net_daemon.monitor().run()
# Old object deleter
cleaner_pid = os.fork()
if cleaner_pid == 0:
cleaner_daemon.monitor().run()
# REST API
external_api_pid = os.fork()
if external_api_pid == 0:
external_api_daemon.monitor().run()
# Triggers
trigger_pid = os.fork()
if trigger_pid == 0:
trigger_daemon.monitor().run()
setproctitle.setproctitle('sf main')
LOG.info('network monitor pid is %d' % net_pid)
LOG.info('external api pid is %d' % external_api_pid)
LOG.info('resources monitor pid is %d' % resource_pid)
LOG.info('cleaner pid is %d' % cleaner_pid)
LOG.info('trigger pid is %d' % trigger_pid)
restore_instances()
procnames = {
external_api_pid: 'external api',
net_pid: 'network monitor',
resource_pid: 'resource monitor',
cleaner_pid: 'cleaner'
}
while True:
time.sleep(10)
wpid, _ = os.waitpid(-1, os.WNOHANG)
if wpid != 0:
LOG.warning('Subprocess %d (%s) died' %
(wpid, procnames.get(wpid, 'unknown')))
def create(self):
subst = self.subst_dict()
with db.get_lock('network', None, self.uuid, ttl=120):
if not util.check_for_interface(subst['vx_interface']):
with util.RecordedOperation('create vxlan interface', self):
util.execute(
None,
'ip link add %(vx_interface)s type vxlan id %(vx_id)s '
'dev %(physical_interface)s dstport 0' % subst)
util.execute(
None,
'sysctl -w net.ipv4.conf.%(vx_interface)s.arp_notify=1'
% subst)
if not util.check_for_interface(subst['vx_bridge']):
with util.RecordedOperation('create vxlan bridge', self):
util.execute(
None, 'ip link add %(vx_bridge)s type bridge' % subst)
util.execute(
None,
'ip link set %(vx_interface)s master %(vx_bridge)s' %
subst)
util.execute(None,
'ip link set %(vx_interface)s up' % subst)
util.execute(None, 'ip link set %(vx_bridge)s up' % subst)
util.execute(
None,
'sysctl -w net.ipv4.conf.%(vx_bridge)s.arp_notify=1' %
subst)
util.execute(None, 'brctl setfd %(vx_bridge)s 0' % subst)
util.execute(None, 'brctl stp %(vx_bridge)s off' % subst)
util.execute(None,
'brctl setageing %(vx_bridge)s 0' % subst)
if util.is_network_node():
if not os.path.exists('/var/run/netns/%(netns)s' % subst):
with util.RecordedOperation('create netns', self):
util.execute(None, 'ip netns add %(netns)s' % subst)
if not util.check_for_interface(subst['vx_veth_outer']):
with util.RecordedOperation('create router veth', self):
util.execute(
None,
'ip link add %(vx_veth_outer)s type veth peer name %(vx_veth_inner)s'
% subst)
util.execute(
None, 'ip link set %(vx_veth_inner)s netns %(netns)s' %
subst)
util.execute(
None,
'brctl addif %(vx_bridge)s %(vx_veth_outer)s' % subst)
util.execute(None,
'ip link set %(vx_veth_outer)s up' % subst)
util.execute(
None, '%(in_netns)s ip link set %(vx_veth_inner)s up' %
subst)
util.execute(
None,
'%(in_netns)s ip addr add %(router)s/%(netmask)s dev %(vx_veth_inner)s'
% subst)
if not util.check_for_interface(subst['physical_veth_outer']):
with util.RecordedOperation('create physical veth', self):
util.execute(
None,
'ip link add %(physical_veth_outer)s type veth peer name '
'%(physical_veth_inner)s' % subst)
util.execute(
None,
'brctl addif %(physical_bridge)s %(physical_veth_outer)s'
% subst)
util.execute(
None, 'ip link set %(physical_veth_outer)s up' % subst)
util.execute(
None,
'ip link set %(physical_veth_inner)s netns %(netns)s' %
subst)
self.deploy_nat()
self.update_dhcp()
else:
db.enqueue('networknode', {
'type': 'deploy',
'network_uuid': self.uuid
})
db.add_event('network', self.uuid, 'deploy', 'enqueued', None,
None)
def test_check_for_interface_present_interface(self, mock_execute):
found = util.check_for_interface('eth0')
self.assertEqual(True, found)
mock_execute.assert_called_with('ip link show eth0',
check_exit_code=[0, 1], shell=True)
def test_check_for_interface_missing_interface(self, mock_execute):
found = util.check_for_interface('banana0')
self.assertEqual(False, found)
mock_execute.assert_called_with('ip link show banana0',
check_exit_code=[0, 1], shell=True)
