def sync_pods(self, pods):
expected_logical_ports = set()
pods = pods.get('items', [])
for pod in pods:
pod_name = pod['metadata']['name']
namespace = pod['metadata']['namespace']
logical_port = "%s_%s" % (namespace, pod_name)
expected_logical_ports.add(logical_port)
try:
existing_logical_ports = ovn_nbctl(
"--data=bare", "--no-heading",
"--columns=name", "find",
"logical_switch_port",
"external_id:pod=true").split()
existing_logical_ports = set(existing_logical_ports)
except Exception as e:
vlog.err("sync_pods: find failed %s" % (str(e)))
return
for logical_port in existing_logical_ports - expected_logical_ports:
try:
ovn_nbctl("--if-exists", "lsp-del", logical_port)
except Exception as e:
vlog.err("sync_pods: failed to delete logical_port %s"
% (logical_port))
continue
vlog.info("sync_pods: Deleted logical port %s"
% (logical_port))
def add_endpoint(self, event):
endpoint_data = event.metadata
service_name = endpoint_data['metadata']['name']
namespace = endpoint_data['metadata']['namespace']
ips = endpoint_data.get('custom', {}).get('ips', [])
vlog.dbg("received endpoint data %s" % (endpoint_data))
cache_key = "%s_%s" % (namespace, service_name)
cached_service = self.service_cache.get(cache_key, {})
if cached_service:
service_data = cached_service
else:
try:
response_json = kubernetes.get_service(
variables.K8S_API_SERVER, namespace, service_name)
except exceptions.NotFound:
vlog.dbg("No service found for endpoint %s " % service_name)
return
except Exception as e:
vlog.err("add_endpoint: k8s get service (%s)" % (str(e)))
return
service_data = response_json
service_type = service_data['spec'].get('type')
if service_type != "ClusterIP" and service_type != "NodePort":
return
self._update_vip(service_data, ips)
def open(name):
"""Attempts to start listening for remote stream connections. 'name'
is a connection name in the form "TYPE:ARGS", where TYPE is an passive
stream class's name and ARGS are stream class-specific. Currently the
only supported TYPE is "punix".
Returns (error, pstream): on success 'error' is 0 and 'pstream' is the
new PassiveStream, on failure 'error' is a positive errno value and
'pstream' is None."""
if not PassiveStream.is_valid_name(name):
return errno.EAFNOSUPPORT, None
bind_path = name[6:]
if name.startswith("punix:"):
bind_path = ovs.util.abs_file_name(ovs.dirs.RUNDIR, bind_path)
error, sock = ovs.socket_util.make_unix_socket(socket.SOCK_STREAM,
True, bind_path, None)
if error:
return error, None
try:
sock.listen(10)
except socket.error as e:
vlog.err("%s: listen: %s" % (name, os.strerror(e.error)))
sock.close()
return e.error, None
return 0, PassiveStream(sock, name, bind_path)
def _create_load_balancer_vip(self, load_balancer, service_ip, ips, port,
target_port, protocol):
# With service_ip:port as a VIP, create an entry in 'load_balancer'
vlog.dbg(
"received event to create/modify load_balancer (%s) vip "
"service_ip=%s, ips=%s, port=%s, target_port=%s, protocol=%s" %
(load_balancer, service_ip, ips, port, target_port, protocol))
if not port or not target_port or not protocol or not load_balancer:
return
# key is of the form "IP:port" (with quotes around)
key = "\"" + service_ip + ":" + str(port) + "\""
if not ips:
try:
ovn_nbctl("remove", "load_balancer", load_balancer, "vips",
key)
except Exception as e:
vlog.err("_create_load_balancer_vip remove: (%s)" % (str(e)))
return
# target is of the form "IP1:port, IP2:port, IP3:port"
target_endpoints = ",".join(
["%s:%s" % (ip, target_port) for ip in ips])
target = "\"" + target_endpoints + "\""
try:
ovn_nbctl("set", "load_balancer", load_balancer,
"vips:" + key + "=" + target)
except Exception as e:
vlog.err("_create_load_balancer_vip add: (%s)" % (str(e)))
def sync_pods(self, pods):
expected_logical_ports = set()
pods = pods.get('items', [])
for pod in pods:
pod_name = pod['metadata']['name']
namespace = pod['metadata']['namespace']
logical_port = "%s_%s" % (namespace, pod_name)
annotations = pod['metadata']['annotations']
expected_logical_ports.add(logical_port)
# We should sync the container port names as there are no
# guarantees that a endpoint creation event will become after
# all the pods creation events.
ip_address = self._get_ip_address_from_annotations(annotations)
if ip_address:
self._add_k8s_l4_port_name_cache(pod, ip_address)
try:
existing_logical_ports = ovn_nbctl(
"--data=bare", "--no-heading", "--columns=name", "find",
"logical_switch_port", "external-ids:pod=true").split()
existing_logical_ports = set(existing_logical_ports)
except Exception as e:
vlog.err("sync_pods: find failed %s" % (str(e)))
return
for logical_port in existing_logical_ports - expected_logical_ports:
try:
ovn_nbctl("--if-exists", "lsp-del", logical_port)
except Exception as e:
vlog.err("sync_pods: failed to delete logical_port %s" %
(logical_port))
continue
vlog.info("sync_pods: Deleted logical port %s" % (logical_port))
def log_event(name, *arg):
found = 0
for i in range(len(content)):
if name == content[i][EV_NAME]:
# Found the event in list!
ev_id = str(content[i][EV_ID])
severity = content[i][EV_SEVERITY]
desc = content[i][EV_DESCRIPTION]
categ = content[i][EV_CATEGORY]
if len(arg):
desc = replace_str(arg, desc)
found = 1
break
if found is NOT_FOUND:
# This means supplied event name is not there in YAML, so return.
vlog.err("Event not Found")
return FAIL
mesg = 'ops-evt|' + ev_id + '|' + severity + '|' + desc
daemon_name = str(sys.argv[0])
if "/" in daemon_name:
daemon_name = daemon_name.split("/")[-1]
journal.send(mesg,
MESSAGE_ID='50c0fa81c2a545ec982a54293f1b1945',
PRIORITY=severity,
OPS_EVENT_ID=ev_id,
OPS_EVENT_CATEGORY=categ,
SYSLOG_IDENTIFIER=daemon_name)
def update_mgmt_intf_status_hostname(hostname):
global idl
status_data = {}
for ovs_rec in idl.tables[SYSTEM_TABLE].rows.itervalues():
if ovs_rec.mgmt_intf_status:
status_data = ovs_rec.mgmt_intf_status
break
dhcp_hostname = status_data.get(MGMT_INTF_KEY_DHCP_HOSTNAME,
MGMT_INTF_NULL_VAL)
if dhcp_hostname != hostname:
if hostname != MGMT_INTF_NULL_VAL:
status_data[MGMT_INTF_KEY_DHCP_HOSTNAME] = hostname
else:
del status_data[MGMT_INTF_KEY_DHCP_HOSTNAME]
# create the transaction
txn = ovs.db.idl.Transaction(idl)
setattr(ovs_rec, "mgmt_intf_status", status_data)
status = txn.commit_block()
if status != "success" and status != "unchanged":
vlog.err("Updating DHCP hostname status column failed \
with status %s" % (status))
return False
return True
def open(name):
"""Attempts to start listening for remote stream connections. 'name'
is a connection name in the form "TYPE:ARGS", where TYPE is an passive
stream class's name and ARGS are stream class-specific. Currently the
only supported TYPE is "punix".
Returns (error, pstream): on success 'error' is 0 and 'pstream' is the
new PassiveStream, on failure 'error' is a positive errno value and
'pstream' is None."""
if not PassiveStream.is_valid_name(name):
return errno.EAFNOSUPPORT, None
bind_path = name[6:]
if name.startswith("punix:"):
bind_path = ovs.util.abs_file_name(ovs.dirs.RUNDIR, bind_path)
error, sock = ovs.socket_util.make_unix_socket(socket.SOCK_STREAM,
True, bind_path, None)
if error:
return error, None
try:
sock.listen(10)
except socket.error, e:
vlog.err("%s: listen: %s" % (name, os.strerror(e.error)))
sock.close()
return e.error, None
def add_endpoint(self, event):
endpoint_data = event.metadata
service_name = endpoint_data['metadata']['name']
namespace = endpoint_data['metadata']['namespace']
ips = endpoint_data.get('custom', {}).get('ips', [])
vlog.dbg("received endpoint data %s" % (endpoint_data))
cache_key = "%s_%s" % (namespace, service_name)
cached_service = self.service_cache.get(cache_key, {})
if cached_service:
service_data = cached_service
else:
try:
response_json = kubernetes.get_service(
variables.K8S_API_SERVER,
namespace, service_name)
except exceptions.NotFound:
vlog.dbg("No service found for endpoint %s " % service_name)
return
except Exception as e:
vlog.err("add_endpoint: k8s get service (%s)" % (str(e)))
return
service_data = response_json
service_type = service_data['spec'].get('type')
if service_type != "ClusterIP" and service_type != "NodePort":
return
self._update_vip(service_data, ips)
def sync_pods(self, pods):
expected_logical_ports = set()
pods = pods.get('items', [])
for pod in pods:
pod_name = pod['metadata']['name']
namespace = pod['metadata']['namespace']
logical_port = "%s_%s" % (namespace, pod_name)
expected_logical_ports.add(logical_port)
try:
existing_logical_ports = ovn_nbctl("--data=bare", "--no-heading",
"--columns=name", "find",
"logical_switch_port",
"external_id:pod=true").split()
existing_logical_ports = set(existing_logical_ports)
except Exception as e:
vlog.err("sync_pods: find failed %s" % (str(e)))
return
for logical_port in existing_logical_ports - expected_logical_ports:
try:
ovn_nbctl("--if-exists", "lsp-del", logical_port)
except Exception as e:
vlog.err("sync_pods: failed to delete logical_port %s" %
(logical_port))
continue
vlog.info("sync_pods: Deleted logical port %s" % (logical_port))
def __wait_windows(self, poller, wait):
if self.socket is not None:
if wait == Stream.W_RECV:
mask = (win32file.FD_READ |
win32file.FD_ACCEPT |
win32file.FD_CLOSE)
event = ovs.poller.POLLIN
else:
mask = (win32file.FD_WRITE |
win32file.FD_CONNECT |
win32file.FD_CLOSE)
event = ovs.poller.POLLOUT
try:
win32file.WSAEventSelect(self.socket,
self._wevent,
mask)
except pywintypes.error as e:
vlog.err("failed to associate events with socket: %s"
% e.strerror)
poller.fd_wait(self._wevent, event)
else:
if wait == Stream.W_RECV:
if self._read:
poller.fd_wait(self._read.hEvent, ovs.poller.POLLIN)
elif wait == Stream.W_SEND:
if self._write:
poller.fd_wait(self._write.hEvent, ovs.poller.POLLOUT)
elif wait == Stream.W_CONNECT:
return
def check_connection_completion(sock):
if sys.platform == "win32":
p = ovs.poller.SelectPoll()
event = winutils.get_new_event(None, False, True, None)
# Receive notification of readiness for writing, of completed
# connection or multipoint join operation, and of socket closure.
win32file.WSAEventSelect(sock, event,
win32file.FD_WRITE |
win32file.FD_CONNECT |
win32file.FD_CLOSE)
p.register(event, ovs.poller.POLLOUT)
else:
p = ovs.poller.get_system_poll()
p.register(sock, ovs.poller.POLLOUT)
pfds = p.poll(0)
if len(pfds) == 1:
revents = pfds[0][1]
if revents & ovs.poller.POLLERR or revents & ovs.poller.POLLHUP:
try:
# The following should raise an exception.
sock.send("\0".encode(), socket.MSG_DONTWAIT)
# (Here's where we end up if it didn't.)
# XXX rate-limit
vlog.err("poll return POLLERR but send succeeded")
return errno.EPROTO
except socket.error as e:
return get_exception_errno(e)
else:
return 0
else:
return errno.EAGAIN
def __wait_windows(self, poller, wait):
if self.socket is not None:
if wait == Stream.W_RECV:
read_flags = (win32file.FD_READ |
win32file.FD_ACCEPT |
win32file.FD_CLOSE)
try:
win32file.WSAEventSelect(self.socket,
self._read.hEvent,
read_flags)
except pywintypes.error as e:
vlog.err("failed to associate events with socket: %s"
% e.strerror)
poller.fd_wait(self._read.hEvent, ovs.poller.POLLIN)
else:
write_flags = (win32file.FD_WRITE |
win32file.FD_CONNECT |
win32file.FD_CLOSE)
try:
win32file.WSAEventSelect(self.socket,
self._write.hEvent,
write_flags)
except pywintypes.error as e:
vlog.err("failed to associate events with socket: %s"
% e.strerror)
poller.fd_wait(self._write.hEvent, ovs.poller.POLLOUT)
else:
if wait == Stream.W_RECV:
if self._read:
poller.fd_wait(self._read.hEvent, ovs.poller.POLLIN)
elif wait == Stream.W_SEND:
if self._write:
poller.fd_wait(self._write.hEvent, ovs.poller.POLLOUT)
elif wait == Stream.W_CONNECT:
return
def _get_switch_gateway_ip(self, logical_switch):
cached_logical_switch = self.logical_switch_cache.get(
logical_switch, {})
if cached_logical_switch:
gateway_ip_mask = cached_logical_switch.get('gateway_ip_mask')
else:
try:
gateway_ip_mask = ovn_nbctl(
"--if-exists", "get", "logical_switch", logical_switch,
"external_ids:gateway_ip").strip('"')
except Exception as e:
vlog.err(
"_get_switch_gateway_ip: failed to get gateway_ip %s" %
(str(e)))
return (None, None)
try:
(gateway_ip, mask) = gateway_ip_mask.split('/')
except Exception as e:
vlog.err("_get_switch_gateway_ip: failed to split ip/mask %s" %
(gateway_ip_mask))
return (None, None)
if not cached_logical_switch:
self.logical_switch_cache[logical_switch] = {
'gateway_ip_mask': gateway_ip_mask
}
return (gateway_ip, mask)
def check_connection_completion(sock):
p = ovs.poller.SelectPoll()
if sys.platform == "win32":
event = winutils.get_new_event(None, False, True, None)
# Receive notification of readiness for writing, of completed
# connection or multipoint join operation, and of socket closure.
win32file.WSAEventSelect(sock, event,
win32file.FD_WRITE |
win32file.FD_CONNECT |
win32file.FD_CLOSE)
p.register(event, ovs.poller.POLLOUT)
else:
p.register(sock, ovs.poller.POLLOUT)
pfds = p.poll(0)
if len(pfds) == 1:
revents = pfds[0][1]
if revents & ovs.poller.POLLERR:
try:
# The following should raise an exception.
socket.send("\0", socket.MSG_DONTWAIT)
# (Here's where we end up if it didn't.)
# XXX rate-limit
vlog.err("poll return POLLERR but send succeeded")
return errno.EPROTO
except socket.error as e:
return get_exception_errno(e)
else:
return 0
else:
return errno.EAGAIN
def _get_switch_gateway_ip(self, logical_switch):
cached_logical_switch = self.logical_switch_cache.get(logical_switch,
{})
if cached_logical_switch:
gateway_ip_mask = cached_logical_switch.get('gateway_ip_mask')
else:
try:
gateway_ip_mask = ovn_nbctl("--if-exists", "get",
"logical_switch", logical_switch,
"external_ids:gateway_ip"
).strip('"')
except Exception as e:
vlog.err("_get_switch_gateway_ip: failed to get gateway_ip %s"
% (str(e)))
return (None, None)
try:
(gateway_ip, mask) = gateway_ip_mask.split('/')
except Exception as e:
vlog.err("_get_switch_gateway_ip: failed to split ip/mask %s"
% (gateway_ip_mask))
return (None, None)
if not cached_logical_switch:
self.logical_switch_cache[logical_switch] = {'gateway_ip_mask':
gateway_ip_mask}
return (gateway_ip, mask)
def _create_external_vip(self, namespace, external_ip, ips,
port, target_port, protocol):
# With external_ip:port as the VIP, create an entry in a gateway
# load-balancer.
# Get the gateway where we can add external_ip:port as a VIP.
physical_gateway = self._get_ovn_external_ip_gateway()
if not physical_gateway:
return
try:
# Get the load-balancer instantiated in the gateway.
external_id_key = protocol + "_lb_gateway_router"
load_balancer = ovn_nbctl("--data=bare", "--no-heading",
"--columns=_uuid", "find",
"load_balancer",
"external_ids:" + external_id_key + "=" +
physical_gateway).strip('"')
except Exception as e:
vlog.err("_create_external_vip: get failed for"
" %s (%s)" % (physical_gateway, str(e)))
return
if not load_balancer:
vlog.warn("physical gateway %s does not have a load_balancer"
% (physical_gateway))
# With external_ip:port as VIP, add an entry in 'load_balancer'.
self._create_load_balancer_vip(namespace, load_balancer, external_ip,
ips, port, target_port, protocol)
def _create_external_vip(self, external_ip, ips, port, target_port,
protocol):
# With external_ip:port as the VIP, create an entry in a gateway
# load-balancer.
# Get the gateway where we can add external_ip:port as a VIP.
physical_gateway = self._get_ovn_external_ip_gateway()
if not physical_gateway:
return
try:
# Get the load-balancer instantiated in the gateway.
external_id_key = protocol + "_lb_gateway_router"
load_balancer = ovn_nbctl(
"--data=bare", "--no-heading", "--columns=_uuid", "find",
"load_balancer", "external_ids:" + external_id_key + "=" +
physical_gateway).strip('"')
except Exception as e:
vlog.err("_create_external_vip: get failed for"
" %s (%s)" % (physical_gateway, str(e)))
return
if not load_balancer:
vlog.warn("physical gateway %s does not have a load_balancer" %
(physical_gateway))
# With external_ip:port as VIP, add an entry in 'load_balancer'.
self._create_load_balancer_vip(load_balancer, external_ip, ips, port,
target_port, protocol)
def _delete_load_balancer_vip(self, load_balancer, vip):
# Remove the 'vip' from the 'load_balancer'.
try:
ovn_nbctl("remove", "load_balancer", load_balancer, "vips", vip)
vlog.info("deleted vip %s from %s" % (vip, load_balancer))
except Exception as e:
vlog.err("_delete_load_balancer_vip: failed to remove vip %s "
" from %s (%s)" % (vip, load_balancer, str(e)))
def _delete_load_balancer_vip(self, load_balancer, vip):
# Remove the 'vip' from the 'load_balancer'.
try:
ovn_nbctl("remove", "load_balancer", load_balancer, "vips", vip)
vlog.info("deleted vip %s from %s" % (vip, load_balancer))
except Exception as e:
vlog.err("_delete_load_balancer_vip: failed to remove vip %s "
" from %s (%s)" % (vip, load_balancer, str(e)))
def _create_load_balancer_vip(self, namespace, load_balancer, service_ip,
ips, port, target_port, protocol):
# With service_ip:port as a VIP, create an entry in 'load_balancer'
vlog.dbg("received event to create/modify load_balancer (%s) vip "
"service_ip=%s, ips=%s, port=%s, target_port=%s, protocol=%s"
% (load_balancer, service_ip, ips, port, target_port,
protocol))
if not port or not target_port or not protocol or not load_balancer:
return
# key is of the form "IP:port" (with quotes around)
key = "\"" + service_ip + ":" + str(port) + "\""
if not ips:
try:
ovn_nbctl("remove", "load_balancer", load_balancer,
"vips", key)
except Exception as e:
vlog.err("_create_load_balancer_vip remove: (%s)" % (str(e)))
return
if target_port.isdigit():
# target is of the form "IP1:port, IP2:port, IP3:port"
target_endpoints = ",".join(["%s:%s" % (ip, target_port)
for ip in ips])
else:
# 'target_port' is a string. We should get its number
# from the cache.
if not self.port_name_cache.get(namespace):
vlog.warn("targetPort of %s in ns %s does not have an "
"associated port. Ignoring endpoint creation."
% (target_port, namespace))
return
target_endpoint_list = []
for ip in ips:
if not self.port_name_cache[namespace].get(ip):
continue
num_port = self.port_name_cache[namespace][ip].get(target_port)
if not num_port:
continue
target_endpoint_list.append("%s:%s" % (ip, num_port))
if not target_endpoint_list:
vlog.warn("targetPort of %s in ns %s does not have any "
"associated ports. Ignoring endpoint creation."
% (target_port, namespace))
return
target_endpoints = ",".join(target_endpoint_list)
target = "\"" + target_endpoints + "\""
try:
ovn_nbctl("set", "load_balancer", load_balancer,
"vips:" + key + "=" + target)
except Exception as e:
vlog.err("_create_load_balancer_vip add: (%s)" % (str(e)))
def _create_load_balancer_vip(self, namespace, load_balancer, service_ip,
ips, port, target_port, protocol):
# With service_ip:port as a VIP, create an entry in 'load_balancer'
vlog.dbg(
"received event to create/modify load_balancer (%s) vip "
"service_ip=%s, ips=%s, port=%s, target_port=%s, protocol=%s" %
(load_balancer, service_ip, ips, port, target_port, protocol))
if not port or not target_port or not protocol or not load_balancer:
return
# key is of the form "IP:port" (with quotes around)
key = "\"" + service_ip + ":" + str(port) + "\""
if not ips:
try:
ovn_nbctl("remove", "load_balancer", load_balancer, "vips",
key)
except Exception as e:
vlog.err("_create_load_balancer_vip remove: (%s)" % (str(e)))
return
if target_port.isdigit():
# target is of the form "IP1:port, IP2:port, IP3:port"
target_endpoints = ",".join(
["%s:%s" % (ip, target_port) for ip in ips])
else:
# 'target_port' is a string. We should get its number
# from the cache.
if not self.port_name_cache.get(namespace):
vlog.warn("targetPort of %s in ns %s does not have an "
"associated port. Ignoring endpoint creation." %
(target_port, namespace))
return
target_endpoint_list = []
for ip in ips:
if not self.port_name_cache[namespace].get(ip):
continue
num_port = self.port_name_cache[namespace][ip].get(target_port)
if not num_port:
continue
target_endpoint_list.append("%s:%s" % (ip, num_port))
if not target_endpoint_list:
vlog.warn("targetPort of %s in ns %s does not have any "
"associated ports. Ignoring endpoint creation." %
(target_port, namespace))
return
target_endpoints = ",".join(target_endpoint_list)
target = "\"" + target_endpoints + "\""
try:
ovn_nbctl("set", "load_balancer", load_balancer,
"vips:" + key + "=" + target)
except Exception as e:
vlog.err("_create_load_balancer_vip add: (%s)" % (str(e)))
def _get_load_balancer_vips(self, load_balancer):
try:
vips = ovn_nbctl("--data=bare", "--no-heading", "get",
"load_balancer", load_balancer,
"vips").replace('=', ":")
return ast.literal_eval(vips)
except Exception as e:
vlog.err("_get_load_balancer_vips: failed to get vips for %s (%s)"
% (load_balancer, str(e)))
return None
def _get_load_balancer_vips(self, load_balancer):
try:
vips = ovn_nbctl("--data=bare", "--no-heading", "get",
"load_balancer", load_balancer,
"vips").replace('=', ":")
return ast.literal_eval(vips)
except Exception as e:
vlog.err("_get_load_balancer_vips: failed to get vips for %s (%s)"
% (load_balancer, str(e)))
return None
def get_null_fd():
"""Returns a readable and writable fd for /dev/null, if successful,
otherwise a negative errno value. The caller must not close the returned
fd (because the same fd will be handed out to subsequent callers)."""
global null_fd
if null_fd < 0:
try:
null_fd = os.open("/dev/null", os.O_RDWR)
except OSError, e:
vlog.err("could not open /dev/null: %s" % os.strerror(e.errno))
return -e.errno
def open(name):
"""Attempts to start listening for remote stream connections. 'name'
is a connection name in the form "TYPE:ARGS", where TYPE is an passive
stream class's name and ARGS are stream class-specific. Currently the
supported values for TYPE are "punix" and "ptcp".
Returns (error, pstream): on success 'error' is 0 and 'pstream' is the
new PassiveStream, on failure 'error' is a positive errno value and
'pstream' is None."""
if not PassiveStream.is_valid_name(name):
return errno.EAFNOSUPPORT, None
bind_path = name[6:]
if name.startswith("punix:"):
bind_path = ovs.util.abs_file_name(ovs.dirs.RUNDIR, bind_path)
if sys.platform != 'win32':
error, sock = ovs.socket_util.make_unix_socket(
socket.SOCK_STREAM, True, bind_path, None)
if error:
return error, None
else:
# Branch used only on Windows
try:
open(bind_path, 'w').close()
except:
return errno.ENOENT, None
pipename = winutils.get_pipe_name(bind_path)
if len(pipename) > 255:
# Return invalid argument if the name is too long
return errno.ENOENT, None
npipe = winutils.create_named_pipe(pipename)
if not npipe:
return errno.ENOENT, None
return 0, PassiveStream(None, name, bind_path, pipe=npipe)
elif name.startswith("ptcp:"):
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
remote = name.split(':')
sock.bind((remote[1], int(remote[2])))
else:
raise Exception('Unknown connection string')
try:
sock.listen(10)
except socket.error as e:
vlog.err("%s: listen: %s" % (name, os.strerror(e.error)))
sock.close()
return e.error, None
return 0, PassiveStream(sock, name, bind_path)
def open(name):
"""Attempts to start listening for remote stream connections. 'name'
is a connection name in the form "TYPE:ARGS", where TYPE is an passive
stream class's name and ARGS are stream class-specific. Currently the
supported values for TYPE are "punix" and "ptcp".
Returns (error, pstream): on success 'error' is 0 and 'pstream' is the
new PassiveStream, on failure 'error' is a positive errno value and
'pstream' is None."""
if not PassiveStream.is_valid_name(name):
return errno.EAFNOSUPPORT, None
bind_path = name[6:]
if name.startswith("punix:"):
bind_path = ovs.util.abs_file_name(ovs.dirs.RUNDIR, bind_path)
if sys.platform != 'win32':
error, sock = ovs.socket_util.make_unix_socket(
socket.SOCK_STREAM, True, bind_path, None)
if error:
return error, None
else:
# Branch used only on Windows
try:
open(bind_path, 'w').close()
except:
return errno.ENOENT, None
pipename = winutils.get_pipe_name(bind_path)
if len(pipename) > 255:
# Return invalid argument if the name is too long
return errno.ENOENT, None
npipe = winutils.create_named_pipe(pipename)
if not npipe:
return errno.ENOENT, None
return 0, PassiveStream(None, name, bind_path, pipe=npipe)
elif name.startswith("ptcp:"):
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
remote = name.split(':')
sock.bind((remote[1], int(remote[2])))
else:
raise Exception('Unknown connection string')
try:
sock.listen(10)
except socket.error as e:
vlog.err("%s: listen: %s" % (name, os.strerror(e.error)))
sock.close()
return e.error, None
return 0, PassiveStream(sock, name, bind_path)
def get_null_fd():
"""Returns a readable and writable fd for /dev/null, if successful,
otherwise a negative errno value. The caller must not close the returned
fd (because the same fd will be handed out to subsequent callers)."""
global null_fd
if null_fd < 0:
try:
null_fd = os.open("/dev/null", os.O_RDWR)
except OSError, e:
vlog.err("could not open /dev/null: %s" % os.strerror(e.errno))
return -e.errno
def _get_ovn_gateways(self):
# Return all created gateways.
physical_gateways = []
try:
physical_gateways = ovn_nbctl("--data=bare", "--no-heading",
"--columns=name", "find",
"logical_router",
"options:chassis!=null").split()
except Exception as e:
vlog.err("_get_ovn_gateways: find failed %s" % (str(e)))
return physical_gateways
def mgmt_intf_clear_dns_conf():
newdata = ""
try:
fd = open(DNS_FILE, 'w')
fd.write(newdata)
fd.close()
except IOError:
vlog.err("File operation failed for file " + DNS_FILE)
return True
def _get_ovn_gateways(self):
# Return all created gateways.
physical_gateways = []
try:
physical_gateways = ovn_nbctl(
"--data=bare", "--no-heading",
"--columns=name", "find",
"logical_router",
"options:chassis!=null").split()
except Exception as e:
vlog.err("_get_ovn_gateways: find failed %s" % (str(e)))
return physical_gateways
def __setattr__(self, column_name, value):
assert self._changes is not None
assert self._idl.txn
column = self._table.columns[column_name]
try:
datum = ovs.db.data.Datum.from_python(column.type, value,
_row_to_uuid)
except error.Error, e:
# XXX rate-limit
vlog.err("attempting to write bad value to column %s (%s)"
% (column_name, e))
return
def __setattr__(self, column_name, value):
assert self._changes is not None
assert self._idl.txn
column = self._table.columns[column_name]
try:
datum = ovs.db.data.Datum.from_python(column.type, value,
_row_to_uuid)
except error.Error, e:
# XXX rate-limit
vlog.err("attempting to write bad value to column %s (%s)" %
(column_name, e))
return
def _get_ovn_external_ip_gateway(self):
# XXX: In case of K8S 'external_ips', we can only expose it in one
# gateway to prevent duplicate ARP responses. We currently allocate
# the first gateway to handle all 'external_ips'.
try:
physical_gateway = ovn_nbctl(
"--data=bare", "--no-heading", "--columns=name", "find",
"logical_router", "external_ids:first_gateway=yes").split()
if not physical_gateway:
return None
except Exception as e:
vlog.err("_get_ovn_external_ip_gateway: find failed %s" % (str(e)))
return physical_gateway[0]
def get_null_fd():
"""Returns a readable and writable fd for /dev/null, if successful,
otherwise a negative errno value. The caller must not close the returned
fd (because the same fd will be handed out to subsequent callers)."""
global null_fd
if null_fd < 0:
try:
# os.devnull ensures compatibility with Windows, returns
# '/dev/null' for Unix and 'nul' for Windows
null_fd = os.open(os.devnull, os.O_RDWR)
except OSError as e:
vlog.err("could not open %s: %s" % (os.devnull,
os.strerror(e.errno)))
return -e.errno
return null_fd
def get_null_fd():
"""Returns a readable and writable fd for /dev/null, if successful,
otherwise a negative errno value. The caller must not close the returned
fd (because the same fd will be handed out to subsequent callers)."""
global null_fd
if null_fd < 0:
try:
# os.devnull ensures compatibility with Windows, returns
# '/dev/null' for Unix and 'nul' for Windows
null_fd = os.open(os.devnull, os.O_RDWR)
except OSError as e:
vlog.err("could not open %s: %s" % (os.devnull,
os.strerror(e.errno)))
return -e.errno
return null_fd
def block(self):
"""Blocks until one or more of the events registered with
self.fd_wait() occurs, or until the minimum duration registered with
self.timer_wait() elapses, or not at all if self.immediate_wake() has
been called."""
try:
try:
events = self.poll.poll(self.timeout)
self.__log_wakeup(events)
except select.error, e:
# XXX rate-limit
error, msg = e
if error != errno.EINTR:
vlog.err("poll: %s" % e[1])
finally:
self.__reset()
def _get_ip_address_from_annotations(self, annotations):
if not annotations.get("ovn"):
return None
ovn_annotated_dict = ast.literal_eval(annotations['ovn'])
ip_address_mask = ovn_annotated_dict.get('ip_address')
if not ip_address_mask:
return None
ip_address = None
try:
(ip_address, mask) = ip_address_mask.split("/")
except Exception:
vlog.err("_get_ip_address_from_annotations: failed to get ip")
return ip_address
def _get_ip_address_from_annotations(self, annotations):
if not annotations.get("ovn"):
return None
ovn_annotated_dict = ast.literal_eval(annotations['ovn'])
ip_address_mask = ovn_annotated_dict.get('ip_address')
if not ip_address_mask:
return None
ip_address = None
try:
(ip_address, mask) = ip_address_mask.split("/")
except Exception:
vlog.err("_get_ip_address_from_annotations: failed to get ip")
return ip_address
def _get_ovn_external_ip_gateway(self):
# XXX: In case of K8S 'external_ips', we can only expose it in one
# gateway to prevent duplicate ARP responses. We currently allocate
# the first gateway to handle all 'external_ips'.
try:
physical_gateway = ovn_nbctl(
"--data=bare", "--no-heading",
"--columns=name", "find",
"logical_router",
"external_ids:first_gateway=yes").split()
if not physical_gateway:
return None
except Exception as e:
vlog.err("_get_ovn_external_ip_gateway: find failed %s" % (str(e)))
return physical_gateway[0]
def block(self):
"""Blocks until one or more of the events registered with
self.fd_wait() occurs, or until the minimum duration registered with
self.timer_wait() elapses, or not at all if self.immediate_wake() has
been called."""
try:
try:
events = self.poll.poll(self.timeout)
self.__log_wakeup(events)
except select.error as e:
# XXX rate-limit
error, msg = e
if error != errno.EINTR:
vlog.err("poll: %s" % e[1])
finally:
self.__reset()
def _create_gateways_vip(self, namespace, ips, port, target_port,
protocol):
# Each gateway has a separate load-balancer for N/S traffic
physical_gateways = self._get_ovn_gateways()
if not physical_gateways:
return
for physical_gateway in physical_gateways:
# Go through each gateway to get its physical_ip and load-balancer.
try:
physical_ip = ovn_nbctl(
"get", "logical_router", physical_gateway,
"external_ids:physical_ip").strip('"')
except Exception as e:
vlog.err("_create_gateways_vip: get failed for"
" %s (%s)" % (physical_gateway, str(e)))
continue
if not physical_ip:
vlog.warn("physical gateway %s does not have physical ip"
% (physical_ip))
continue
try:
external_id_key = protocol + "_lb_gateway_router"
load_balancer = ovn_nbctl(
"--data=bare", "--no-heading",
"--columns=_uuid", "find", "load_balancer",
"external_ids:" + external_id_key + "=" +
physical_gateway
).strip('"')
except Exception as e:
vlog.err("_create_gateways_vip: find failed for"
" %s (%s)" % (physical_gateway, str(e)))
continue
if not load_balancer:
vlog.warn("physical gateway %s does not have load_balancer"
% (physical_gateway))
continue
# With the physical_ip:port as the VIP, add an entry in
# 'load_balancer'.
self._create_load_balancer_vip(namespace, load_balancer,
physical_ip, ips,
port, target_port, protocol)
def _create_gateways_vip(self, namespace, ips, port, target_port,
protocol):
# Each gateway has a separate load-balancer for N/S traffic
physical_gateways = self._get_ovn_gateways()
if not physical_gateways:
return
for physical_gateway in physical_gateways:
# Go through each gateway to get its physical_ip and load-balancer.
try:
physical_ip = ovn_nbctl(
"get", "logical_router", physical_gateway,
"external_ids:physical_ip").strip('"')
except Exception as e:
vlog.err("_create_gateways_vip: get failed for"
" %s (%s)" % (physical_gateway, str(e)))
continue
if not physical_ip:
vlog.warn("physical gateway %s does not have physical ip"
% (physical_ip))
continue
try:
external_id_key = protocol + "_lb_gateway_router"
load_balancer = ovn_nbctl(
"--data=bare", "--no-heading",
"--columns=_uuid", "find", "load_balancer",
"external_ids:" + external_id_key + "=" +
physical_gateway
).strip('"')
except Exception as e:
vlog.err("_create_gateways_vip: find failed for"
" %s (%s)" % (physical_gateway, str(e)))
continue
if not load_balancer:
vlog.warn("physical gateway %s does not have load_balancer"
% (physical_gateway))
continue
# With the physical_ip:port as the VIP, add an entry in
# 'load_balancer'.
self._create_load_balancer_vip(namespace, load_balancer,
physical_ip, ips,
port, target_port, protocol)
def _get_physical_gateway_ips(self):
if self.physical_gateway_ips:
return self.physical_gateway_ips
try:
physical_gateway_ip_networks = ovn_nbctl(
"--data=bare", "--no-heading", "--columns=network", "find",
"logical_router_port",
"external_ids:gateway-physical-ip=yes").split()
except Exception as e:
vlog.err("_populate_gateway_ip: find failed %s" % (str(e)))
for physical_gateway_ip_network in physical_gateway_ip_networks:
(ip, mask) = physical_gateway_ip_network.split('/')
self.physical_gateway_ips.append(ip)
return self.physical_gateway_ips
def delete_logical_port(self, event):
data = event.metadata
pod_name = data['metadata']['name']
namespace = data['metadata']['namespace']
logical_port = "%s_%s" % (namespace, pod_name)
if not logical_port:
vlog.err("absent pod name in pod %s. "
"Not creating logical port" % (data))
return
try:
ovn_nbctl("--if-exists", "lsp-del", logical_port)
except Exception as e:
vlog.err("_delete_logical_port: lsp-add (%s)" % (str(e)))
return
vlog.dbg("deleted logical port %s" % (logical_port))
def delete_logical_port(self, event):
data = event.metadata
pod_name = data['metadata']['name']
namespace = data['metadata']['namespace']
logical_port = "%s_%s" % (namespace, pod_name)
if not pod_name:
vlog.err("absent pod name in pod %s. "
"unable to delete logical port" % data)
return
try:
ovn_nbctl("--if-exists", "lsp-del", logical_port)
except Exception:
vlog.exception("failure in delete_logical_port: lsp-del")
return
vlog.info("deleted logical port %s" % logical_port)
def delete_logical_port(self, event):
data = event.metadata
pod_name = data['metadata']['name']
namespace = data['metadata']['namespace']
logical_port = "%s_%s" % (namespace, pod_name)
if not pod_name:
vlog.err("absent pod name in pod %s. "
"unable to delete logical port" % data)
return
try:
ovn_nbctl("--if-exists", "lsp-del", logical_port)
except Exception:
vlog.exception("failure in delete_logical_port: lsp-del")
return
vlog.info("deleted logical port %s" % logical_port)
def _get_physical_gateway_ips(self):
if self.physical_gateway_ips:
return self.physical_gateway_ips
try:
physical_gateway_ip_networks = ovn_nbctl(
"--data=bare", "--no-heading",
"--columns=network", "find",
"logical_router_port",
"external_ids:gateway-physical-ip=yes").split()
except Exception as e:
vlog.err("_populate_gateway_ip: find failed %s" % (str(e)))
for physical_gateway_ip_network in physical_gateway_ip_networks:
ip, _mask = physical_gateway_ip_network.split('/')
self.physical_gateway_ips.append(ip)
return self.physical_gateway_ips
def _monitor_daemon(daemon_pid):
# XXX should log daemon's stderr output at startup time
# XXX should use setproctitle module if available
last_restart = None
while True:
retval, status = _waitpid(daemon_pid, 0)
if retval < 0:
sys.stderr.write("waitpid failed\n")
sys.exit(1)
elif retval == daemon_pid:
status_msg = ("pid %d died, %s"
% (daemon_pid, ovs.process.status_msg(status)))
if _should_restart(status):
if os.WCOREDUMP(status) and sys.platform != "win32":
import resource
# Disable further core dumps to save disk space.
try:
resource.setrlimit(resource.RLIMIT_CORE, (0, 0))
except resource.error:
vlog.warn("failed to disable core dumps")
# Throttle restarts to no more than once every 10 seconds.
if (last_restart is not None and
ovs.timeval.msec() < last_restart + 10000):
vlog.warn("%s, waiting until 10 seconds since last "
"restart" % status_msg)
while True:
now = ovs.timeval.msec()
wakeup = last_restart + 10000
if now > wakeup:
break
sys.stdout.write("sleep %f\n" % (
(wakeup - now) / 1000.0))
time.sleep((wakeup - now) / 1000.0)
last_restart = ovs.timeval.msec()
vlog.err("%s, restarting" % status_msg)
daemon_pid = fork_and_wait_for_startup()
if not daemon_pid:
break
else:
vlog.info("%s, exiting" % status_msg)
sys.exit(0)
def _create_load_balancer_vip(self, service_type, service_ip, ips, port,
target_port, protocol):
vlog.dbg("received event to create load_balancer vip")
if not port or not target_port or not protocol or not service_type:
return
load_balancer = ""
if protocol == "TCP" and service_type == "ClusterIP":
load_balancer = variables.K8S_CLUSTER_LB_TCP
if protocol == "UDP" and service_type == "ClusterIP":
load_balancer = variables.K8S_CLUSTER_LB_UDP
if protocol == "TCP" and service_type == "NodePort":
load_balancer = variables.K8S_NS_LB_TCP
if protocol == "UDP" and service_type == "NodePort":
load_balancer = variables.K8S_NS_LB_UDP
if not load_balancer:
return
# key is of the form "IP:port" (with quotes around)
key = "\"" + service_ip + ":" + str(port) + "\""
if not ips:
try:
ovn_nbctl("remove", "load_balancer", load_balancer, "vips",
key)
except Exception as e:
vlog.err("_create_load_balancer_vip remove: (%s)" % (str(e)))
return
# target is of the form "IP1:port, IP2:port, IP3:port"
target = ""
for ip in ips:
target = target + ip + ":" + str(target_port) + ","
target = target[:-1]
target = "\"" + target + "\""
try:
ovn_nbctl("set", "load_balancer", load_balancer,
"vips:" + key + "=" + target)
except Exception as e:
vlog.err("_create_load_balancer_vip add: (%s)" % (str(e)))
def _monitor_daemon(daemon_pid):
# XXX should log daemon's stderr output at startup time
# XXX should use setproctitle module if available
last_restart = None
while True:
retval, status = _waitpid(daemon_pid, 0)
if retval < 0:
sys.stderr.write("waitpid failed\n")
sys.exit(1)
elif retval == daemon_pid:
status_msg = ("pid %d died, %s" %
(daemon_pid, ovs.process.status_msg(status)))
if _should_restart(status):
if sys.platform != 'win32' and os.WCOREDUMP(status):
# Disable further core dumps to save disk space.
try:
resource.setrlimit(resource.RLIMIT_CORE, (0, 0))
except resource.error:
vlog.warn("failed to disable core dumps")
# Throttle restarts to no more than once every 10 seconds.
if (last_restart is not None
and ovs.timeval.msec() < last_restart + 10000):
vlog.warn("%s, waiting until 10 seconds since last "
"restart" % status_msg)
while True:
now = ovs.timeval.msec()
wakeup = last_restart + 10000
if now > wakeup:
break
sys.stdout.write("sleep %f\n" %
((wakeup - now) / 1000.0))
time.sleep((wakeup - now) / 1000.0)
last_restart = ovs.timeval.msec()
vlog.err("%s, restarting" % status_msg)
daemon_pid = _fork_and_wait_for_startup()
if not daemon_pid:
break
else:
vlog.info("%s, exiting" % status_msg)
sys.exit(0)
def event_log_init(cat):
global content
global category
found = 0
# Search whether category is already initialised
for i in range(len(category)):
if cat in category[i]:
# Already initialised, so return.
return FAIL
try:
with open('/etc/openswitch/supportability/ops_events.yaml', 'r') as f:
doc = yaml.load(f)
f.close()
for txt in range(len(doc[EV_DEFINITION])):
yaml_cat = doc[EV_DEFINITION][txt][EV_CATEGORY]
if yaml_cat == cat:
mydic = {
EV_CATEGORY:
yaml_cat,
EV_NAME:
doc[EV_DEFINITION][txt][EV_NAME],
EV_ID:
doc[EV_DEFINITION][txt][EV_ID],
EV_SEVERITY:
doc[EV_DEFINITION][txt][EV_SEVERITY],
EV_DESCRIPTION:
doc[EV_DEFINITION][txt][EV_DESCRIPTION_YAML],
}
# Now add it to global event list
content.append(mydic)
found = 1
if found is NOT_FOUND:
# This means supplied category name is not there in YAML, so return.
vlog.err("Event Category not Found")
return FAIL
else:
# Add category to global category list
category.append(cat)
except:
vlog.err("Event Log Initialization Failed")
return FAIL
def check_connection_completion(sock):
p = ovs.poller.SelectPoll()
p.register(sock, ovs.poller.POLLOUT)
pfds = p.poll(0)
if len(pfds) == 1:
revents = pfds[0][1]
if revents & ovs.poller.POLLERR:
try:
# The following should raise an exception.
socket.send("\0", socket.MSG_DONTWAIT)
# (Here's where we end up if it didn't.)
# XXX rate-limit
vlog.err("poll return POLLERR but send succeeded")
return errno.EPROTO
except socket.error, e:
return get_exception_errno(e)
else:
return 0
def check_connection_completion(sock):
p = ovs.poller.SelectPoll()
p.register(sock, ovs.poller.POLLOUT)
pfds = p.poll(0)
if len(pfds) == 1:
revents = pfds[0][1]
if revents & ovs.poller.POLLERR:
try:
# The following should raise an exception.
socket.send("\0", socket.MSG_DONTWAIT)
# (Here's where we end up if it didn't.)
# XXX rate-limit
vlog.err("poll return POLLERR but send succeeded")
return errno.EPROTO
except socket.error, e:
return get_exception_errno(e)
else:
return 0
def _create_load_balancer_vip(self, service_type, service_ip, ips, port,
target_port, protocol):
vlog.dbg("received event to create/modify load_balancer vip with "
"service_type=%s, service_ip=%s, ips=%s, port=%s,"
"target_port=%s, protocol=%s"
% (service_type, service_ip, ips, port, target_port,
protocol))
if not port or not target_port or not protocol or not service_type:
return
load_balancer = ""
if protocol == "TCP" and service_type == "ClusterIP":
load_balancer = variables.K8S_CLUSTER_LB_TCP
elif protocol == "UDP" and service_type == "ClusterIP":
load_balancer = variables.K8S_CLUSTER_LB_UDP
elif protocol == "TCP" and service_type == "NodePort":
load_balancer = variables.K8S_NS_LB_TCP
elif protocol == "UDP" and service_type == "NodePort":
load_balancer = variables.K8S_NS_LB_UDP
if not load_balancer:
return
# key is of the form "IP:port" (with quotes around)
key = "\"" + service_ip + ":" + str(port) + "\""
if not ips:
try:
ovn_nbctl("remove", "load_balancer", load_balancer,
"vips", key)
except Exception as e:
vlog.err("_create_load_balancer_vip remove: (%s)" % (str(e)))
return
# target is of the form "IP1:port, IP2:port, IP3:port"
target_endpoints = ",".join(["%s:%s" % (ip, target_port)
for ip in ips])
target = "\"" + target_endpoints + "\""
try:
ovn_nbctl("set", "load_balancer", load_balancer,
"vips:" + key + "=" + target)
except Exception as e:
vlog.err("_create_load_balancer_vip add: (%s)" % (str(e)))
def block(self):
"""Blocks until one or more of the events registered with
self.fd_wait() occurs, or until the minimum duration registered with
self.timer_wait() elapses, or not at all if self.immediate_wake() has
been called."""
try:
try:
events = self.poll.poll(self.timeout)
self.__log_wakeup(events)
except OSError as e:
""" On Windows, the select function from poll raises OSError
exception if the polled array is empty."""
if e.errno != errno.EINTR:
vlog.err("poll: %s" % os.strerror(e.errno))
except select.error as e:
# XXX rate-limit
error, msg = e
if error != errno.EINTR:
vlog.err("poll: %s" % e[1])
finally:
self.__reset()
def delete_logical_port(self, event):
data = event.metadata
pod_name = data['metadata']['name']
namespace = data['metadata']['namespace']
logical_port = "%s_%s" % (namespace, pod_name)
if not pod_name:
vlog.err("absent pod name in pod %s. "
"unable to delete logical port" % data)
return
annotations = data['metadata']['annotations']
ip_address = self._get_ip_address_from_annotations(annotations)
if ip_address:
self._delete_k8s_l4_port_name_cache(data, ip_address)
try:
ovn_nbctl("--if-exists", "lsp-del", logical_port)
except Exception:
vlog.exception("failure in delete_logical_port: lsp-del")
return
vlog.info("deleted logical port %s" % logical_port)
def sync_pods(self, pods):
expected_logical_ports = set()
pods = pods.get('items', [])
for pod in pods:
pod_name = pod['metadata']['name']
namespace = pod['metadata']['namespace']
logical_port = "%s_%s" % (namespace, pod_name)
annotations = pod['metadata']['annotations']
expected_logical_ports.add(logical_port)
# We should sync the container port names as there are no
# guarantees that a endpoint creation event will become after
# all the pods creation events.
ip_address = self._get_ip_address_from_annotations(annotations)
if ip_address:
self._add_k8s_l4_port_name_cache(pod, ip_address)
try:
existing_logical_ports = ovn_nbctl(
"--data=bare", "--no-heading",
"--columns=name", "find",
"logical_switch_port",
"external-ids:pod=true").split()
existing_logical_ports = set(existing_logical_ports)
except Exception as e:
vlog.err("sync_pods: find failed %s" % (str(e)))
return
for logical_port in existing_logical_ports - expected_logical_ports:
try:
ovn_nbctl("--if-exists", "lsp-del", logical_port)
except Exception as e:
vlog.err("sync_pods: failed to delete logical_port %s"
% (logical_port))
continue
vlog.info("sync_pods: Deleted logical port %s"
% (logical_port))
def set_nonblocking(sock):
try:
sock.setblocking(0)
except socket.error, e:
vlog.err("could not set nonblocking mode on socket: %s"
% os.strerror(get_exception_errno(e)))
def _fatal(msg):
vlog.err(msg)
sys.stderr.write("%s\n" % msg)
sys.exit(1)