def _unpack_rcv(self):
"""Switch a UMAD AH back into an IBPath. Note this is only
used for recv'd AH's where the meaning of the fields is altered.
Our convention is that the path describes the packet headers as they
existed on the wire, so this untwiddles things."""
(sqpn,
qkey,
SLID,
self.SL,
DLID_bits,
self.has_grh,
DGID_index,
self.hop_limit,
self.traffic_class,
self.SGID,
flow_label,
pkey_index) = \
UMAD.ib_mad_addr_t.unpack(self._cached_umad_ah)
self.sqpn = cpu_to_be32(sqpn)
# There is no pkey validation for SMPs (see IBA figure 156), so the
# pkey should always be the default NOTE: mtcha at least has been seen
# to return random values for pkey_index on SMPs, which is why we need
# this check.
if self.dqpn != 0:
self.pkey_index = pkey_index
self.qkey = cpu_to_be32(qkey)
self.DLID = DLID_bits | self.end_port.lid
self.SLID = cpu_to_be16(SLID)
if self.has_grh:
self.SGID = IBA.GID(self.SGID, True)
self.DGID = self.end_port.gids[DGID_index]
self.flow_label = cpu_to_be32(flow_label)
else:
del self.SGID
def set_mad_attr(attr, name, v):
try:
# Need to use eval because name could have dots in it.
arg = eval("attr.%s" % (name))
except AttributeError:
raise CmdError("%r is not a valid attribute for %r" % (name, attr))
try:
if isinstance(arg, int) or isinstance(arg, long):
v = int(v, 0)
elif isinstance(arg, IBA.GID):
v = IBA.GID(v)
elif isinstance(arg, IBA.GUID):
v = IBA.GUID(v)
elif isinstance(arg, bytearray):
v = v.decode("string_escape")
if len(v) > len(arg):
raise CmdError("String %r is too long, can only be up to %u" %
(v, len(arg)))
if len(v) < len(arg):
v = v + bytearray(len(arg) - len(v))
elif isinstance(arg, list):
raise CmdError("Lists currently cannot be set.")
else:
raise CmdError("Internal Error, I don't know what %s %r is." %
(type(arg), arg))
except ValueError as err:
raise CmdError("String %r did not parse: %s" % (v, err))
exec "attr.%s = v" % (name)
def test_basic(self):
print self.ctx.query_port();
print self.ctx.query_device();
pd = self.ctx.pd();
print pd,repr(pd)
cq = self.ctx.cq(100);
print cq,repr(cq)
try:
cq.resize(200);
except rdma.SysError as e:
if e.errno != errno.ENOSYS:
raise;
self.assertEqual(cq.poll(),[]);
comp = self.ctx.comp_channel();
print comp,repr(comp)
qp = pd.qp(ibv.IBV_QPT_UD,100,cq,100,cq);
print qp,repr(qp)
print qp.query(0xFFFF);
mpath = rdma.path.IBPath(self.ctx.end_port,DLID=0xC000,
DGID=IBA.GID("ff02::1"));
qp.attach_mcast(mpath);
qp.detach_mcast(mpath);
buf = mmap.mmap(-1,4096);
mr = pd.mr(buf,ibv.IBV_ACCESS_LOCAL_WRITE|ibv.IBV_ACCESS_REMOTE_WRITE);
print mr,repr(mr)
print "MR",mr.addr,mr.length,mr.lkey,mr.rkey
self.assertRaises(TypeError,pd.ah,None);
ah = pd.ah(self.end_port.sa_path);
print ah,repr(ah)
srq = pd.srq();
print srq,repr(srq)
print srq.query();
srq.modify(100);
def cmd_ibaddr(argv,o):
"""Display the GID and LID addresses for end ports.
Usage: %prog [-glL] [TARGET]"""
o.add_option("-l","--lid_show",action="store_true",dest="lid",
help="Show LID information");
o.add_option("-L","--Lid_show",action="store_true",dest="lid",
help="Show LID information");
o.add_option("-g","--gid_show",action="store_true",dest="gid",
help="Show GID information");
LibIBOpts.setup(o);
(args,values) = o.parse_args(argv);
lib = LibIBOpts(o,args,values,1,(tmpl_target,));
if not values:
values = (None,);
if args.lid is None and args.gid is None:
args.lid = True;
args.gid = True;
with lib.get_umad_for_target(values[0]) as umad:
path = lib.path;
ninf = umad.SubnGet(IBA.SMPNodeInfo,path);
path.DGID = IBA.GID(prefix=IBA.GID_DEFAULT_PREFIX,guid=ninf.portGUID);
pinf = umad.SubnGet(IBA.SMPPortInfo,path,0);
if args.gid:
print "GID %s"%(path.DGID),
if args.lid:
print "LID start %u end %u"%(pinf.LID,pinf.LID + (1 << pinf.LMC)-1),
print
return lib.done();
def test_fail(self):
"""Test valid get_end_port calls that fail."""
devices = rdma.get_devices()
dev = devices.first()
self.assertRaises(rdma.RDMAError, rdma.get_end_port, IBA.GID("::"))
self.assertRaises(rdma.RDMAError, rdma.get_end_port,
IBA.GUID("0000:0000:0000:0000"))
self.assertRaises(rdma.RDMAError, rdma.get_end_port, "xxx")
self.assertRaises(rdma.RDMAError, rdma.get_end_port,
"%s/99" % (dev.name))
def ninf(self):
try:
return self.__dict__["ninf"];
except KeyError:
pass
ninf = IBA.ComponentMask(IBA.SANodeRecord());
ninf.LID = self.path.DLID;
ninf = self.umad.SubnAdmGet(ninf);
self.__dict__["ninf"] = ninf;
if self.path.DGID is None:
self.path.DGID = IBA.GID(prefix=IBA.GID_DEFAULT_PREFIX,
guid=ninf.nodeInfo.portGUID);
return ninf;
def unpack_from(self, buffer, offset=0):
from socket import htonl as cpu_to_be32
from socket import htons as cpu_to_be16
(self.agent_id,self.status,self.timeout_ms,self.retries,self.length,
self.qpn,self.qkey,self.lid,self.sl,self.path_bits,self.grh_present,self.gid_index,
self.hop_limit,self.traffic_class,self.gid,self.flow_label,self.pkey_index,self.reserved_58) = \
struct.unpack_from("=LLLLLLLHBBBBBB16sLH6s",buffer,offset+0)
self.qpn = cpu_to_be32(self.qpn)
self.qkey = cpu_to_be32(self.qkey)
self.lid = cpu_to_be16(self.lid)
self.gid = IBA.GID(self.gid, raw=True)
self.flow_label = cpu_to_be32(self.flow_label)
def get_end_port(name=None):
"""Return a :class:`rdma.devices.EndPort` for the default end port if name
is ``None``, or for the end port described by name.
The end port string format is one of:
=========== ===================
Format Example
=========== ===================
device mlx4_0 (defaults to the first port)
device/port mlx4_0/1
Port GID fe80::2:c903:0:1491
Port GUID 0002:c903:0000:1491
=========== ===================
:rtype: :class:`rdma.devices.EndPort`
:raises rdma.RDMAError: If no matching device is found or name is invalid."""
devices = get_devices()
if len(devices) == 0:
raise RDMAError("No RDMA devices found.")
if name is None:
return devices.first().end_ports.first()
# Try for a port GID
import rdma.devices
import rdma.IBA
try:
gid = IBA.GID(name)
except ValueError:
pass
else:
return rdma.devices.find_port_gid(devices, gid)[0]
# Port GUID
try:
guid = IBA.GUID(name)
except ValueError:
pass
else:
return rdma.devices.find_port_guid(devices, guid)
# Device name string
return rdma.devices.find_port_name(devices, name)
def link_end_port(self,
port,
portIdx=None,
nodeGUID=None,
portGUID=None,
path=None,
LID=None,
LMC=0):
"""Use the provided information about *port* to update the database.
Note: For switches *portIdx* must be 0."""
assert (port == port.to_end_port())
if (LID is None and path is not None
and not isinstance(path, rdma.path.IBDRPath)):
LID = path.DLID
node = port.parent
if portIdx is not None:
node.set_port(portIdx, port)
if portGUID is not None and port.portGUID is None:
port.portGUID = portGUID
self.ports[portGUID] = port
if LID is not None:
port.LID = LID
if LMC is None:
LMC = 0
self.set_max_lid(LID + (1 << LMC) - 1)
for I in IBA.lid_lmc_range(LID, LMC):
self.lids[I] = port
if path is not None:
path._cached_subnet_end_port = port
# Since we know it, record the DGID into the path. This produces
# error messages that include the DGID..
if portGUID is not None and path.DGID is None:
path.DGID = IBA.GID(prefix=IBA.GID_DEFAULT_PREFIX,
guid=portGUID)
if self.paths is not None:
self.paths[port] = path
return port
def _do_loop_test_mc(self):
"""Test HCA loop back between two QPs as well as SRQ."""
qp_type = ibv.IBV_QPT_UD;
print "Testing QP to QP loop type %u UD MULTICAST"%(qp_type)
with self.ctx.pd() as pd:
path_a,qp_a,path_b,qp_b,poller,srq,pool = \
self._get_loop(pd,qp_type);
# Note: since both our QPs are on the same end port then the DLID
# does not matter as far as forwarding is concerned, so the HCA
# should replicate entirely based on the DLID.
mcpath = path_b.copy(DGID=IBA.GID("FF02::1"),DLID=0xC000,dqpn=0xFFFFFF,
traffic_class=0x89,
flow_label=0x1234,
hop_limit=23,
has_grh=True);
qp_a.attach_mcast(mcpath);
qp_b.attach_mcast(mcpath);
qp_b.post_send(pool.make_send_wr(pool.pop(),256,mcpath));
qp_a.post_send(pool.make_send_wr(pool.pop(),256,mcpath));
recvs = 0;
sends = 0
for wc in poller.iterwc(count=6,timeout=0.5):
if wc.opcode & ibv.IBV_WC_RECV:
recvs = recvs + 1
path = ibv.WCPath(mcpath.end_port,wc,
pool._mem,
(wc.wr_id & pool.BUF_ID_MASK)*pool.size);
self.assertEquals(path.DGID,mcpath.DGID);
self.assertEquals(path.SGID,mcpath.SGID);
self.assertEquals(path.flow_label,mcpath.flow_label);
self.assertEquals(path.traffic_class,mcpath.traffic_class);
self.assertEquals(path.hop_limit,mcpath.hop_limit);
if wc.opcode == ibv.IBV_WC_SEND:
sends = sends + 1
pool.finish_wcs(srq,wc);
self.assertFalse(poller.timedout);
self.assertEquals(recvs,4);
self.assertEquals(sends,2);
def test_str_good(self):
"""Check good input to :func:`rdma.path.from_string`."""
self.check_path(None, DGID=IBA.GID("fe80::1"))
self.check_path(None, DGID=IBA.GID("fe80::1"), SL=10, pkey=0xff)
self.check_path(None, DGID=IBA.GID("fe80::1"), SLID=0x12, DLID=15)
self.check_path_str(None, "fe80::1", DGID=IBA.GID("fe80::1"))
self.check_path_str(None, "0:0:0:1", DGID=IBA.GID("fe80::1"))
self.check_path_str(None, "1", DLID=1)
self.check_path_str(None, "0xff", DLID=0xff)
self.check_path_str(None, "0,1", cls=rdma.path.IBDRPath, drPath="\0\1")
self.check_path_str(None, "0,", cls=rdma.path.IBDRPath, drPath="\0")
for I in rdma.get_devices():
for J in I.end_ports:
self.check_path_str(J,
"fe80::1%%%s" % (J),
DGID=IBA.GID("fe80::1"))
for G in J.gids:
if int(G) >> 64 != IBA.GID_DEFAULT_PREFIX:
self.check_path_str(J, "%s" % (G), DGID=G)
def from_string(s,default_end_port=None,require_dev=None,require_ep=None):
"""Convert the string *s* into an instance of :class:`Path` or
derived.
Supported formats for *s* are:
=========== ============================ ========================
Format Example Creates
=========== ============================ ========================
Port GID fe80::2:c903:0:1491 IBPath.DGID = s
Scope'd GID fe80::2:c903:0:1491%mlx4_0/1 IBPath.DGID = s
Port GUID 0002:c903:0000:1491 IBPath.DGID = fe80:: + s
LID 12 IBPath.DLID = 12
Hex LID 0xc IBPath.DLID = 12
DR Path 0,1, IBDRPath.drPath = '\\\\0\\\\1'
Path Spec IBPath(DLID=2,SL=2) IBPath.{DLID=2,SL=2}
=========== ============================ ========================
If the format unambiguously specifies an end port, eg due to a provided
scope or by specifying the subnet prefix then the result will have `end_port`
set appropriately. Otherwise `end_port` is set to `default_end_port`.
*require_dev* and *require_ep* will restrict the lookup to returning
a path for those conditions. If a scoped address is given that doesn't
match then :exc:`ValueError` is raised. These options should be used when
a path is being parsed for use with an existing bound resource (eg
a :class:`rdma.ibverbs.Context` or :class:`rdma.ibverbs.`)
FUTURE: This may return paths other than IB for other technologies.
:raises ValueError: If the string can not be parsed."""
if require_ep is not None:
default_end_port = require_ep;
if s.find("(") != -1:
ret = from_spec_string(s);
if ret.end_port is None:
ret.end_port = default_end_port;
else:
_check_ep(ret.end_port,require_dev,require_ep);
return ret;
dr = s.split(",");
if len(dr) != 1:
if dr[-1] == '':
dr = [int(I) for I in dr[:-1]];
else:
dr = [int(I) for I in dr];
for I in dr:
if I >= 255:
raise ValueError("Invalid DR path specification %r"%(s,));
if len(dr) == 0:
raise ValueError("Invalid DR path specification %r"%(s,));
if dr[0] != 0:
raise ValueError("Invalid DR path specification %r"%(s,));
drPath = bytes("").join("%c"%(I) for I in dr);
return IBDRPath(default_end_port,drPath=drPath);
a = s.split('%');
if len(a) == 2:
DGID = IBA.GID(a[0])
try:
end_port = rdma.get_end_port(a[1]);
_check_ep(end_port,require_dev,require_ep);
except rdma.RDMAError, e:
raise ValueError("Could not find %r: %s"%(a[1],e));
return IBPath(end_port,DGID=DGID);
def from_spec_string(s):
"""Construct a *Path* (or derived) instance from it's `repr` string.
This parser is safe to use with untrusted data.
:raises ValueError: If the string can not be parsed."""
import re,itertools;
m = re.match("^(.+?)\(\s*?(.*?)\s*?\)$",s);
if m is None:
raise ValueError("Invalid path specification %r"%(s,));
m = m.groups();
cls = getattr(sys.modules[__name__],m[0],None)
if cls is None or not issubclass(cls,Path):
raise ValueError("Invalid path specification %r, bad path type"%(s,));
kwargs = dict((t[0].strip(), t[2].strip())
for t in (I.partition('=')
for I in m[1].split(',')));
if len(kwargs) < 1:
raise ValueError("Invalid path specification %r"%(s,));
for k,v in kwargs.iteritems():
if v == '':
raise ValueError("Invalid path specification %r"%(s,));
if not hasattr(cls,k):
raise ValueError("Path attribute %r is not known"%(k,));
if v.startswith("GID("):
v = v[4:-1];
if v[0] == '"' or v[0] == "'":
v = v[1:-1];
kwargs[k] = IBA.GID(v);
elif k.find("GID") != -1:
kwargs[k] = IBA.GID(v);
elif k == "drPath":
# Using : because I am too lazy to fix the splitter to respect quotes.
dr = v.split(":");
if len(dr) == 1:
raise ValueError("Invalid DR path specification %r"%(v,));
if dr[-1] == '':
dr = [int(I) for I in dr[:-1]];
else:
dr = [int(I) for I in dr];
for I in dr:
if I >= 255:
raise ValueError("Invalid DR path specification %r"%(v,));
if len(dr) == 0:
raise ValueError("Invalid DR path specification %r"%(s,));
if dr[0] != 0:
raise ValueError("Invalid DR path specification %r"%(s,));
kwargs[k] = bytes("").join("%c"%(I) for I in dr);
elif k == "end_port":
if v[0] == '"' or v[0] == "'":
v = v[1:-1];
try:
if v == "None":
kwargs[k] = None;
else:
kwargs[k] = rdma.get_end_port(v);
except rdma.RDMAError, e:
raise ValueError("Could not find %r: %s"%(v,e));
else:
try:
kwargs[k] = int(v,0);
except ValueError:
raise ValueError("%r=%r is not a valid integer"%(k,v));
def _conv_gid2guid(s):
"""Return the GUID portion of a GID string.
:raises ValueError: If the string is invalid."""
return IBA.GID(s).guid()
def test_gid(self):
"""Test IBA.GID class"""
pg = IBA.GID("fe80::21c:23ff:fee2:946a")
self.assertEquals(pg, pg)
self.assertEquals(pg, IBA.GID("fe80::21c:23ff:fee2:946a"))
self.assertEquals(pg, IBA.GID(pg))
def struct_dotted(F,
s,
name_prefix='',
dump_list=False,
skip_reserved=True,
column=33,
colon=False,
name_map=None):
"""This tries to emulate the libib structure print format. Members are
printed one per line with values aligned on column 32."""
for name, mbits, count in s.MEMBERS:
if skip_reserved and name.startswith("reserved_"):
continue
attr = getattr(s, name)
if attr is None:
continue
cname = name[0].upper() + name[1:]
if name_map:
cname = name_map.get(cname, cname)
# Special automagic decode of format data members based on
# attribute ID.
if name == "data" and isinstance(s, rdma.binstruct.BinFormat):
nattr = IBA.ATTR_TO_STRUCT.get((s.__class__, s.attributeID))
if nattr != None:
if nattr.MAD_LENGTH <= len(attr):
attr = nattr(attr)
if isinstance(attr, rdma.binstruct.BinStruct):
struct_dotted(F,
attr,
"%s%s." % (name_prefix, name),
dump_list=dump_list,
skip_reserved=skip_reserved,
column=column,
colon=colon,
name_map=name_map)
continue
if count != 1 and len(attr) == count:
ref = attr[0]
else:
ref = attr
conv = None
if isinstance(ref, IBA.GID) or isinstance(ref, IBA.GUID):
fmt = "%s"
else:
fmt = IBA.MEMBER_FORMATS.get(name, "%r")
if fmt == "hex":
fmt = "0x%%0%ux" % ((mbits + 3) // 4)
if fmt == "str":
fmt = "%s"
conv = lambda value: dstr(description(value), quotes=True)
if fmt == "gid_prefix":
fmt = "%s/64"
conv = lambda value: IBA.GID(prefix=value, guid=IBA.GUID(0))
if count != 1 and len(attr) == count and conv == None:
if isinstance(attr[0], rdma.binstruct.BinStruct):
for I, v in enumerate(attr):
struct_dotted(F,
v,
"%s%s[%u]." % (name_prefix, name, I),
dump_list=dump_list,
skip_reserved=skip_reserved,
column=column,
colon=colon,
name_map=name_map)
continue
if mbits > 16 or dump_list:
for I, v in enumerate(attr):
n = "%s%s[%u]" % (name_prefix, cname, I)
if colon:
n = n + ":"
if conv:
v = conv(v)
print >> F, ("%s%s" + fmt) % (n, "." *
(column - len(n)), v)
continue
else:
attr = "[%s]" % (", ".join(
("%u:" + fmt) % (I, v) for I, v in enumerate(attr)))
fmt = "%s"
n = "%s%s" % (name_prefix, cname)
if colon:
n = n + ":"
if conv:
attr = conv(attr)
print >> F, ("%s%s" + fmt) % (n, "." * (column - len(n)), attr)
