def _unpack_rcv(self): """Switch a UMAD AH back into an IBPath. Note this is only used for recv'd AH's where the meaning of the fields is altered. Our convention is that the path describes the packet headers as they existed on the wire, so this untwiddles things.""" (sqpn, qkey, SLID, self.SL, DLID_bits, self.has_grh, DGID_index, self.hop_limit, self.traffic_class, self.SGID, flow_label, pkey_index) = \ UMAD.ib_mad_addr_t.unpack(self._cached_umad_ah) self.sqpn = cpu_to_be32(sqpn) # There is no pkey validation for SMPs (see IBA figure 156), so the # pkey should always be the default NOTE: mtcha at least has been seen # to return random values for pkey_index on SMPs, which is why we need # this check. if self.dqpn != 0: self.pkey_index = pkey_index self.qkey = cpu_to_be32(qkey) self.DLID = DLID_bits | self.end_port.lid self.SLID = cpu_to_be16(SLID) if self.has_grh: self.SGID = IBA.GID(self.SGID, True) self.DGID = self.end_port.gids[DGID_index] self.flow_label = cpu_to_be32(flow_label) else: del self.SGID
def set_mad_attr(attr, name, v): try: # Need to use eval because name could have dots in it. arg = eval("attr.%s" % (name)) except AttributeError: raise CmdError("%r is not a valid attribute for %r" % (name, attr)) try: if isinstance(arg, int) or isinstance(arg, long): v = int(v, 0) elif isinstance(arg, IBA.GID): v = IBA.GID(v) elif isinstance(arg, IBA.GUID): v = IBA.GUID(v) elif isinstance(arg, bytearray): v = v.decode("string_escape") if len(v) > len(arg): raise CmdError("String %r is too long, can only be up to %u" % (v, len(arg))) if len(v) < len(arg): v = v + bytearray(len(arg) - len(v)) elif isinstance(arg, list): raise CmdError("Lists currently cannot be set.") else: raise CmdError("Internal Error, I don't know what %s %r is." % (type(arg), arg)) except ValueError as err: raise CmdError("String %r did not parse: %s" % (v, err)) exec "attr.%s = v" % (name)
def test_basic(self): print self.ctx.query_port(); print self.ctx.query_device(); pd = self.ctx.pd(); print pd,repr(pd) cq = self.ctx.cq(100); print cq,repr(cq) try: cq.resize(200); except rdma.SysError as e: if e.errno != errno.ENOSYS: raise; self.assertEqual(cq.poll(),[]); comp = self.ctx.comp_channel(); print comp,repr(comp) qp = pd.qp(ibv.IBV_QPT_UD,100,cq,100,cq); print qp,repr(qp) print qp.query(0xFFFF); mpath = rdma.path.IBPath(self.ctx.end_port,DLID=0xC000, DGID=IBA.GID("ff02::1")); qp.attach_mcast(mpath); qp.detach_mcast(mpath); buf = mmap.mmap(-1,4096); mr = pd.mr(buf,ibv.IBV_ACCESS_LOCAL_WRITE|ibv.IBV_ACCESS_REMOTE_WRITE); print mr,repr(mr) print "MR",mr.addr,mr.length,mr.lkey,mr.rkey self.assertRaises(TypeError,pd.ah,None); ah = pd.ah(self.end_port.sa_path); print ah,repr(ah) srq = pd.srq(); print srq,repr(srq) print srq.query(); srq.modify(100);
def cmd_ibaddr(argv,o): """Display the GID and LID addresses for end ports. Usage: %prog [-glL] [TARGET]""" o.add_option("-l","--lid_show",action="store_true",dest="lid", help="Show LID information"); o.add_option("-L","--Lid_show",action="store_true",dest="lid", help="Show LID information"); o.add_option("-g","--gid_show",action="store_true",dest="gid", help="Show GID information"); LibIBOpts.setup(o); (args,values) = o.parse_args(argv); lib = LibIBOpts(o,args,values,1,(tmpl_target,)); if not values: values = (None,); if args.lid is None and args.gid is None: args.lid = True; args.gid = True; with lib.get_umad_for_target(values[0]) as umad: path = lib.path; ninf = umad.SubnGet(IBA.SMPNodeInfo,path); path.DGID = IBA.GID(prefix=IBA.GID_DEFAULT_PREFIX,guid=ninf.portGUID); pinf = umad.SubnGet(IBA.SMPPortInfo,path,0); if args.gid: print "GID %s"%(path.DGID), if args.lid: print "LID start %u end %u"%(pinf.LID,pinf.LID + (1 << pinf.LMC)-1), print return lib.done();
def test_fail(self): """Test valid get_end_port calls that fail.""" devices = rdma.get_devices() dev = devices.first() self.assertRaises(rdma.RDMAError, rdma.get_end_port, IBA.GID("::")) self.assertRaises(rdma.RDMAError, rdma.get_end_port, IBA.GUID("0000:0000:0000:0000")) self.assertRaises(rdma.RDMAError, rdma.get_end_port, "xxx") self.assertRaises(rdma.RDMAError, rdma.get_end_port, "%s/99" % (dev.name))
def ninf(self): try: return self.__dict__["ninf"]; except KeyError: pass ninf = IBA.ComponentMask(IBA.SANodeRecord()); ninf.LID = self.path.DLID; ninf = self.umad.SubnAdmGet(ninf); self.__dict__["ninf"] = ninf; if self.path.DGID is None: self.path.DGID = IBA.GID(prefix=IBA.GID_DEFAULT_PREFIX, guid=ninf.nodeInfo.portGUID); return ninf;
def unpack_from(self, buffer, offset=0): from socket import htonl as cpu_to_be32 from socket import htons as cpu_to_be16 (self.agent_id,self.status,self.timeout_ms,self.retries,self.length, self.qpn,self.qkey,self.lid,self.sl,self.path_bits,self.grh_present,self.gid_index, self.hop_limit,self.traffic_class,self.gid,self.flow_label,self.pkey_index,self.reserved_58) = \ struct.unpack_from("=LLLLLLLHBBBBBB16sLH6s",buffer,offset+0) self.qpn = cpu_to_be32(self.qpn) self.qkey = cpu_to_be32(self.qkey) self.lid = cpu_to_be16(self.lid) self.gid = IBA.GID(self.gid, raw=True) self.flow_label = cpu_to_be32(self.flow_label)
def get_end_port(name=None): """Return a :class:`rdma.devices.EndPort` for the default end port if name is ``None``, or for the end port described by name. The end port string format is one of: =========== =================== Format Example =========== =================== device mlx4_0 (defaults to the first port) device/port mlx4_0/1 Port GID fe80::2:c903:0:1491 Port GUID 0002:c903:0000:1491 =========== =================== :rtype: :class:`rdma.devices.EndPort` :raises rdma.RDMAError: If no matching device is found or name is invalid.""" devices = get_devices() if len(devices) == 0: raise RDMAError("No RDMA devices found.") if name is None: return devices.first().end_ports.first() # Try for a port GID import rdma.devices import rdma.IBA try: gid = IBA.GID(name) except ValueError: pass else: return rdma.devices.find_port_gid(devices, gid)[0] # Port GUID try: guid = IBA.GUID(name) except ValueError: pass else: return rdma.devices.find_port_guid(devices, guid) # Device name string return rdma.devices.find_port_name(devices, name)
def link_end_port(self, port, portIdx=None, nodeGUID=None, portGUID=None, path=None, LID=None, LMC=0): """Use the provided information about *port* to update the database. Note: For switches *portIdx* must be 0.""" assert (port == port.to_end_port()) if (LID is None and path is not None and not isinstance(path, rdma.path.IBDRPath)): LID = path.DLID node = port.parent if portIdx is not None: node.set_port(portIdx, port) if portGUID is not None and port.portGUID is None: port.portGUID = portGUID self.ports[portGUID] = port if LID is not None: port.LID = LID if LMC is None: LMC = 0 self.set_max_lid(LID + (1 << LMC) - 1) for I in IBA.lid_lmc_range(LID, LMC): self.lids[I] = port if path is not None: path._cached_subnet_end_port = port # Since we know it, record the DGID into the path. This produces # error messages that include the DGID.. if portGUID is not None and path.DGID is None: path.DGID = IBA.GID(prefix=IBA.GID_DEFAULT_PREFIX, guid=portGUID) if self.paths is not None: self.paths[port] = path return port
def _do_loop_test_mc(self): """Test HCA loop back between two QPs as well as SRQ.""" qp_type = ibv.IBV_QPT_UD; print "Testing QP to QP loop type %u UD MULTICAST"%(qp_type) with self.ctx.pd() as pd: path_a,qp_a,path_b,qp_b,poller,srq,pool = \ self._get_loop(pd,qp_type); # Note: since both our QPs are on the same end port then the DLID # does not matter as far as forwarding is concerned, so the HCA # should replicate entirely based on the DLID. mcpath = path_b.copy(DGID=IBA.GID("FF02::1"),DLID=0xC000,dqpn=0xFFFFFF, traffic_class=0x89, flow_label=0x1234, hop_limit=23, has_grh=True); qp_a.attach_mcast(mcpath); qp_b.attach_mcast(mcpath); qp_b.post_send(pool.make_send_wr(pool.pop(),256,mcpath)); qp_a.post_send(pool.make_send_wr(pool.pop(),256,mcpath)); recvs = 0; sends = 0 for wc in poller.iterwc(count=6,timeout=0.5): if wc.opcode & ibv.IBV_WC_RECV: recvs = recvs + 1 path = ibv.WCPath(mcpath.end_port,wc, pool._mem, (wc.wr_id & pool.BUF_ID_MASK)*pool.size); self.assertEquals(path.DGID,mcpath.DGID); self.assertEquals(path.SGID,mcpath.SGID); self.assertEquals(path.flow_label,mcpath.flow_label); self.assertEquals(path.traffic_class,mcpath.traffic_class); self.assertEquals(path.hop_limit,mcpath.hop_limit); if wc.opcode == ibv.IBV_WC_SEND: sends = sends + 1 pool.finish_wcs(srq,wc); self.assertFalse(poller.timedout); self.assertEquals(recvs,4); self.assertEquals(sends,2);
def test_str_good(self): """Check good input to :func:`rdma.path.from_string`.""" self.check_path(None, DGID=IBA.GID("fe80::1")) self.check_path(None, DGID=IBA.GID("fe80::1"), SL=10, pkey=0xff) self.check_path(None, DGID=IBA.GID("fe80::1"), SLID=0x12, DLID=15) self.check_path_str(None, "fe80::1", DGID=IBA.GID("fe80::1")) self.check_path_str(None, "0:0:0:1", DGID=IBA.GID("fe80::1")) self.check_path_str(None, "1", DLID=1) self.check_path_str(None, "0xff", DLID=0xff) self.check_path_str(None, "0,1", cls=rdma.path.IBDRPath, drPath="\0\1") self.check_path_str(None, "0,", cls=rdma.path.IBDRPath, drPath="\0") for I in rdma.get_devices(): for J in I.end_ports: self.check_path_str(J, "fe80::1%%%s" % (J), DGID=IBA.GID("fe80::1")) for G in J.gids: if int(G) >> 64 != IBA.GID_DEFAULT_PREFIX: self.check_path_str(J, "%s" % (G), DGID=G)
def from_string(s,default_end_port=None,require_dev=None,require_ep=None): """Convert the string *s* into an instance of :class:`Path` or derived. Supported formats for *s* are: =========== ============================ ======================== Format Example Creates =========== ============================ ======================== Port GID fe80::2:c903:0:1491 IBPath.DGID = s Scope'd GID fe80::2:c903:0:1491%mlx4_0/1 IBPath.DGID = s Port GUID 0002:c903:0000:1491 IBPath.DGID = fe80:: + s LID 12 IBPath.DLID = 12 Hex LID 0xc IBPath.DLID = 12 DR Path 0,1, IBDRPath.drPath = '\\\\0\\\\1' Path Spec IBPath(DLID=2,SL=2) IBPath.{DLID=2,SL=2} =========== ============================ ======================== If the format unambiguously specifies an end port, eg due to a provided scope or by specifying the subnet prefix then the result will have `end_port` set appropriately. Otherwise `end_port` is set to `default_end_port`. *require_dev* and *require_ep* will restrict the lookup to returning a path for those conditions. If a scoped address is given that doesn't match then :exc:`ValueError` is raised. These options should be used when a path is being parsed for use with an existing bound resource (eg a :class:`rdma.ibverbs.Context` or :class:`rdma.ibverbs.`) FUTURE: This may return paths other than IB for other technologies. :raises ValueError: If the string can not be parsed.""" if require_ep is not None: default_end_port = require_ep; if s.find("(") != -1: ret = from_spec_string(s); if ret.end_port is None: ret.end_port = default_end_port; else: _check_ep(ret.end_port,require_dev,require_ep); return ret; dr = s.split(","); if len(dr) != 1: if dr[-1] == '': dr = [int(I) for I in dr[:-1]]; else: dr = [int(I) for I in dr]; for I in dr: if I >= 255: raise ValueError("Invalid DR path specification %r"%(s,)); if len(dr) == 0: raise ValueError("Invalid DR path specification %r"%(s,)); if dr[0] != 0: raise ValueError("Invalid DR path specification %r"%(s,)); drPath = bytes("").join("%c"%(I) for I in dr); return IBDRPath(default_end_port,drPath=drPath); a = s.split('%'); if len(a) == 2: DGID = IBA.GID(a[0]) try: end_port = rdma.get_end_port(a[1]); _check_ep(end_port,require_dev,require_ep); except rdma.RDMAError, e: raise ValueError("Could not find %r: %s"%(a[1],e)); return IBPath(end_port,DGID=DGID);
def from_spec_string(s): """Construct a *Path* (or derived) instance from it's `repr` string. This parser is safe to use with untrusted data. :raises ValueError: If the string can not be parsed.""" import re,itertools; m = re.match("^(.+?)\(\s*?(.*?)\s*?\)$",s); if m is None: raise ValueError("Invalid path specification %r"%(s,)); m = m.groups(); cls = getattr(sys.modules[__name__],m[0],None) if cls is None or not issubclass(cls,Path): raise ValueError("Invalid path specification %r, bad path type"%(s,)); kwargs = dict((t[0].strip(), t[2].strip()) for t in (I.partition('=') for I in m[1].split(','))); if len(kwargs) < 1: raise ValueError("Invalid path specification %r"%(s,)); for k,v in kwargs.iteritems(): if v == '': raise ValueError("Invalid path specification %r"%(s,)); if not hasattr(cls,k): raise ValueError("Path attribute %r is not known"%(k,)); if v.startswith("GID("): v = v[4:-1]; if v[0] == '"' or v[0] == "'": v = v[1:-1]; kwargs[k] = IBA.GID(v); elif k.find("GID") != -1: kwargs[k] = IBA.GID(v); elif k == "drPath": # Using : because I am too lazy to fix the splitter to respect quotes. dr = v.split(":"); if len(dr) == 1: raise ValueError("Invalid DR path specification %r"%(v,)); if dr[-1] == '': dr = [int(I) for I in dr[:-1]]; else: dr = [int(I) for I in dr]; for I in dr: if I >= 255: raise ValueError("Invalid DR path specification %r"%(v,)); if len(dr) == 0: raise ValueError("Invalid DR path specification %r"%(s,)); if dr[0] != 0: raise ValueError("Invalid DR path specification %r"%(s,)); kwargs[k] = bytes("").join("%c"%(I) for I in dr); elif k == "end_port": if v[0] == '"' or v[0] == "'": v = v[1:-1]; try: if v == "None": kwargs[k] = None; else: kwargs[k] = rdma.get_end_port(v); except rdma.RDMAError, e: raise ValueError("Could not find %r: %s"%(v,e)); else: try: kwargs[k] = int(v,0); except ValueError: raise ValueError("%r=%r is not a valid integer"%(k,v));
def _conv_gid2guid(s): """Return the GUID portion of a GID string. :raises ValueError: If the string is invalid.""" return IBA.GID(s).guid()
def test_gid(self): """Test IBA.GID class""" pg = IBA.GID("fe80::21c:23ff:fee2:946a") self.assertEquals(pg, pg) self.assertEquals(pg, IBA.GID("fe80::21c:23ff:fee2:946a")) self.assertEquals(pg, IBA.GID(pg))
def struct_dotted(F, s, name_prefix='', dump_list=False, skip_reserved=True, column=33, colon=False, name_map=None): """This tries to emulate the libib structure print format. Members are printed one per line with values aligned on column 32.""" for name, mbits, count in s.MEMBERS: if skip_reserved and name.startswith("reserved_"): continue attr = getattr(s, name) if attr is None: continue cname = name[0].upper() + name[1:] if name_map: cname = name_map.get(cname, cname) # Special automagic decode of format data members based on # attribute ID. if name == "data" and isinstance(s, rdma.binstruct.BinFormat): nattr = IBA.ATTR_TO_STRUCT.get((s.__class__, s.attributeID)) if nattr != None: if nattr.MAD_LENGTH <= len(attr): attr = nattr(attr) if isinstance(attr, rdma.binstruct.BinStruct): struct_dotted(F, attr, "%s%s." % (name_prefix, name), dump_list=dump_list, skip_reserved=skip_reserved, column=column, colon=colon, name_map=name_map) continue if count != 1 and len(attr) == count: ref = attr[0] else: ref = attr conv = None if isinstance(ref, IBA.GID) or isinstance(ref, IBA.GUID): fmt = "%s" else: fmt = IBA.MEMBER_FORMATS.get(name, "%r") if fmt == "hex": fmt = "0x%%0%ux" % ((mbits + 3) // 4) if fmt == "str": fmt = "%s" conv = lambda value: dstr(description(value), quotes=True) if fmt == "gid_prefix": fmt = "%s/64" conv = lambda value: IBA.GID(prefix=value, guid=IBA.GUID(0)) if count != 1 and len(attr) == count and conv == None: if isinstance(attr[0], rdma.binstruct.BinStruct): for I, v in enumerate(attr): struct_dotted(F, v, "%s%s[%u]." % (name_prefix, name, I), dump_list=dump_list, skip_reserved=skip_reserved, column=column, colon=colon, name_map=name_map) continue if mbits > 16 or dump_list: for I, v in enumerate(attr): n = "%s%s[%u]" % (name_prefix, cname, I) if colon: n = n + ":" if conv: v = conv(v) print >> F, ("%s%s" + fmt) % (n, "." * (column - len(n)), v) continue else: attr = "[%s]" % (", ".join( ("%u:" + fmt) % (I, v) for I, v in enumerate(attr))) fmt = "%s" n = "%s%s" % (name_prefix, cname) if colon: n = n + ":" if conv: attr = conv(attr) print >> F, ("%s%s" + fmt) % (n, "." * (column - len(n)), attr)