def send_reply(self,ofmt,payload,path,attributeModifier=0,
status=0,class_code=0):
"""Generate a reply packet. *ofmt* should be the request format."""
fmt = ofmt.__class__();
fmt.baseVersion = IBA.MAD_BASE_VERSION;
fmt.mgmtClass = fmt.MAD_CLASS;
fmt.classVersion = fmt.MAD_CLASS_VERSION;
fmt.status = (status & 0x1F) | ((class_code & IBA.MAD_STATUS_CLASS_MASK) << IBA.MAD_STATUS_CLASS_SHIFT)
if ofmt.method == IBA.MAD_METHOD_SET:
fmt.method = IBA.MAD_METHOD_GET_RESP
elif ofmt.method == IBA.MAD_METHOD_TRAP:
if status != 0:
return;
fmt.method = IBA.MAD_METHOD_TRAP_REPRESS;
else:
fmt.method = ofmt.method | IBA.MAD_METHOD_RESPONSE;
fmt.transactionID = ofmt.transactionID;
fmt.attributeID = ofmt.attributeID;
fmt.attributeModifier = attributeModifier;
if not isinstance(payload,type):
payload.pack_into(fmt.data);
buf = bytearray(fmt.MAD_LENGTH);
fmt.pack_into(buf);
path.reverse();
if self.trace_func is not None:
self.trace_func(self,TRACE_REPLY,fmt=fmt,path=path);
self.sendto(buf,path);
def send_reply(self,ofmt,payload,path,attributeModifier=0,
status=0,class_code=0):
"""Generate a reply packet. *ofmt* should be the request format."""
fmt = ofmt.__class__();
fmt.baseVersion = IBA.MAD_BASE_VERSION;
fmt.mgmtClass = fmt.MAD_CLASS;
fmt.classVersion = fmt.MAD_CLASS_VERSION;
fmt.status = (status & 0x1F) | ((class_code & IBA.MAD_STATUS_CLASS_MASK) << IBA.MAD_STATUS_CLASS_SHIFT)
if ofmt.method == IBA.MAD_METHOD_SET:
fmt.method = IBA.MAD_METHOD_GET_RESP
elif ofmt.method == IBA.MAD_METHOD_TRAP:
if status != 0:
return;
fmt.method = IBA.MAD_METHOD_TRAP_REPRESS;
else:
fmt.method = ofmt.method | IBA.MAD_METHOD_RESPONSE;
fmt.transactionID = ofmt.transactionID;
fmt.attributeID = ofmt.attributeID;
fmt.attributeModifier = attributeModifier;
if not isinstance(payload,type):
payload.pack_into(fmt.data);
buf = bytearray(fmt.MAD_LENGTH);
fmt.pack_into(buf);
path.reverse();
if self.trace_func is not None:
self.trace_func(self,TRACE_REPLY,fmt=fmt,path=path);
self.sendto(buf,path);
def _gen_error(self,buf,path):
"""Sadly the kernel can return EINVAL if it could not process the MAD,
eg if you ask for PortInfo of the local CA with an invalid attributeID
the Mellanox driver will return EINVAL rather than construct an error
MAD. I consider this to be a bug in the kernel, but we fix it here
by constructing an error MAD."""
buf = copy.copy(buf);
rmatch = self._get_reply_match_key(buf);
buf[3] = rmatch[1];
buf[4] = 0;
buf[5] = IBA.MAD_STATUS_INVALID_ATTR_OR_MODIFIER; # Guessing.
path = path.copy();
path.reverse();
return (buf,path);
def client_mode(hostname,infilename,dev):
f = open(infilename, "r+")
sz = os.path.getsize(infilename)
with Endpoint(f.fileno(), sz, dev) as end:
ret = socket.getaddrinfo(hostname,str(ip_port),0,
socket.SOCK_STREAM);
ret = ret[0];
with contextlib.closing(socket.socket(ret[0],ret[1])) as sock:
sock.connect(ret[4]);
path = rdma.path.IBPath(dev,SGID=end.ctx.end_port.default_gid);
rdma.path.fill_path(end.qp,path,max_rd_atomic=0);
path.reverse(for_reply=False);
sock.send(pickle.dumps(infotype(path=path,
addr=end.mr.addr,
rkey=end.mr.rkey,
size=end.mem.size(),
iters=1)))
buf = sock.recv(1024)
peerinfo = pickle.loads(buf)
end.path = peerinfo.path;
end.path.reverse(for_reply=False);
end.path.end_port = end.ctx.end_port;
print "path to peer %r\nMR peer raddr=%x peer rkey=%x"%(
end.path,peerinfo.addr,peerinfo.rkey);
end.connect(peerinfo)
# Synchronize the transition to RTS
sock.send("Ready");
sock.recv(1024);
end.rdma()
print "-- rmda end"
sock.shutdown(socket.SHUT_WR);
sock.recv(1024);
print "---client end"
print "---sock close"
print "--- endpoint close"
def _gen_error(self,buf,path):
"""Sadly the kernel can return EINVAL if it could not process the MAD,
eg if you ask for PortInfo of the local CA with an invalid attributeID
the Mellanox driver will return EINVAL rather than construct an error
MAD. I consider this to be a bug in the kernel, but we fix it here
by constructing an error MAD."""
buf = copy.copy(buf);
meth = buf[3];
if meth == IBA.MAD_METHOD_SET:
meth = IBA.MAD_METHOD_GET_RESP;
else:
meth = meth | IBA.MAD_METHOD_RESPONSE;
buf[3] = meth;
buf[4] = 0;
buf[5] = IBA.MAD_STATUS_INVALID_ATTR_OR_MODIFIER; # Guessing.
path = path.copy();
path.reverse();
return (buf,path);
def send_error_reply(self,buf,path,status,class_code=0):
"""Generate an error reply for a MAD. *buf* is the full original
packet. This entire packet is returned with an appropriate error code
set. *status* and *class_code* should be set to the appropriate result
code."""
hdr = _MADFormat(buf);
hdr.status = (status & 0x1F) | ((class_code & IBA.MAD_STATUS_CLASS_MASK) << IBA.MAD_STATUS_CLASS_SHIFT)
if hdr.method == IBA.MAD_METHOD_SET:
hdr.method = IBA.MAD_METHOD_GET_RESP
elif hdr.method == IBA.MAD_METHOD_TRAP:
return;
else:
hdr.method = hdr.method | IBA.MAD_METHOD_RESPONSE;
buf = bytearray(buf);
hdr.pack_into(buf);
path.reverse();
if self.trace_func is not None:
self.trace_func(self,TRACE_REPLY,fmt=hdr,path=path);
self.sendto(buf,path);
def send_error_reply(self,buf,path,status,class_code=0):
"""Generate an error reply for a MAD. *buf* is the full original
packet. This entire packet is returned with an appropriate error code
set. *status* and *class_code* should be set to the appropriate result
code."""
hdr = _MADFormat(buf);
hdr.status = (status & 0x1F) | ((class_code & IBA.MAD_STATUS_CLASS_MASK) << IBA.MAD_STATUS_CLASS_SHIFT)
if hdr.method == IBA.MAD_METHOD_SET:
hdr.method = IBA.MAD_METHOD_GET_RESP
elif hdr.method == IBA.MAD_METHOD_TRAP:
return;
else:
hdr.method = hdr.method | IBA.MAD_METHOD_RESPONSE;
buf = bytearray(buf);
hdr.pack_into(buf);
path.reverse();
if self.trace_func is not None:
self.trace_func(self,TRACE_REPLY,fmt=hdr,path=path);
self.sendto(buf,path);
def client_mode(hostname, opt, dev):
with Endpoint(opt, dev) as end:
ret = socket.getaddrinfo(hostname, str(opt.ip_port), opt.af,
socket.SOCK_STREAM)
ret = ret[0]
with contextlib.closing(socket.socket(ret[0], ret[1])) as sock:
if opt.debug >= 1:
print "Connecting to %r %r" % (ret[4][0], ret[4][1])
sock.connect(ret[4])
path = rdma.path.IBPath(dev, SGID=end.ctx.end_port.default_gid)
rdma.path.fill_path(end.qp, path, max_rd_atomic=0)
path.reverse(for_reply=False)
sock.send(
pickle.dumps(
infotype(path=path,
addr=end.mr.addr,
rkey=end.mr.rkey,
size=opt.size,
iters=opt.iters)))
buf = sock.recv(1024)
peerinfo = pickle.loads(buf)
end.path = peerinfo.path
end.path.reverse(for_reply=False)
end.path.set_end_port(end.ctx.node)
print "path to peer %r\nMR peer raddr=%x peer rkey=%x" % (
end.path.forward_path, peerinfo.addr, peerinfo.rkey)
print "%u iterations of %u is %u bytes" % (opt.iters, opt.size,
opt.iters * opt.size)
end.connect(peerinfo)
# Synchronize the transition to RTS
sock.send("Ready")
sock.recv(1024)
end.rdma()
sock.shutdown(socket.SHUT_WR)
sock.recv(1024)
def _gen_error(self, buf, path):
"""Sadly the kernel can return EINVAL if it could not process the MAD,
eg if you ask for PortInfo of the local CA with an invalid attributeID
the Mellanox driver will return EINVAL rather than construct an error
MAD. I consider this to be a bug in the kernel, but we fix it here
by constructing an error MAD."""
buf = copy.copy(buf)
meth = buf[3]
if meth == IBA.MAD_METHOD_SET:
meth = IBA.MAD_METHOD_GET_RESP
else:
meth = meth | IBA.MAD_METHOD_RESPONSE
buf[3] = meth
buf[4] = 0
buf[5] = IBA.MAD_STATUS_INVALID_ATTR_OR_MODIFIER
# Guessing.
path = path.copy()
path.reverse()
return (buf, path)
def send_rmpp_reply(self,ofmt,attrClass,payload,path,attributeModifier=0,
status=0,class_code=0):
"""Like send_reply, but generates an RMPP reply packet.
*ofmt* should be the request format.
*attrClass* is the class of the attribute, eg. IBA.SANodeRecord.
*payload* is a list or tuple of type attrClass, potentially with zero elements."""
fmt = ofmt.__class__();
hdrlen = fmt.MAD_LENGTH - len(fmt.data);
attrlen = attrClass.MAD_LENGTH + (8 - attrClass.MAD_LENGTH % 8);
buflen = hdrlen + len(payload)*attrlen;
fmt.baseVersion = IBA.MAD_BASE_VERSION;
fmt.mgmtClass = fmt.MAD_CLASS;
fmt.classVersion = fmt.MAD_CLASS_VERSION;
fmt.status = (status & 0x1F) | ((class_code & IBA.MAD_STATUS_CLASS_MASK) << IBA.MAD_STATUS_CLASS_SHIFT)
if ofmt.method == IBA.MAD_METHOD_SET:
fmt.method = IBA.MAD_METHOD_GET_RESP
else:
fmt.method = ofmt.method | IBA.MAD_METHOD_RESPONSE;
fmt.transactionID = ofmt.transactionID;
fmt.attributeID = ofmt.attributeID;
fmt.attributeModifier = attributeModifier;
fmt.attributeOffset = attrlen / 8;
fmt.RMPPFlags = IBA.RMPP_ACTIVE;
fmt.data1 = 1;
fmt.data2 = attrlen * len(payload);
buf = bytearray(max(buflen,fmt.MAD_LENGTH));
fmt.pack_into(buf);
offset = hdrlen;
for chunk in payload:
chunk.pack_into(buf, offset);
offset += attrlen;
if len(buf) > buflen:
del buf[buflen:];
path.reverse();
if self.trace_func is not None:
self.trace_func(self,TRACE_REPLY,fmt=fmt,path=path);
self.sendto(buf,path);
def send_rmpp_reply(self,ofmt,attrClass,payload,path,attributeModifier=0,
status=0,class_code=0):
"""Like send_reply, but generates an RMPP reply packet.
*ofmt* should be the request format.
*attrClass* is the class of the attribute, eg. IBA.SANodeRecord.
*payload* is a list or tuple of type attrClass, potentially with zero elements."""
fmt = ofmt.__class__();
hdrlen = fmt.MAD_LENGTH - len(fmt.data);
attrlen = attrClass.MAD_LENGTH + (8 - attrClass.MAD_LENGTH % 8);
buflen = hdrlen + len(payload)*attrlen;
fmt.baseVersion = IBA.MAD_BASE_VERSION;
fmt.mgmtClass = fmt.MAD_CLASS;
fmt.classVersion = fmt.MAD_CLASS_VERSION;
fmt.status = (status & 0x1F) | ((class_code & IBA.MAD_STATUS_CLASS_MASK) << IBA.MAD_STATUS_CLASS_SHIFT)
if ofmt.method == IBA.MAD_METHOD_SET:
fmt.method = IBA.MAD_METHOD_GET_RESP
else:
fmt.method = ofmt.method | IBA.MAD_METHOD_RESPONSE;
fmt.transactionID = ofmt.transactionID;
fmt.attributeID = ofmt.attributeID;
fmt.attributeModifier = attributeModifier;
fmt.attributeOffset = attrlen / 8;
fmt.RMPPFlags = IBA.RMPP_ACTIVE;
fmt.data1 = 1;
fmt.data2 = attrlen * len(payload);
buf = bytearray(max(buflen,fmt.MAD_LENGTH));
fmt.pack_into(buf);
offset = hdrlen;
for chunk in payload:
chunk.pack_into(buf, offset);
offset += attrlen;
if len(buf) > buflen:
del buf[buflen:];
path.reverse();
if self.trace_func is not None:
self.trace_func(self,TRACE_REPLY,fmt=fmt,path=path);
self.sendto(buf,path);
def client_mode(hostname,opt,dev):
with Endpoint(opt,dev) as end:
ret = socket.getaddrinfo(hostname,str(opt.ip_port),opt.af,
socket.SOCK_STREAM);
ret = ret[0];
with contextlib.closing(socket.socket(ret[0],ret[1])) as sock:
if opt.debug >= 1:
print "Connecting to %r %r"%(ret[4][0],ret[4][1]);
sock.connect(ret[4]);
path = rdma.path.IBPath(dev,SGID=end.ctx.end_port.default_gid);
rdma.path.fill_path(end.qp,path,max_rd_atomic=0);
path.reverse(for_reply=False);
sock.send(pickle.dumps(infotype(path=path,
addr=end.mr.addr,
rkey=end.mr.rkey,
size=opt.size,
iters=opt.iters)))
buf = sock.recv(1024)
peerinfo = pickle.loads(buf)
end.path = peerinfo.path;
end.path.reverse(for_reply=False);
end.path.end_port = end.ctx.end_port;
print "path to peer %r\nMR peer raddr=%x peer rkey=%x"%(
end.path,peerinfo.addr,peerinfo.rkey);
print "%u iterations of %u is %u bytes"%(opt.iters,opt.size,
opt.iters*opt.size);
end.connect(peerinfo)
# Synchronize the transition to RTS
sock.send("Ready");
sock.recv(1024);
end.rdma()
sock.shutdown(socket.SHUT_WR);
sock.recv(1024);
