def __init__(self,
nl,
map,
path,
fab_uuid=None,
grand_plan=None,
random_cids=False,
accept_cids=False,
conf=None,
mgr_uuid=None,
verbosity=0):
self.nl = nl
self.map = map
self.path = path
self.fabnum = component_num(path)
self.fab_uuid = fab_uuid
self.mgr_uuid = uuid4() if mgr_uuid is None else mgr_uuid
self.random_cids = random_cids
self.accept_cids = accept_cids
self.conf = conf
self.verbosity = verbosity
self.bridges = [] # indexed by bridge number
self.components = {} # key: comp.uuid
self.cuuid_serial = {} # key: cuuid:serial
self.comp_gcids = {} # key: comp.gcid
self.assigned_gcids = []
self.refill_gcids = True
self.nonce_list = [0]
self.routes = Routes(fab_uuid=fab_uuid)
self.resources = Resources(self)
super().__init__(fab_uuid=self.fab_uuid, mgr_uuids=[self.mgr_uuid])
log.info('fabric: {}, num={}, fab_uuid={}, mgr_uuid={}'.format(
path, self.fabnum, self.fab_uuid, self.mgr_uuid))
def iface_reset(self, key, br, sender, iface, pkt):
log.info('{}: {} from {} on {}'.format(br, key, sender, iface))
status = iface.phy_init() # get PHY status (no actual init)
state = iface.iface_state()
if not iface.usable:
self.iface_unusable(iface)
return {key: 'ok'}
def fab_init(self):
br_paths = self.path.glob('bridge*')
# Revisit: deal with multiple bridges that may or may not be on same fabric
for br_path in br_paths:
cuuid = get_cuuid(br_path)
serial = get_serial(br_path)
cuuid_serial = str(cuuid) + ':' + serial
cur_gcid = get_gcid(br_path)
brnum = component_num(br_path)
cclass = int(get_cclass(br_path))
tmp_gcid = cur_gcid if cur_gcid.sid == TEMP_SUBNET else INVALID_GCID
br = LocalBridge(cclass,
self,
self.map,
br_path,
self.mgr_uuid,
local_br=True,
brnum=brnum,
dr=None,
tmp_gcid=tmp_gcid,
netlink=self.nl,
verbosity=self.verbosity)
gcid = self.assign_gcid(br,
ssdt_sz=br.ssdt_size(haveCore=False)[0])
self.set_pfm(br)
log.info('{}:{} bridge{} {}'.format(self.fabnum, gcid, brnum,
cuuid_serial))
br.comp_init(self.pfm)
self.bridges.append(br)
br.explore_interfaces(self.pfm)
def new_peer_comp(self, key, br, sender, iface, pkt):
log.info('{}: {} from {} on {}'.format(br, key, sender, iface))
iup = iface.iface_init()
# Revisit: check iup
# find previous Component (if there is one)
prev_comp = iface.peer_comp
sender.explore_interfaces(
self.pfm,
ingress_iface=None, # Revisit
explore_ifaces=[iface],
prev_comp=prev_comp)
return {key: 'ok'}
def iface_error(self, key, br, sender, iface, pkt):
genz = zephyr_conf.genz
es = genz.IErrorES(pkt['ES'])
log.info('{}: {}:{}({}) from {} on {}'.format(br, key, es.errName,
es.errSeverity, sender,
iface))
status = iface.phy_init() # get PHY status (no actual init)
state = iface.iface_state()
# Revisit: Containment and RootCause
if not iface.usable:
self.iface_unusable(iface)
return {key: 'ok'}
def setup_routing(self,
fr: Component,
to: Component,
write_ssdt=True,
routes=None) -> List[Route]:
try:
cur_rts = self.routes.get_routes(fr, to)
except KeyError:
cur_rts = []
new_rts = []
for route in self.find_routes(fr,
to,
routes=routes,
max_routes=zephyr_conf.args.max_routes):
if route in cur_rts:
log.debug('skipping existing route {}'.format(route))
else:
log.info('adding route from {} to {} via {}'.format(
fr, to, route))
self.write_route(route, write_ssdt)
self.routes.add(fr, to, route)
new_rts.append(route)
return new_rts
def main():
global args
global cols
global genz
parser = argparse.ArgumentParser()
parser.add_argument('-k', '--keyboard', action='count', default=0,
help='break to interactive keyboard at certain points')
parser.add_argument('-v', '--verbosity', action='count', default=0,
help='increase output verbosity')
parser.add_argument('-A', '--accept-cids', action='store_true',
help='accept pre-existing HW CIDs for all components')
parser.add_argument('-r', '--reclaim', action='store_true',
help='reclaim C-Up components via reset')
parser.add_argument('-M', '--max-routes', action='store', default=None, type=int,
help='limit number of routes between components')
parser.add_argument('-R', '--random-cids', action='store_true',
help='generate random CIDs for all components')
parser.add_argument('-S', '--sleep', type=float, default=0.0,
help='sleep time inserted at certain points')
parser.add_argument('-G', '--genz-version', choices=['1.1'],
default='1.1',
help='Gen-Z spec version of Control Space structures')
parser.add_argument('-P', '--post_mortem', action='store_true',
help='enter debugger on uncaught exception')
args = parser.parse_args()
log.debug('Gen-Z version = {}'.format(args.genz_version))
genz = import_module('genz.genz_{}'.format(args.genz_version.replace('.', '_')))
zephyr_conf.init(args, genz)
args_vars = vars(args)
log.debug('args={}'.format(args_vars))
nl = NetlinkManager(config='./zephyr-fm/alpaka.conf')
map = genz.ControlStructureMap()
mgr_uuid = None # by default, generate new mgr_uuid every run
conf = Conf('zephyr-fm/zephyr-fabric.conf')
try:
data = conf.read_conf_file()
fab_uuid = UUID(data['fabric_uuid'])
if args.reclaim:
mgr_uuid = UUID(data['mgr_uuid'])
except FileNotFoundError:
# create new skeleton file
data = {}
fab_uuid = uuid4()
data['fabric_uuid'] = str(fab_uuid)
data['add_resources'] = []
data['boundary_interfaces'] = []
conf.write_conf_file(data)
log.debug('conf={}'.format(conf))
fabrics = {}
if args.keyboard > 3:
set_trace()
mainapp = FMServer(conf, 'zephyr', **args_vars)
thread = Thread(target=mainapp.run)
thread.start()
if args.keyboard > 3:
set_trace()
mp.set_start_method('forkserver')
uep_args = { 'genz_version': args.genz_version,
'verbosity': args.verbosity,
'url': 'http://localhost:2021/fabric/uep' }
uep_proc = mp.Process(target=netlink_reader, kwargs=uep_args)
uep_proc.start()
sys_devices = Path('/sys/devices')
fab_paths = sys_devices.glob('genz*')
for fab_path in sorted(fab_paths):
fab = Fabric(nl, map, fab_path, random_cids=args.random_cids,
accept_cids=args.accept_cids, fab_uuid=fab_uuid,
conf=conf, mgr_uuid=mgr_uuid, verbosity=args.verbosity)
fabrics[fab_path] = fab
conf.set_fab(fab)
if args.keyboard > 1:
set_trace()
fab.fab_init()
log.info('finished exploring fabric {}'.format(fab.fabnum))
if len(fabrics) == 0:
log.info('no local Gen-Z bridges found')
return
if args.keyboard > 3:
set_trace()
conf.add_resources() # Revisit: multiple fabrics
if args.keyboard > 3:
set_trace()
thread.join()
def iface_init(self, prefix='control'):
genz = zephyr_conf.genz
self.setup_paths(prefix)
iface_file = self.iface_dir / 'interface'
is_switch = self.comp.has_switch
with iface_file.open(mode='rb+') as f:
data = bytearray(f.read())
iface = self.comp.map.fileToStruct('interface',
data,
fd=f.fileno(),
verbosity=self.comp.verbosity)
log.debug('{}: iface_init interface{}={}'.format(
self.comp.gcid, self.num, iface))
self.hvs = iface.HVS
if not self.phy_init():
log.info('{}: interface{} is not PHY-Up'.format(
self.comp.gcid, self.num))
self.usable = False
# Revisit: should config iface even if not PHY-Up
return False
icap1 = genz.ICAP1(iface.ICAP1, iface)
self.ierror_init(iface, icap1)
# Revisit: select compatible LLR/P2PNextHdr/P2PEncrypt settings
# Revisit: set CtlOpClassPktFiltEnb, if Switch (for now)
# enable Explicit OpCodes, and LPRT (if Switch)
icap1ctl = genz.ICAP1Control(iface.ICAP1Control, iface)
icap1ctl.field.OpClassSelect = 0x1
icap1ctl.field.LPRTEnb = is_switch
iface.ICAP1Control = icap1ctl.val
log.debug('{}: writing ICAP1Control'.format(self))
self.comp.control_write(iface,
genz.InterfaceStructure.ICAP1Control,
sz=4,
off=4)
# set LinkCTLControl (depending on local_br, is_switch)
lctl = genz.LinkCTLControl(iface.LinkCTLControl, iface)
# xmit bits set on local_br and all switch ports
xmit = 1 if (self.comp.local_br or is_switch) else 0
lctl.field.XmitPeerCUpEnb = xmit
lctl.field.XmitPeerCResetEnb = xmit
lctl.field.XmitPeerEnterLinkUpLPEnb = xmit
lctl.field.XmitPeerEnterLinkLPEnb = xmit
lctl.field.XmitLinkResetEnb = xmit
# Revisit: recv bits should be set everywhere except on local_br
# or "hostile" sw ports
recv = 0 if (self.comp.local_br) else 1
lctl.field.RecvPeerCUpEnb = recv
lctl.field.RecvPeerCResetEnb = recv
lctl.field.RecvPeerEnterLinkUpLPEnb = recv
lctl.field.RecvPeerEnterLinkLPEnb = recv
lctl.field.RecvLinkResetEnb = recv
iface.LinkCTLControl = lctl.val
log.debug('{}: writing LinkCTLControl'.format(self))
self.comp.control_write(iface,
genz.InterfaceStructure.LinkCTLControl,
sz=4,
off=4)
# send Peer-Attribute 1 Link CTL - HW did this at link-up time,
# but we don't know when that was, and things may have changed
status = self.send_peer_attr1(iface, timeout=100000)
if status == 0:
log.warning(
'{}: send_peer_attr1 timeout on interface{}'.format(
self.comp.gcid, self.num))
# Revisit: path time does not currently work in HW
# send Path Time Link CTL
#status = self.send_path_time(iface, timeout=100000)
#if status == 0:
# log.warning('{}: send_path_time timeout on interface{}'.format(
# self.comp.gcid, self.num))
# save PeerInterfaceID
self.peer_iface_num = self.get_peer_iface_num(iface)
ictl = genz.IControl(iface.IControl, iface)
# set IfaceAKeyValidationEnb (if supported)
ictl.field.IfaceAKeyValidationEnb = (
1 if icap1.field.IfaceAKeyValidationSup else 0)
# Revisit: set Ingress/Egress AKeyMask
# Revisit: set IngressDREnb only when needed
ictl.field.IngressDREnb = 1
# enable interface
ictl.field.IfaceEnb = 1
iface.IControl = ictl.val
log.debug('{}: writing IControl IfaceEnb'.format(self))
self.comp.control_write(iface,
genz.InterfaceStructure.IControl,
sz=4,
off=4)
# clear IStatus RW1C bits that we might care about later
istatus = genz.IStatus(0, iface) # all 0 IStatus
istatus.field.FullIfaceReset = 1
istatus.field.WarmIfaceReset = 1
istatus.field.LinkRFCStatus = 1
istatus.field.PeerLinkRFCReady = 1
istatus.field.ExceededTransientErrThresh = 1
istatus.field.LUpToLLPTransitionFailed = 1
istatus.field.IfaceContainment = 1
iface.IStatus = istatus.val
log.debug('{}: writing IStatus, val={:#x}'.format(
self, istatus.val))
self.comp.control_write(iface,
genz.InterfaceStructure.IStatus,
sz=4,
off=0)
# verify I-Up
state = self.check_i_state(iface)
self.usable = (state is IState.IUp)
# Revisit: orthus goes I-Down if we do this earlier
# set LinkRFCDisable (depending on local_br)
ictl.field.LinkRFCDisable = 1 if self.comp.local_br else 0
iface.IControl = ictl.val
log.debug('{}: writing IControl LinkRFCDisable'.format(self))
self.comp.control_write(iface,
genz.InterfaceStructure.IControl,
sz=4,
off=4)
# save PeerCState & PeerGCID
self.peer_cstate = self.get_peer_cstate(iface)
self.peer_gcid = self.get_peer_gcid(iface)
# end with
if is_switch:
# initialize VCAT
# Revisit: multiple Action columns
log.debug('{}: writing VCAT'.format(self))
for vc in range(0, self.hvs + 1):
# Revisit: vc policies
self.vcat_write(vc, (1 << vc))
log.debug('{}: iface_init done'.format(self))
return self.usable