def __init__(self, obj, dlog=None):
'''The StatsCollection object holds many statistics objects and
distributes the calls to them. It also simplifies the collection
of information when report() and cluster_report() are called.'''
self.obj = obj
self.c = utils.Bunch()
self.disable = False
self.methods = []
if dlog is None:
self.dlog = DataLog()
else:
self.dlog = dlog
def __init__(self,obj,dlog=None):
'''The StatsCollection object holds many statistics objects and
distributes the calls to them. It also simplifies the collection
of information when report() and cluster_report() are called.'''
self.obj = obj
self.c = utils.Bunch()
self.disable = False
self.methods = []
if dlog is None:
self.dlog = DataLog()
else:
self.dlog = dlog
class StatsCollection:
def __init__(self, obj, dlog=None):
'''The StatsCollection object holds many statistics objects and
distributes the calls to them. It also simplifies the collection
of information when report() and cluster_report() are called.'''
self.obj = obj
self.c = utils.Bunch()
self.disable = False
self.methods = []
if dlog is None:
self.dlog = DataLog()
else:
self.dlog = dlog
def start(self):
'''The start() method is called once per simulation'''
for m in self.methods:
m.connect(self)
m.start(self.c, self.obj)
m.clear(self.c, self.obj)
def clear(self):
'''The clear() method is called at the start of an epoch that
will be monitored'''
for m in self.methods:
m.clear(self.c, self.obj)
def add(self):
if self.disable:
return
for m in self.methods:
m.add(self.c, self.obj)
def _report(self):
'''report() is called at the end of an epoch and returns a list
of results in the form:
[(name,collection,value)]
where:
name = name of statistic
collection = how to communicate statistic when on a cluster
value = the value observed.
'''
l = []
for m in self.methods:
val = m.report(self.c, self.obj)
if isinstance(val, list):
for (name, collection, v) in val:
if v.size == 0:
continue
l.append((name, collection, v))
else:
if val.size == 0:
continue
l.append((m.name, m.collection, val))
return l
def single_report(self):
l = self._report()
for (name, coll, val) in l:
self.dlog.append(name, val)
def cluster_report(self, cluster):
'''Same intent as single_report(), but communicate data across
the cluster. The cluster variable that is passed needs to have
the following attribute:
cluster.NUMBER_OF_CORES
'''
rank = comm.rank
#Same logic from report()
l = self._report()
#Now we do cluster communication
#~ print 'cluster reporting for node ', rank
for (name, coll, val) in l:
if coll is "reduce":
#~ print 'Node',rank,'reducing',name
temp = comm.reduce(val)
temp = temp / cluster.NUMBER_OF_CORES
#~ print 'Node',rank,'reduced',name
if coll is "gather":
if rank == 0:
temp = empty((comm.size, ) + (prod(val.shape), ))
else:
temp = None
# for debugging
#~ print 'Node',rank,'gathering',name
#~ print name
#~ print shape(val)
#~ print shape(val.flatten())
comm.Gather(val.flatten(), temp, root=0)
if rank == 0:
temp = [temp[i].reshape(val.shape) for i in \
range(comm.size)]
#~ print 'Node',rank,'gathered',name
if coll is "gatherv": #Variable gather size
#~ print 'Node',rank,'gatherving',name
arrsizes = empty(cluster.NUMBER_OF_CORES, dtype=int)
arrsize = array(prod(val.shape))
comm.Allgather(sendbuf=[arrsize, MPI.LONG],
recvbuf=[arrsizes, MPI.LONG])
if comm.rank == 0:
temp = zeros(sum(arrsizes))
else:
temp = zeros(0)
comm.Gatherv([val.flatten(), (arrsize, None), MPI.DOUBLE],
[temp, (arrsizes, None), MPI.DOUBLE],
root=0)
#~ print 'Node',rank,'gathervd',name
if coll is "root":
if rank == 0:
temp = val
else:
temp = array([])
self.dlog.append(name, temp)
del temp #Delete temp to ensure failing if coll is unknown
class StatsCollection:
def __init__(self,obj,dlog=None):
'''The StatsCollection object holds many statistics objects and
distributes the calls to them. It also simplifies the collection
of information when report() and cluster_report() are called.'''
self.obj = obj
self.c = utils.Bunch()
self.disable = False
self.methods = []
if dlog is None:
self.dlog = DataLog()
else:
self.dlog = dlog
def start(self):
'''The start() method is called once per simulation'''
for m in self.methods:
m.connect(self)
m.start(self.c,self.obj)
m.clear(self.c,self.obj)
def clear(self):
'''The clear() method is called at the start of an epoch that
will be monitored'''
for m in self.methods:
m.clear(self.c,self.obj)
def add(self):
if self.disable:
return
for m in self.methods:
m.add(self.c,self.obj)
def _report(self):
'''report() is called at the end of an epoch and returns a list
of results in the form:
[(name,collection,value)]
where:
name = name of statistic
collection = how to communicate statistic when on a cluster
value = the value observed.
'''
l = []
for m in self.methods:
val = m.report(self.c,self.obj)
if isinstance(val, list):
for (name,collection,v) in val:
if v.size == 0:
continue
l.append((name,collection,v))
else:
if val.size==0:
continue
l.append((m.name,m.collection,val))
return l
def single_report(self):
l = self._report()
for (name,coll, val) in l:
self.dlog.append(name,val)
def cluster_report(self,cluster):
'''Same intent as single_report(), but communicate data across
the cluster. The cluster variable that is passed needs to have
the following attribute:
cluster.NUMBER_OF_CORES
'''
rank = comm.rank
#Same logic from report()
l = self._report()
#Now we do cluster communication
#~ print 'cluster reporting for node ', rank
for (name,coll, val) in l:
if coll is "reduce":
#~ print 'Node',rank,'reducing',name
temp = comm.reduce(val)
temp = temp/cluster.NUMBER_OF_CORES
#~ print 'Node',rank,'reduced',name
if coll is "gather":
if rank == 0:
temp = empty((comm.size,)+(prod(val.shape),))
else:
temp = None
# for debugging
#~ print 'Node',rank,'gathering',name
#~ print name
#~ print shape(val)
#~ print shape(val.flatten())
comm.Gather(val.flatten(), temp, root=0)
if rank == 0:
temp = [temp[i].reshape(val.shape) for i in \
range(comm.size)]
#~ print 'Node',rank,'gathered',name
if coll is "gatherv": #Variable gather size
#~ print 'Node',rank,'gatherving',name
arrsizes = empty( cluster.NUMBER_OF_CORES, dtype=int )
arrsize = array( prod(val.shape) )
comm.Allgather(sendbuf=[arrsize, MPI.LONG],
recvbuf=[arrsizes,MPI.LONG])
if comm.rank==0:
temp = zeros(sum(arrsizes))
else:
temp = zeros(0)
comm.Gatherv([val.flatten(),(arrsize, None),MPI.DOUBLE],
[temp, (arrsizes,None),MPI.DOUBLE],
root=0)
#~ print 'Node',rank,'gathervd',name
if coll is "root":
if rank == 0:
temp = val
else:
temp = array([])
self.dlog.append(name,temp)
del temp #Delete temp to ensure failing if coll is unknown
