def NNClassifier(dataset, options, method='dtw'):
classnum = dataset.classnum
k_pool = [1]
k_num = len(k_pool)
Acc = np.zeros(k_num)
#dtw_knn_map = zeros(vidsetnum,)
tic = time.time()
# Macro = np.zeros(testsetdatanum)
# Micro = np.zeros(testsetdatanum)
# rightnum = np.zeros(k_num)
Macro = Value(ctypes.c_float * dataset.testsetdatanum)
Micro = Value(ctypes.c_float * dataset.testsetdatanum)
rightnum = Value(ctypes.c_uint * k_num)
cpu_count = os.cpu_count()
task_per_cpu = dataset.testsetdatanum // (cpu_count - 1) + 1
inner_process = [
Process(target=knn,
args=(i, task_per_cpu, k_num, k_pool, dataset, options, Macro,
Micro, rightnum)) for i in range(cpu_count - 1)
]
for p in inner_process:
p.start()
for p in inner_process:
p.join()
rightnum = np.ctypeslib.as_array(rightnum.get_obj())
Acc = rightnum / dataset.testsetdatanum
Macro = np.ctypeslib.as_array(Macro.get_obj())
Micro = np.ctypeslib.as_array(Micro.get_obj())
macro = np.mean(Macro)
micro = np.mean(Micro)
knn_time = time.time() - tic
knn_averagetime = knn_time / dataset.testsetdatanum
# logger.info(vars())
return macro, micro, Acc, knn_averagetime
def GetHists(self, N=1, use_custom_tqdm=True, file_cache=None):
self.pre_execution()
_, con_width = map(int,os.popen('stty size', 'r').read().split())
# if user wants to cache histograms in file, then make sure
# the hash of all the queued draws + list of all files in tchain
# + all the aliases and their values
# matches what was in the file
# In principle, if those match, then results must be identical
file_hash = hash("".join(sorted([x.GetTitle() for x in (self.ch.GetListOfFiles())])))
# FIXME on osx, the below line will cause a crash after looping when an alias is used
# does something special happen when calling GetAlias() or GetListOfAliases() before running?
if os.uname()[0] == "Darwin":
alias_hash = 1
else:
alias_hash = hash(tuple(sorted([(x.GetName(),self.ch.GetAlias(x.GetName())) for x in (self.ch.GetListOfAliases() or [])])))
queue_hash = hash(tuple(map(tuple,sorted(self.queued))))
total_hash = hash(queue_hash+file_hash+alias_hash)
if file_cache and os.path.exists(file_cache):
with open(file_cache,"r") as fh:
data = pickle.load(fh)
hash_cache = data["hash"]
if hash_cache == total_hash:
return data["hists"]
# take list of dicts of histname:histogram items
# and reduce them by adding and removing the prefix up to the first
# underscore
def reduce_hists(dicts):
d_master = {}
for idx,d in enumerate(dicts):
for hname, h in d.items():
if not h: continue
stripped_name = hname.split("_",1)[-1]
if stripped_name not in d_master:
d_master[stripped_name] = h.Clone(stripped_name)
else:
d_master[stripped_name].Add(h)
return d_master
# take variety of things and put histograms from loop into a queue
def get_hists(q, ch, queued, num, first, numentries, done, total, bytesread):
prefix = "md{}_".format(num)
clone = ch.Clone("ch_{}".format(prefix))
md = BaseTChain(clone)
for x_ in queued:
x = x_[:]
x[1] = "{}{}".format(prefix,x[1])
md.queue(*x)
md.execute_parallel(first,numentries,done.get_obj(),total.get_obj(), bytesread.get_obj())
hists = {}
hist_names = map(lambda x:x[1].split("(",1)[0], queued)
for hn in set(hist_names):
hists[hn] = r.gDirectory.Get(prefix+hn)
q.put(hists)
return 0
# compute event splitting for N jobs
first, last = 0, int(self.ch.GetEntries())
size = int((last-first) / N)
firsts = [first+i*size for i in range(N)] + [last]
diffs = map(lambda x: x[1]-x[0],zip(firsts[:-1],firsts[1:]))
firsts_and_nentries = zip(firsts[:-1],diffs)
if use_custom_tqdm:
r.gROOT.ProcessLine(".L {}/tqdm.h".format(os.path.realpath(__file__).rsplit("/",1)[0]))
bar = r.tqdm()
os.nice(10)
q = Queue(N)
dones, totals, bytess, workers = [], [], [], []
for i, (first, numentries) in enumerate(firsts_and_nentries):
done, total, bytesread = Value("i",0), Value("i",0), Value("i",0)
worker = Process(target=get_hists, args=[q,self.ch,self.queued,i,first,numentries,done,total,bytesread])
workers.append(worker)
worker.start()
dones.append(done.get_obj())
bytess.append(bytesread.get_obj())
totals.append(total.get_obj())
def get_sum(cs):
return sum(map(lambda x:x.value, cs))
try:
ioq = TimedQueue(N=30)
if use_custom_tqdm:
total = last
done = get_sum(dones)
bytesread = get_sum(bytess)
while done < total:
done = get_sum(dones)
bytesread = get_sum(bytess)
ioq.add_val(1.0*bytesread/1e6)
bar.progress(done,total,True)
which_done = map(lambda x:(x[0].value==x[1].value)and(x[0].value>0), zip(dones,totals))
if con_width > 110:
label = "[{:.1f}MB @ {:.1f}MB/s]".format(ioq.get_last_val(),ioq.get_rate())
label += " [{}]".format("".join(map(lambda x:unichr(0x2022) if x else unichr(0x2219),which_done)).encode("utf-8"))
else:
label = ""
bar.set_label(label)
time.sleep(0.04)
label = "[{:.1f}MB @ {:.1f}MB/s]".format(ioq.get_last_val(),ioq.get_rate())
label += " [{}]".format("".join([unichr(0x2022) for _ in dones]).encode("utf-8"))
bar.set_label(label)
bar.progress(total,total,True)
else:
from tqdm import tqdm
total = last
prev_done = get_sum(dones)
done = get_sum(dones)
with tqdm(total=total) as pbar:
while done < total-1:
done = get_sum(dones)
update = done - prev_done
prev_done = done
pbar.update(update)
time.sleep(0.1)
except KeyboardInterrupt as e:
print("[!] Early keyboard interruption...continuing with histograms anyway")
dicts = []
for iw in range(len(workers)):
dicts.append(q.get())
for worker in workers:
worker.join()
# don't let one tqdm bar clobber another
print
reduced_hists = reduce_hists(dicts)
# if user wants to cache histograms in file,
# then dump the hists as well as a hash
if file_cache:
with open(file_cache,"w") as fh:
data = {"hash": total_hash, "hists": reduced_hists}
pickle.dump(data,fh)
self.executed = True
return reduced_hists
class SharedObject:
"""
This is a wrapper for Value() that implements lots of ease-of-port features and allows for subclassing into "types"
Value() itself is a function that generates an object which cannot be inherited easily – hence, a wrapper is used
"""
NULL = object()
def __init__(self, type_, value, lock=True):
self.type = type_
if value != self.NULL:
self._cobj = Value(type_, value, lock=lock)
else:
self._cobj = Value(type_, lock=lock)
def __repr__(self):
return "<Shared %s=%s object at %s>" % (self.__class__.__name__,
self.value, hex(id(self)))
@property
def lock(self):
return self._cobj.get_lock()
@property
def cobj(self):
return self._cobj.get_obj()
@property
def value(self):
return self._cobj.value
@value.setter
def value(self, other):
with self.lock:
self._cobj.value = other
def set(self, v):
self.value = v
return v
def get(self):
return self.value
def __bool__(self):
return bool(self.value)
def __str__(self):
return str(self.value)
def __int__(self):
return int(self.value)
def __float__(self):
return float(self.value)
def __long__(self):
return self.value.__long__()
def __abs__(self):
return abs(self.value)
def __neg__(self):
return -self.value
def __pos__(self):
return +self.value
def __add__(self, other):
return self.value + other
def __sub__(self, other):
return -self.value + other
def __mul__(self, other):
return self.value * other
def __truediv__(self, other):
return self.value / other
def __floordiv__(self, other):
return self.value // other
def __pow__(self, power):
return self.value**power
__radd__ = __add__
__rsub__ = __sub__
__rmul__ = __mul__
def __coerce__(self, other):
return self, type(other)(self.value)
def __lt__(self, other):
return self.value < other
def __gt__(self, other):
return self.value > other
def __le__(self, other):
return self.value <= other
def __ge__(self, other):
return self.value >= other
def __eq__(self, other):
return self.value == other
def __ne__(self, other):
return self.value != other
def __iadd__(self, other):
with self.lock:
self.value += other
return self
def __isub__(self, other):
with self.lock:
self.value -= other
return self
def __imul__(self, other):
with self.lock:
self.value *= other
return self
def __idiv__(self, other):
with self.lock:
self.value /= other
return self
def __index__(self):
return self.__int__()
def __floor__(self):
return self.value // 1
def __ceil__(self):
return self.value // 1 + 1
