def NNClassifier(dataset, options, method='dtw'): classnum = dataset.classnum k_pool = [1] k_num = len(k_pool) Acc = np.zeros(k_num) #dtw_knn_map = zeros(vidsetnum,) tic = time.time() # Macro = np.zeros(testsetdatanum) # Micro = np.zeros(testsetdatanum) # rightnum = np.zeros(k_num) Macro = Value(ctypes.c_float * dataset.testsetdatanum) Micro = Value(ctypes.c_float * dataset.testsetdatanum) rightnum = Value(ctypes.c_uint * k_num) cpu_count = os.cpu_count() task_per_cpu = dataset.testsetdatanum // (cpu_count - 1) + 1 inner_process = [ Process(target=knn, args=(i, task_per_cpu, k_num, k_pool, dataset, options, Macro, Micro, rightnum)) for i in range(cpu_count - 1) ] for p in inner_process: p.start() for p in inner_process: p.join() rightnum = np.ctypeslib.as_array(rightnum.get_obj()) Acc = rightnum / dataset.testsetdatanum Macro = np.ctypeslib.as_array(Macro.get_obj()) Micro = np.ctypeslib.as_array(Micro.get_obj()) macro = np.mean(Macro) micro = np.mean(Micro) knn_time = time.time() - tic knn_averagetime = knn_time / dataset.testsetdatanum # logger.info(vars()) return macro, micro, Acc, knn_averagetime
def GetHists(self, N=1, use_custom_tqdm=True, file_cache=None): self.pre_execution() _, con_width = map(int,os.popen('stty size', 'r').read().split()) # if user wants to cache histograms in file, then make sure # the hash of all the queued draws + list of all files in tchain # + all the aliases and their values # matches what was in the file # In principle, if those match, then results must be identical file_hash = hash("".join(sorted([x.GetTitle() for x in (self.ch.GetListOfFiles())]))) # FIXME on osx, the below line will cause a crash after looping when an alias is used # does something special happen when calling GetAlias() or GetListOfAliases() before running? if os.uname()[0] == "Darwin": alias_hash = 1 else: alias_hash = hash(tuple(sorted([(x.GetName(),self.ch.GetAlias(x.GetName())) for x in (self.ch.GetListOfAliases() or [])]))) queue_hash = hash(tuple(map(tuple,sorted(self.queued)))) total_hash = hash(queue_hash+file_hash+alias_hash) if file_cache and os.path.exists(file_cache): with open(file_cache,"r") as fh: data = pickle.load(fh) hash_cache = data["hash"] if hash_cache == total_hash: return data["hists"] # take list of dicts of histname:histogram items # and reduce them by adding and removing the prefix up to the first # underscore def reduce_hists(dicts): d_master = {} for idx,d in enumerate(dicts): for hname, h in d.items(): if not h: continue stripped_name = hname.split("_",1)[-1] if stripped_name not in d_master: d_master[stripped_name] = h.Clone(stripped_name) else: d_master[stripped_name].Add(h) return d_master # take variety of things and put histograms from loop into a queue def get_hists(q, ch, queued, num, first, numentries, done, total, bytesread): prefix = "md{}_".format(num) clone = ch.Clone("ch_{}".format(prefix)) md = BaseTChain(clone) for x_ in queued: x = x_[:] x[1] = "{}{}".format(prefix,x[1]) md.queue(*x) md.execute_parallel(first,numentries,done.get_obj(),total.get_obj(), bytesread.get_obj()) hists = {} hist_names = map(lambda x:x[1].split("(",1)[0], queued) for hn in set(hist_names): hists[hn] = r.gDirectory.Get(prefix+hn) q.put(hists) return 0 # compute event splitting for N jobs first, last = 0, int(self.ch.GetEntries()) size = int((last-first) / N) firsts = [first+i*size for i in range(N)] + [last] diffs = map(lambda x: x[1]-x[0],zip(firsts[:-1],firsts[1:])) firsts_and_nentries = zip(firsts[:-1],diffs) if use_custom_tqdm: r.gROOT.ProcessLine(".L {}/tqdm.h".format(os.path.realpath(__file__).rsplit("/",1)[0])) bar = r.tqdm() os.nice(10) q = Queue(N) dones, totals, bytess, workers = [], [], [], [] for i, (first, numentries) in enumerate(firsts_and_nentries): done, total, bytesread = Value("i",0), Value("i",0), Value("i",0) worker = Process(target=get_hists, args=[q,self.ch,self.queued,i,first,numentries,done,total,bytesread]) workers.append(worker) worker.start() dones.append(done.get_obj()) bytess.append(bytesread.get_obj()) totals.append(total.get_obj()) def get_sum(cs): return sum(map(lambda x:x.value, cs)) try: ioq = TimedQueue(N=30) if use_custom_tqdm: total = last done = get_sum(dones) bytesread = get_sum(bytess) while done < total: done = get_sum(dones) bytesread = get_sum(bytess) ioq.add_val(1.0*bytesread/1e6) bar.progress(done,total,True) which_done = map(lambda x:(x[0].value==x[1].value)and(x[0].value>0), zip(dones,totals)) if con_width > 110: label = "[{:.1f}MB @ {:.1f}MB/s]".format(ioq.get_last_val(),ioq.get_rate()) label += " [{}]".format("".join(map(lambda x:unichr(0x2022) if x else unichr(0x2219),which_done)).encode("utf-8")) else: label = "" bar.set_label(label) time.sleep(0.04) label = "[{:.1f}MB @ {:.1f}MB/s]".format(ioq.get_last_val(),ioq.get_rate()) label += " [{}]".format("".join([unichr(0x2022) for _ in dones]).encode("utf-8")) bar.set_label(label) bar.progress(total,total,True) else: from tqdm import tqdm total = last prev_done = get_sum(dones) done = get_sum(dones) with tqdm(total=total) as pbar: while done < total-1: done = get_sum(dones) update = done - prev_done prev_done = done pbar.update(update) time.sleep(0.1) except KeyboardInterrupt as e: print("[!] Early keyboard interruption...continuing with histograms anyway") dicts = [] for iw in range(len(workers)): dicts.append(q.get()) for worker in workers: worker.join() # don't let one tqdm bar clobber another print reduced_hists = reduce_hists(dicts) # if user wants to cache histograms in file, # then dump the hists as well as a hash if file_cache: with open(file_cache,"w") as fh: data = {"hash": total_hash, "hists": reduced_hists} pickle.dump(data,fh) self.executed = True return reduced_hists
class SharedObject: """ This is a wrapper for Value() that implements lots of ease-of-port features and allows for subclassing into "types" Value() itself is a function that generates an object which cannot be inherited easily – hence, a wrapper is used """ NULL = object() def __init__(self, type_, value, lock=True): self.type = type_ if value != self.NULL: self._cobj = Value(type_, value, lock=lock) else: self._cobj = Value(type_, lock=lock) def __repr__(self): return "<Shared %s=%s object at %s>" % (self.__class__.__name__, self.value, hex(id(self))) @property def lock(self): return self._cobj.get_lock() @property def cobj(self): return self._cobj.get_obj() @property def value(self): return self._cobj.value @value.setter def value(self, other): with self.lock: self._cobj.value = other def set(self, v): self.value = v return v def get(self): return self.value def __bool__(self): return bool(self.value) def __str__(self): return str(self.value) def __int__(self): return int(self.value) def __float__(self): return float(self.value) def __long__(self): return self.value.__long__() def __abs__(self): return abs(self.value) def __neg__(self): return -self.value def __pos__(self): return +self.value def __add__(self, other): return self.value + other def __sub__(self, other): return -self.value + other def __mul__(self, other): return self.value * other def __truediv__(self, other): return self.value / other def __floordiv__(self, other): return self.value // other def __pow__(self, power): return self.value**power __radd__ = __add__ __rsub__ = __sub__ __rmul__ = __mul__ def __coerce__(self, other): return self, type(other)(self.value) def __lt__(self, other): return self.value < other def __gt__(self, other): return self.value > other def __le__(self, other): return self.value <= other def __ge__(self, other): return self.value >= other def __eq__(self, other): return self.value == other def __ne__(self, other): return self.value != other def __iadd__(self, other): with self.lock: self.value += other return self def __isub__(self, other): with self.lock: self.value -= other return self def __imul__(self, other): with self.lock: self.value *= other return self def __idiv__(self, other): with self.lock: self.value /= other return self def __index__(self): return self.__int__() def __floor__(self): return self.value // 1 def __ceil__(self): return self.value // 1 + 1