def shape(self,instance): l=len(instance) if (l==0): from pycvf.core.errors import pycvf_warning pycvf_warning("len of reader equal 0, strange, (debug return 1000 instead") return 1000 return l
def cimg_code(code,realcode="",prod=False): if not prod: pycvf_warning("once you finished developping, please add option 'prod=True' to avoid computing a md5 checksum at each call") base_install="/usr/" base_include=os.path.join(base_install,"include") cimg_info={'include_dirs': [base_include, ] , 'library_dirs':[base_install+"/lib"]} def fct(**context): args=[ code+("" if prod else "/* hash : %s */"%(hashlib.md5(realcode).hexdigest(),)), context.keys() ] kwargs=dict( support_code="# 1 \"inlinecode\"\n"+realcode, headers=[], libraries=['X11'], include_dirs=cimg_info['include_dirs'], library_dirs=cimg_info['library_dirs'], compiler='gcc', #type_converters = converters.blitz ) assert(not context.has_key("__inlineargs__")) context["__inlineargs__"]=args context["__inlinekwargs__"]=kwargs r= eval("inline(*__inlineargs__,**__inlinekwargs__)",globals(),context) context["__inlineargs__"]=None return r return fct
def getitems(self, queries, numelem=1, exact=False, *args, **kwargs): # pycvf_debug(10, "query getitems:"+str( queries)) if not self.distance: if type(queries[0]) in [int, float, long]: queries = map(lambda x: [x], queries) nqueries = to2d(numpy.asarray(queries)).astype(numpy.float32).copy("C") else: nqueries = to2d(numpy.asarray(queries)) res = [] qrk, qrd, qrc = self.lshindex.query(nqueries, numelem, self.recall, *args, **kwargs) for c in range(qrc.shape[0]): ci = qrc[c] if ci < numelem: # if LSH did not return the good number of elements we add some random points pycvf_warning( "Missing neighbors : %d returned instead of %d... remaining value will be noised\n" % (ci, numelem) ) qrk[c][ci:] = numpy.array([random.randint(0, self._keys.shape[0] - 1) for x in range(numelem)]) dm = map(lambda x: (((nqueries[c] - x) ** 2).sum()) ** 0.5, self._keys.take(qrk[c][ci:])) print dm qrd[c][ci:] = dm perm = qrd[c].argsort() qrd[c] = qrd[c].take(perm) qrk[c] = qrk[c].take(perm) # raise Exception res += [zip(map(lambda i: self._values[i], qrk[c]), qrd[c])] return res # numpy.vstack(res)
def __iter__(self): try: while True: yield self.next() except KeyboardInterrupt: raise except Exception,e: pycvf_warning(str(e)) pass
def save(self): if (self.dirty): pycvf_debug(10,"saving index "+self.filename+"...") try: self.idx.save(self.filename) except TypeError: pycvf_warning("Strange, strange, strange ... it seems that your index class "+str(type(self.idx))+(str(self.idx))+" does not accept a name while saving ") self.idx.save() pycvf_debug(20,"cache save without name done") self.dirty=False
def step(self): if (self.fno==0): try: self.reader.seek_to(self.addrfrom) except Exception, e: pycvf_warning("Subsequence Reader : Unable to seek : the result may be invalid ! reason :"+str(e)) import traceback, sys if (hasattr(sys,"last_traceback")): traceback.print_tb(sys.last_traceback) else: traceback.print_tb(sys.exc_traceback)
def add_many(self,keys,values): if (not self.distance): try: self._keys=to2d(keys).astype(numpy.float32).copy('C') except ValueError: pycvf_warning("Errors LSH keys must be dense float vector in this implementation of LshIndex") pycvf_warning(u"here an example of your keys " +unicode(keys[0])) raise self._values=to2d(values) else: self._keys=to2d(keys) self._values=to2d(values) self.lshindex=MPLSHIndex(self._keys)
def add_many(self, keys, values): if not self.distance: try: self._keys = to2d(keys).astype(numpy.float32).copy("C") except ValueError: pycvf_warning("Errors LSH keys must be dense float vector in this implementation of LshIndex") pycvf_warning(u"here an example of your keys " + unicode(keys[0])) raise self._values = to2d(values) else: self._keys = to2d(keys) self._values = to2d(values) assert self._keys.min() >= 0 assert self._keys.min() <= 1 ## LSHKIT assumes vector in-between 0 and 1 self.lshindex = MPLSHIndex(self._keys, *self.args, **self.kwargs)
def add_many(self,keys,values): print "checking empty sash" assert(self._values==None) # SASH INDEX IS NOT INCREMENTAL assert(self._keys==None) # SASH INDEX IS NOT INCREMENTAL if (not self.distance): try: self._keys=to2d(keys).astype(numpy.float32).copy('C') except ValueError: pycvf_warning("Errors SASH keys must be dense float vector in this implementation of SashIndex") pycvf_warning(u"here an example of your keys " +unicode(keys[0])) raise #print type(values), values#,values[0] self._values=to2d(values) #print "shapes : ", self._keys.shape, self._values.shape else: self._keys=to2d(keys) self._values=to2d(values) self.sashindex.build(self._keys)
def __init__(self,class_, filename,factory=None,suppkwargs=None,verbose=True): if (suppkwargs==None): suppkwargs={} self.class_=class_ self.factory=factory self.filename=filename self.verbose=verbose self.dirty=False try: f=file(self.filename,"rb") self.instance=self.class_.load(f,**suppkwargs) except Exception, e: if (self.verbose): pycvf_warning( (u"Failed to load "+ self.filename+u":"+str(type(e))+u","+str(e)+u"\n").encode('utf8')) #if (hasattr(sys,"last_traceback")): # traceback.print_tb(sys.last_traceback) #else: # traceback.print_tb(sys.exc_traceback) self.instance=(self.factory(**suppkwargs) if self.factory else self.class_(**suppkwargs))
def torch3_code(code,realcode,prod=False): if not prod: pycvf_warning("once you finished developping, please add option 'prod=True' to avoid computing a md5 checksum at each call") base_install="/usr/local" base_install="/home/tranx/build/Torch3vision2.1/" base_include=os.path.join(base_install,"") base_include_vision=os.path.join(base_include,"vision2.1") vxl_info={'include_dirs': [base_include, base_include+"/core", base_include+"/matrix", base_include+"/matrix_double", base_include+"/distributions", base_include+"/kernels", base_include+"/convolutions", base_include+"/gradients", base_include+"/datasets", base_include_vision+"/core", base_include_vision+"/patternrecognition", base_include_vision+"/geometry", base_include_vision+"/trainers", ] , 'library_dirs':[base_install+"/lib"]} def fct(**context): args=[ code+"/* hash : %s */"%(("" if prod else hashlib.md5(realcode).hexdigest()),), context.keys() ] kwargs=dict( support_code="# 1 \"inlinecode\"\n"+realcode, headers=[], libraries=['torch'], include_dirs=vxl_info['include_dirs'], library_dirs=vxl_info['library_dirs'], compiler='gcc', #type_converters = converters.blitz ) assert(not context.has_key("__inlineargs__")) context["__inlineargs__"]=args context["__inlinekwargs__"]=kwargs r= eval("inline(*__inlineargs__,**__inlinekwargs__)",globals(),context) context["__inlineargs__"]=None return r return fct
def train_online(self,tset,tnset=None): if (hasattr(tset,"dtype") and tset.dtype==object): pycvf_warning("pushing object array of online train !!!") tset=map(lambda x:x,tset.flat) if (hasattr(tset,"dtype") and tset.dtype in [numpy.uint8, numpy.int8,numpy.uint16, numpy.int16 ,numpy.uint32, numpy.int32 ,object ] ): if tset.dtype in [numpy.uint8, numpy.int8,numpy.uint16, numpy.int16 ,numpy.uint32, numpy.int32 ]: tset=tset.astype(numpy.int64) else: print tset[0] if (type(tset)==list): tset=map(lambda x:x.astype(numpy.float64),tset) else: print type(tset) print tset.dtype assert(False) else: try: tset=map(lambda x:x.astype(numpy.float64),tset) except: pass tset=numpy.array(tset)# self.no+=len(tset) self.so+=reduce(lambda x,y: x+y,tset, 0 ) self.svc+=reduce(lambda x,y: x+y**2,tset, 0 )
# -*- coding: utf-8 -*- import os from pycvf.core import settings DISPLAY_DRIVER=settings.DISPLAY_DRIVER if ("DISPLAY" not in os.environ or not(len(os.environ["DISPLAY"]))) and DISPLAY_DRIVER in [ "qt" , "pyglet" ]: from pycvf.core.errors import pycvf_warning pycvf_warning("No DISPLAY variable changing default displayt to 'caca'") DISPLAY_DRIVER="caca" if DISPLAY_DRIVER=="pyglet": from pycvf.lib.video.render.pyglet_lazy import LazyDisplay elif DISPLAY_DRIVER=="aa": from pycvf.lib.video.render.aa_lazy import LazyDisplay elif DISPLAY_DRIVER=="caca": from pycvf.lib.video.render.caca_lazy import LazyDisplay elif DISPLAY_DRIVER=="qt": from pycvf.lib.video.render.qt_lazy import LazyDisplayQt as LazyDisplay else: raise ValueError, "For the moment, PyCVF only support one of the following video display : pyglet, aa, caca, qt"
def datatype(self): pycvf_warning("You are using a old style database... Please implement 'datatype' for specifying the datatype type") return self