def shape(self,instance):
l=len(instance)
if (l==0):
from pycvf.core.errors import pycvf_warning
pycvf_warning("len of reader equal 0, strange, (debug return 1000 instead")
return 1000
return l
def cimg_code(code,realcode="",prod=False):
if not prod:
pycvf_warning("once you finished developping, please add option 'prod=True' to avoid computing a md5 checksum at each call")
base_install="/usr/"
base_include=os.path.join(base_install,"include")
cimg_info={'include_dirs': [base_include,
]
, 'library_dirs':[base_install+"/lib"]}
def fct(**context):
args=[ code+("" if prod else "/* hash : %s */"%(hashlib.md5(realcode).hexdigest(),)),
context.keys()
]
kwargs=dict(
support_code="# 1 \"inlinecode\"\n"+realcode,
headers=[],
libraries=['X11'],
include_dirs=cimg_info['include_dirs'],
library_dirs=cimg_info['library_dirs'],
compiler='gcc',
#type_converters = converters.blitz
)
assert(not context.has_key("__inlineargs__"))
context["__inlineargs__"]=args
context["__inlinekwargs__"]=kwargs
r= eval("inline(*__inlineargs__,**__inlinekwargs__)",globals(),context)
context["__inlineargs__"]=None
return r
return fct
def getitems(self, queries, numelem=1, exact=False, *args, **kwargs):
# pycvf_debug(10, "query getitems:"+str( queries))
if not self.distance:
if type(queries[0]) in [int, float, long]:
queries = map(lambda x: [x], queries)
nqueries = to2d(numpy.asarray(queries)).astype(numpy.float32).copy("C")
else:
nqueries = to2d(numpy.asarray(queries))
res = []
qrk, qrd, qrc = self.lshindex.query(nqueries, numelem, self.recall, *args, **kwargs)
for c in range(qrc.shape[0]):
ci = qrc[c]
if ci < numelem:
# if LSH did not return the good number of elements we add some random points
pycvf_warning(
"Missing neighbors : %d returned instead of %d... remaining value will be noised\n" % (ci, numelem)
)
qrk[c][ci:] = numpy.array([random.randint(0, self._keys.shape[0] - 1) for x in range(numelem)])
dm = map(lambda x: (((nqueries[c] - x) ** 2).sum()) ** 0.5, self._keys.take(qrk[c][ci:]))
print dm
qrd[c][ci:] = dm
perm = qrd[c].argsort()
qrd[c] = qrd[c].take(perm)
qrk[c] = qrk[c].take(perm)
# raise Exception
res += [zip(map(lambda i: self._values[i], qrk[c]), qrd[c])]
return res # numpy.vstack(res)
def __iter__(self):
try:
while True:
yield self.next()
except KeyboardInterrupt:
raise
except Exception,e:
pycvf_warning(str(e))
pass
def save(self):
if (self.dirty):
pycvf_debug(10,"saving index "+self.filename+"...")
try:
self.idx.save(self.filename)
except TypeError:
pycvf_warning("Strange, strange, strange ... it seems that your index class "+str(type(self.idx))+(str(self.idx))+" does not accept a name while saving ")
self.idx.save()
pycvf_debug(20,"cache save without name done")
self.dirty=False
def step(self):
if (self.fno==0):
try:
self.reader.seek_to(self.addrfrom)
except Exception, e:
pycvf_warning("Subsequence Reader : Unable to seek : the result may be invalid ! reason :"+str(e))
import traceback, sys
if (hasattr(sys,"last_traceback")):
traceback.print_tb(sys.last_traceback)
else:
traceback.print_tb(sys.exc_traceback)
def add_many(self,keys,values):
if (not self.distance):
try:
self._keys=to2d(keys).astype(numpy.float32).copy('C')
except ValueError:
pycvf_warning("Errors LSH keys must be dense float vector in this implementation of LshIndex")
pycvf_warning(u"here an example of your keys " +unicode(keys[0]))
raise
self._values=to2d(values)
else:
self._keys=to2d(keys)
self._values=to2d(values)
self.lshindex=MPLSHIndex(self._keys)
def add_many(self, keys, values):
if not self.distance:
try:
self._keys = to2d(keys).astype(numpy.float32).copy("C")
except ValueError:
pycvf_warning("Errors LSH keys must be dense float vector in this implementation of LshIndex")
pycvf_warning(u"here an example of your keys " + unicode(keys[0]))
raise
self._values = to2d(values)
else:
self._keys = to2d(keys)
self._values = to2d(values)
assert self._keys.min() >= 0
assert self._keys.min() <= 1 ## LSHKIT assumes vector in-between 0 and 1
self.lshindex = MPLSHIndex(self._keys, *self.args, **self.kwargs)
def add_many(self,keys,values):
print "checking empty sash"
assert(self._values==None) # SASH INDEX IS NOT INCREMENTAL
assert(self._keys==None) # SASH INDEX IS NOT INCREMENTAL
if (not self.distance):
try:
self._keys=to2d(keys).astype(numpy.float32).copy('C')
except ValueError:
pycvf_warning("Errors SASH keys must be dense float vector in this implementation of SashIndex")
pycvf_warning(u"here an example of your keys " +unicode(keys[0]))
raise
#print type(values), values#,values[0]
self._values=to2d(values)
#print "shapes : ", self._keys.shape, self._values.shape
else:
self._keys=to2d(keys)
self._values=to2d(values)
self.sashindex.build(self._keys)
def __init__(self,class_, filename,factory=None,suppkwargs=None,verbose=True):
if (suppkwargs==None):
suppkwargs={}
self.class_=class_
self.factory=factory
self.filename=filename
self.verbose=verbose
self.dirty=False
try:
f=file(self.filename,"rb")
self.instance=self.class_.load(f,**suppkwargs)
except Exception, e:
if (self.verbose):
pycvf_warning( (u"Failed to load "+ self.filename+u":"+str(type(e))+u","+str(e)+u"\n").encode('utf8'))
#if (hasattr(sys,"last_traceback")):
# traceback.print_tb(sys.last_traceback)
#else:
# traceback.print_tb(sys.exc_traceback)
self.instance=(self.factory(**suppkwargs) if self.factory else self.class_(**suppkwargs))
def torch3_code(code,realcode,prod=False):
if not prod:
pycvf_warning("once you finished developping, please add option 'prod=True' to avoid computing a md5 checksum at each call")
base_install="/usr/local"
base_install="/home/tranx/build/Torch3vision2.1/"
base_include=os.path.join(base_install,"")
base_include_vision=os.path.join(base_include,"vision2.1")
vxl_info={'include_dirs': [base_include,
base_include+"/core",
base_include+"/matrix",
base_include+"/matrix_double",
base_include+"/distributions",
base_include+"/kernels",
base_include+"/convolutions",
base_include+"/gradients",
base_include+"/datasets",
base_include_vision+"/core",
base_include_vision+"/patternrecognition",
base_include_vision+"/geometry",
base_include_vision+"/trainers",
]
, 'library_dirs':[base_install+"/lib"]}
def fct(**context):
args=[ code+"/* hash : %s */"%(("" if prod else hashlib.md5(realcode).hexdigest()),),
context.keys()
]
kwargs=dict(
support_code="# 1 \"inlinecode\"\n"+realcode,
headers=[],
libraries=['torch'],
include_dirs=vxl_info['include_dirs'],
library_dirs=vxl_info['library_dirs'],
compiler='gcc',
#type_converters = converters.blitz
)
assert(not context.has_key("__inlineargs__"))
context["__inlineargs__"]=args
context["__inlinekwargs__"]=kwargs
r= eval("inline(*__inlineargs__,**__inlinekwargs__)",globals(),context)
context["__inlineargs__"]=None
return r
return fct
def train_online(self,tset,tnset=None):
if (hasattr(tset,"dtype") and tset.dtype==object):
pycvf_warning("pushing object array of online train !!!")
tset=map(lambda x:x,tset.flat)
if (hasattr(tset,"dtype") and tset.dtype in [numpy.uint8, numpy.int8,numpy.uint16, numpy.int16 ,numpy.uint32, numpy.int32 ,object ] ):
if tset.dtype in [numpy.uint8, numpy.int8,numpy.uint16, numpy.int16 ,numpy.uint32, numpy.int32 ]:
tset=tset.astype(numpy.int64)
else:
print tset[0]
if (type(tset)==list):
tset=map(lambda x:x.astype(numpy.float64),tset)
else:
print type(tset)
print tset.dtype
assert(False)
else:
try:
tset=map(lambda x:x.astype(numpy.float64),tset)
except:
pass
tset=numpy.array(tset)#
self.no+=len(tset)
self.so+=reduce(lambda x,y: x+y,tset, 0 )
self.svc+=reduce(lambda x,y: x+y**2,tset, 0 )
# -*- coding: utf-8 -*-
import os
from pycvf.core import settings
DISPLAY_DRIVER=settings.DISPLAY_DRIVER
if ("DISPLAY" not in os.environ or not(len(os.environ["DISPLAY"]))) and DISPLAY_DRIVER in [ "qt" , "pyglet" ]:
from pycvf.core.errors import pycvf_warning
pycvf_warning("No DISPLAY variable changing default displayt to 'caca'")
DISPLAY_DRIVER="caca"
if DISPLAY_DRIVER=="pyglet":
from pycvf.lib.video.render.pyglet_lazy import LazyDisplay
elif DISPLAY_DRIVER=="aa":
from pycvf.lib.video.render.aa_lazy import LazyDisplay
elif DISPLAY_DRIVER=="caca":
from pycvf.lib.video.render.caca_lazy import LazyDisplay
elif DISPLAY_DRIVER=="qt":
from pycvf.lib.video.render.qt_lazy import LazyDisplayQt as LazyDisplay
else:
raise ValueError, "For the moment, PyCVF only support one of the following video display : pyglet, aa, caca, qt"
def datatype(self):
pycvf_warning("You are using a old style database... Please implement 'datatype' for specifying the datatype type")
return self