def get_databatch(self, batchsize=None, testing=False):
if batchsize is None:
batchsize = self.batchsize
if self.datasource == "berkeleysegmentation":
patch_sz = self.model.patch_sz
if not hasattr(self, "images"):
from config import data_dir
from scipy import misc
if testing:
img_dir = os.path.join(data_dir, "BSR/BSDS500/data/images/test")
nimg = 200
else:
img_dir = os.path.join(data_dir, "BSR/BSDS500/data/images/train")
nimg = 200
img_files = [os.path.join(img_dir, img_name) for img_name in os.listdir(img_dir)]
if len(img_files) != nimg:
raise IOError("image files missing - found %d of %d in %s" % (len(img_files), nimg, img_dir))
self.images = np.zeros((481, 321, nimg), dtype=np.uint8)
for i, fname in enumerate(img_files):
a = misc.imread(fname, flatten=True)
if a.shape[0] == 321:
a = a.T
self.images[:, :, i] = a
batch = np.zeros((self.model.D, batchsize))
for i in xrange(batchsize):
x, y, t = self.images.shape
rt = np.random.randint(t)
rx = np.random.randint(x - patch_sz + 1)
ry = np.random.randint(y - patch_sz + 1)
batch[:, i] = self.images[rx : rx + patch_sz, ry : ry + patch_sz, rt].ravel()
return batch
elif self.datasource == "vid075-chunks":
if not hasattr(self, "videos"):
from config import data_dir
nvideos = 56
videoheight = 128
videowidth = 128
videot = 64
self.BUFF = 4
self.topmargin = 15
self.videos = np.zeros((nvideos, videoheight, videowidth, videot))
for video_ind in range(1, nvideos + 1):
video_fname = os.path.join(data_dir, "vid075-chunks", "chunk" + str(video_ind))
video_cstring = (
np.fromfile(video_fname, dtype=">f4")
.reshape(videot, videowidth, videoheight)
.astype(np.double)
.tostring("F")
)
self.videos[video_ind - 1, ...] = np.fromstring(video_cstring).reshape(
videoheight, videowidth, videot
)
self.nvideos = nvideos
self.videoheight = videoheight
self.videowidth = videowidth
self.videot = videot
if batchsize > self.videot:
batch = np.zeros((self.model.D, batchsize))
done = False
batch_remaining = batchsize
t0 = 0
while not done:
tsz = min(self.videot, batch_remaining)
batch0 = self.crop_videos(tsz)
batch[:, t0 : t0 + tsz] = batch0.reshape(self.model.D, tsz)
t0 += tsz
batch_remaining -= tsz
if batch_remaining <= 0:
done = True
else:
batch = self.crop_videos(batchsize).reshape(self.model.D, batchsize)
return batch
elif self.datasource == "YouTubeFaces_aligned":
import cPickle
from scipy import misc
from config import scratch_local_dir, scratch_dir, public_dir
patch_sz = self.model.patch_sz
if not hasattr(self, "YouTubeInfo"):
pcrop = 80 # crop pixels from center
newsize = int(np.ceil(patch_sz * 1.5))
TOTAL_FRAMES = 1000000
cache_dir = os.path.join(scratch_dir, "hdl", "YouTubeFaces/aligned_images_DB")
cache_name = os.path.join(
cache_dir, "newsize_%d_pcrop_%d_TOTAL_FRAMES_%d.npz" % (newsize, pcrop, TOTAL_FRAMES)
)
if not os.path.exists(cache_name):
self.YouTubeInfo = {"videos": [], "video_weights": []}
base_dir = os.path.join(scratch_local_dir, "YouTubeFaces/aligned_images_DB")
if not os.path.exists(base_dir):
base_dir = os.path.join(public_dir, "YouTubeFaces/YouTubeFaces/aligned_images_DB")
identities = os.listdir(base_dir)
total_frames = 0
for identity in identities:
if total_frames > TOTAL_FRAMES:
continue
video_numbers = os.listdir(os.path.join(base_dir, identity))
for video_number in video_numbers:
video_number_path = os.path.join(base_dir, identity, video_number)
frame_partial_names = os.listdir(video_number_path)
frame_partial_index = sorted([int(item.split(".")[1]) for item in frame_partial_names])
first_part, dummy, last_part = frame_partial_names[0].split(".")
frame_fnames = [
os.path.join(video_number_path, first_part + "." + str(img_index) + "." + last_part)
for img_index in frame_partial_index
]
frames = len(frame_fnames)
video_array = np.zeros((newsize, newsize, frames), dtype=np.uint8)
for frame_index, frame_fname in enumerate(frame_fnames):
image = misc.imread(frame_fname, flatten=True)
sxy, szx = image.shape
cent = int(np.floor(np.float(szx) / 2))
video_array[..., frame_index] = misc.imresize(
image[(cent - pcrop) : (cent + pcrop), (cent - pcrop) : (cent + pcrop)],
(newsize, newsize),
)
self.YouTubeInfo["videos"].append(video_array)
self.YouTubeInfo["video_weights"].append(frames)
total_frames += frames
print "\rtotal_frames loaded=%d" % total_frames
num_videos = len(self.YouTubeInfo["videos"])
video_weights = np.zeros((num_videos,))
for ind, val in enumerate(self.YouTubeInfo["video_weights"]):
video_weights[ind] = float(val) / total_frames
self.YouTubeInfo["video_weights"] = video_weights
self.YouTubeInfo["num_videos"] = num_videos
print "\nSaving cache file:", cache_name, "...",
if not os.path.exists(cache_dir):
os.makedirs(cache_dir)
with open(cache_name, "wb") as fh:
cPickle.dump(dict(YouTubeInfo=self.YouTubeInfo), fh)
print "Done"
else:
print "Loading cache file:", cache_name, "..."
with open(cache_name, "rb") as fh:
ldict = cPickle.load(fh)
self.YouTubeInfo = ldict["YouTubeInfo"]
print "Done"
video_index = np.where(np.random.multinomial(1, self.YouTubeInfo["video_weights"]))[0][0]
video_array = self.YouTubeInfo["videos"][video_index]
num_frames = video_array.shape[2]
if batchsize > num_frames:
batch = np.zeros((self.model.D, batchsize))
done = False
batch_remaining = batchsize
t0 = 0
while not done:
video_index = np.where(np.random.multinomial(1, self.YouTubeInfo["video_weights"]))[0][0]
video_array = self.YouTubeInfo["videos"][video_index]
num_frames = video_array.shape[2]
tsz = min(num_frames, batch_remaining)
batch0 = self.crop_single_video(video_array, tsz)
batch[:, t0 : t0 + tsz] = batch0.reshape(self.model.D, tsz)
t0 += tsz
batch_remaining -= tsz
if batch_remaining <= 0:
done = True
else:
batch = self.crop_single_video(video_array, batchsize).reshape(self.model.D, batchsize)
return batch.astype(np.single)
elif self.datasource == "YouTubeFaces_aligned_asymmetric":
import cPickle
from scipy import misc
from config import scratch_local_dir, scratch_dir, public_dir
patch_sz = self.model.patch_sz
if not hasattr(self, "YouTubeInfo"):
normcrop = 80
xcrop = 60 # crop pixels from center
ycrop = 100
newxsize = int(np.ceil(float(xcrop) / normcrop * patch_sz * 1.5))
newysize = int(np.ceil(float(ycrop) / normcrop * patch_sz * 1.5))
TOTAL_FRAMES = 1000000
cache_dir = os.path.join(scratch_dir, "hdl", "YouTubeFaces/aligned_images_DB")
cache_name = os.path.join(
cache_dir,
"newxsize_%d_newysize_%d_xcrop_%d_ycrop_%d_normcrop_%d_TOTAL_FRAMES_%d.npz"
% (newxsize, newysize, xcrop, ycrop, normcrop, TOTAL_FRAMES),
)
if not os.path.exists(cache_name):
self.YouTubeInfo = {"videos": [], "video_weights": []}
base_dir = os.path.join(scratch_local_dir, "YouTubeFaces/aligned_images_DB")
if not os.path.exists(base_dir):
base_dir = os.path.join(public_dir, "YouTubeFaces/YouTubeFaces/aligned_images_DB")
identities = os.listdir(base_dir)
total_frames = 0
for identity in identities:
if total_frames > TOTAL_FRAMES:
continue
video_numbers = os.listdir(os.path.join(base_dir, identity))
for video_number in video_numbers:
video_number_path = os.path.join(base_dir, identity, video_number)
frame_partial_names = os.listdir(video_number_path)
frame_partial_index = sorted([int(item.split(".")[1]) for item in frame_partial_names])
first_part, dummy, last_part = frame_partial_names[0].split(".")
frame_fnames = [
os.path.join(video_number_path, first_part + "." + str(img_index) + "." + last_part)
for img_index in frame_partial_index
]
frames = len(frame_fnames)
video_array = np.zeros((newysize, newxsize, frames), dtype=np.uint8)
for frame_index, frame_fname in enumerate(frame_fnames):
image = misc.imread(frame_fname, flatten=True)
sxy, szx = image.shape
cent = int(np.floor(np.float(szx) / 2))
video_array[..., frame_index] = misc.imresize(
image[(cent - ycrop) : (cent + ycrop), (cent - xcrop) : (cent + xcrop)],
(newysize, newxsize),
)
self.YouTubeInfo["videos"].append(video_array)
self.YouTubeInfo["video_weights"].append(frames)
total_frames += frames
print "\rtotal_frames loaded=%d" % total_frames
num_videos = len(self.YouTubeInfo["videos"])
video_weights = np.zeros((num_videos,))
for ind, val in enumerate(self.YouTubeInfo["video_weights"]):
video_weights[ind] = float(val) / total_frames
self.YouTubeInfo["video_weights"] = video_weights
self.YouTubeInfo["num_videos"] = num_videos
print "\nSaving cache file:", cache_name, "...",
if not os.path.exists(cache_dir):
os.makedirs(cache_dir)
with open(cache_name, "wb") as fh:
cPickle.dump(dict(YouTubeInfo=self.YouTubeInfo), fh)
print "Done"
else:
print "Loading cache file:", cache_name, "..."
with open(cache_name, "rb") as fh:
ldict = cPickle.load(fh)
self.YouTubeInfo = ldict["YouTubeInfo"]
print "Done"
video_index = np.where(np.random.multinomial(1, self.YouTubeInfo["video_weights"]))[0][0]
video_array = self.YouTubeInfo["videos"][video_index]
num_frames = video_array.shape[2]
if batchsize > num_frames:
batch = np.zeros((self.model.D, batchsize))
done = False
batch_remaining = batchsize
t0 = 0
while not done:
video_index = np.where(np.random.multinomial(1, self.YouTubeInfo["video_weights"]))[0][0]
video_array = self.YouTubeInfo["videos"][video_index]
num_frames = video_array.shape[2]
tsz = min(num_frames, batch_remaining)
batch0 = self.crop_single_video(video_array, tsz)
batch[:, t0 : t0 + tsz] = batch0.reshape(self.model.D, tsz)
t0 += tsz
batch_remaining -= tsz
if batch_remaining <= 0:
done = True
else:
batch = self.crop_single_video(video_array, batchsize).reshape(self.model.D, batchsize)
return batch.astype(np.single)
elif self.datasource == "PLoS09_Cars_Planes":
if not hasattr(self, "videos"):
from kairos.data import PLoS09
# movie_size = (96,96)
movie_size = self.kargs.get("movie_size", (40, 40))
self.videos, self.chunk_labels = PLoS09.load_movies(movie_size)
self.BUFF = 0
self.topmargin = 0
self.nvideos, self.videoheight, self.videowidth, self.videot = self.videos.shape
if batchsize > self.videot:
batch = np.zeros((self.model.D, batchsize))
done = False
batch_remaining = batchsize
t0 = 0
while not done:
tsz = min(self.videot, batch_remaining)
batch0 = self.crop_videos(tsz)
batch[:, t0 : t0 + tsz] = batch0.reshape(self.model.D, tsz)
t0 += tsz
batch_remaining -= tsz
if batch_remaining <= 0:
done = True
else:
batch = self.crop_videos(batchsize).reshape(self.model.D, batchsize)
return batch
elif self.datasource == "TorontoFaces48":
from scipy.io import loadmat
from config import public_dir
patch_sz = self.model.patch_sz
if not hasattr(self, "images"):
loadfile = os.path.join(public_dir, "TorontoFaces", "TFD_ranzato_48x48.mat")
mfile = loadmat(loadfile)
self.images = mfile["images"]
batch = np.zeros((self.model.D, batchsize))
for i in xrange(batchsize):
t, x, y = self.images.shape
rt = np.random.randint(t)
rx = np.random.randint(x - patch_sz + 1)
ry = np.random.randint(y - patch_sz + 1)
batch[:, i] = self.images[rt, rx : rx + patch_sz, ry : ry + patch_sz].ravel()
return batch
elif self.datasource == "TorontoFaces96":
from scipy.io import loadmat
from config import public_dir
patch_sz = self.model.patch_sz
if not hasattr(self, "images"):
loadfile = os.path.join(public_dir, "TorontoFaces", "TFD_ranzato_96x96.mat")
mfile = loadmat(loadfile)
self.images = mfile["images"]
batch = np.zeros((self.model.D, batchsize))
for i in xrange(batchsize):
t, x, y = self.images.shape
rt = np.random.randint(t)
rx = np.random.randint(x - patch_sz + 1)
ry = np.random.randint(y - patch_sz + 1)
batch[:, i] = self.images[rt, rx : rx + patch_sz, ry : ry + patch_sz].ravel()
return batch
elif self.datasource == "3Dvideo_color":
from config import data_dir
import tables
if not hasattr(self, "videos"):
print "Loading 3Dvideo_color"
self.video_buffer = 100
self.video_reload = 200
self.video_counter = 0
self.binocular_offset = +13
videopath = os.path.join(data_dir, "3Dvideo", "processed")
self.video_list = [
os.path.join(videopath, item) for item in filter(lambda x: x.count(".h5"), os.listdir(videopath))
]
self.videos = []
for video_ind in range(self.video_buffer):
rind = np.random.randint(len(self.video_list))
h5file = tables.openFile(self.video_list[rind])
binocvideo = np.hstack((h5file.root.left.read(), h5file.root.right.read()))
self.videos.append(binocvideo)
h5file.close()
self.video_counter += 1
if not self.video_counter % self.video_reload:
rind = np.random.randint(len(self.video_list))
print "Load video", self.video_list[rind]
h5file = tables.openFile(self.video_list[rind])
binocvideo = np.hstack((h5file.root.left.read(), h5file.root.right.read()))
h5file.close()
self.videos.pop()
self.videos = [binocvideo] + self.videos
vidind = np.random.randint(self.video_buffer)
video = self.videos[vidind]
if batchsize > video.shape[0]:
batch = np.zeros((self.model.D, batchsize))
done = False
batch_remaining = batchsize
t0 = 0
while not done:
batch0 = None
tsz = 0
while batch0 is None:
vidind = np.random.randint(self.video_buffer)
video = self.videos[vidind]
tsz = min(video.shape[0], batch_remaining)
batch0 = self.crop_single_binoc_video(video, tsz)
batch[:, t0 : t0 + tsz] = batch0.reshape(tsz, self.model.D).T
t0 += tsz
batch_remaining -= tsz
if batch_remaining <= 0:
done = True
else:
batch0 = None
while batch0 is None:
batch0 = self.crop_single_binoc_video(video, batchsize)
batch = np.double(batch0.reshape(batchsize, self.model.D).T)
return batch
elif self.datasource == "randn":
return np.random.randn(self.model.D, batchsize)
elif self.datasource == "X":
if not hasattr(self, "X"):
self.X = self.kargs["input_data"]
batch = np.zeros((self.model.D, batchsize))
for i in xrange(batchsize):
t, x = self.X.shape
rt = np.random.randint(t)
batch[:, i] = self.X[rt, :].ravel()
return batch
else:
assert NotImplementedError, self.datasource
whitenpatches = 150000
#l = SGD(model=SparseSlowModel(patch_sz=48,N=2048,T=48,sparse_cost='subspacel1',slow_cost='dist'),datasource='PLoS09_Cars_Planes',batchsize=48,display_every=20000,save_every=20000,eta_target_maxupdate=.05)
#l = SGD(model=SparseSlowModel(patch_sz=32,N=1028,T=64,sparse_cost='subspacel1',slow_cost='dist'),datasource='PLoS09_Cars_Planes',batchsize=64,display_every=20000,save_every=20000,eta_target_maxupdate=.05)
l = SGD(model=SparseSlowModel(patch_sz=20,N=400,T=48,sparse_cost='subspacel1',slow_cost='dist',perc_var=99.9),datasource='PLoS09_Cars_Planes',batchsize=48,display_every=1000,save_every=20000,eta_target_maxupdate=.05)
databatch = l.get_databatch(whitenpatches)
l.model.learn_whitening(databatch)
l.model.setup()
from kairos.data import PLoS09
#from kairos.metrics.kanalysis import kanalysis
from devthor.kanalysis import kanalysis_linear as kanalysis
X, Y = PLoS09.get_testing_selection(l.videos,l.chunk_labels)
def evaluate_kernel(model,X,Y,output_function='infer_abs'):
offset_y = (X.shape[1] - model.patch_sz)/2
offset_x = (X.shape[2] - model.patch_sz)/2
X = X[:,offset_y:offset_y+model.patch_sz,offset_x:offset_x+model.patch_sz]
batch = X.reshape(X.shape[0],X.shape[1]*X.shape[2]).T
new_X = model.output(model.preprocess(batch),output_function=output_function).T
k_curve, k_auc = kanalysis(new_X,Y)
return k_curve, k_auc
def get_databatch(self,batchsize=None,testing=False):
if batchsize is None: batchsize = self.batchsize
if self.datasource == 'berkeleysegmentation':
patch_sz = self.model.patch_sz
if not hasattr(self,'images'):
from config import data_dir
from scipy import misc
if testing:
img_dir = os.path.join(data_dir,'BSR/BSDS500/data/images/test')
nimg = 200
else:
img_dir = os.path.join(data_dir,'BSR/BSDS500/data/images/train')
nimg = 200
img_files = [os.path.join(img_dir,img_name) for img_name in os.listdir(img_dir)]
if len(img_files) != nimg:
raise IOError('image files missing - found %d of %d in %s'%(len(img_files),nimg,img_dir))
self.images = np.zeros((481,321,nimg),dtype=np.uint8)
for i,fname in enumerate(img_files):
a = misc.imread(fname,flatten=True)
if a.shape[0] == 321: a = a.T
self.images[:,:,i] = a
batch = np.zeros((self.model.D,batchsize))
for i in xrange(batchsize):
x,y,t = self.images.shape
rt = np.random.randint(t)
rx = np.random.randint(x-patch_sz+1)
ry = np.random.randint(y-patch_sz+1)
batch[:,i] = self.images[rx:rx+patch_sz,ry:ry+patch_sz,rt].ravel()
return batch
elif self.datasource == 'vid075-chunks':
if not hasattr(self,'videos'):
from config import data_dir
nvideos = 56
videoheight = 128
videowidth = 128
videot = 64
self.BUFF = 4
self.topmargin = 15
self.videos = np.zeros((nvideos, videoheight, videowidth, videot))
for video_ind in range(1,nvideos+1):
video_fname = os.path.join(data_dir,'vid075-chunks','chunk' + str(video_ind))
video_cstring = np.fromfile(video_fname,dtype='>f4').reshape(videot,videowidth,videoheight).astype(np.double).tostring('F')
self.videos[video_ind-1,...] = np.fromstring(video_cstring).reshape(videoheight,videowidth,videot)
self.nvideos = nvideos
self.videoheight = videoheight
self.videowidth = videowidth
self.videot = videot
if batchsize > self.videot:
batch = np.zeros((self.model.D,batchsize))
done = False
batch_remaining = batchsize
t0 = 0
while not done:
tsz = min(self.videot,batch_remaining)
batch0 = self.crop_videos(tsz)
batch[:,t0:t0+tsz] = batch0.reshape(self.model.D,tsz)
t0 += tsz
batch_remaining -= tsz
if batch_remaining <= 0: done = True
else:
batch = self.crop_videos(batchsize).reshape(self.model.D,batchsize)
return batch
elif self.datasource == 'YouTubeFaces_aligned':
import cPickle
from scipy import misc
from config import scratch_local_dir, scratch_dir, public_dir
patch_sz = self.model.patch_sz
cache_dir = os.path.join(scratch_dir,'hdl','YouTubeFaces/aligned_images_DB')
cache_video_basename = os.path.join(cache_dir,'newsize_%d_pcrop_%d_TOTAL_FRAMES_%d_video_%d.npz')
if not hasattr(self,'YouTubeInfo'):
pcrop = self.kargs.get('pixels_from_center',80) # crop pixels from center
newsize = int(np.ceil(patch_sz*self.kargs.get('new_patch_size_fraction',1.5)))
TOTAL_FRAMES = self.kargs.get('TOTAL_FRAMES',1000000)
cache_info_name = os.path.join(cache_dir,'newsize_%d_pcrop_%d_TOTAL_FRAMES_%d.npz'%(newsize,pcrop,TOTAL_FRAMES))
if not os.path.exists(cache_info_name):
self.YouTubeInfo = {'video_weights':[]}
self.YouTubeVideos = []
base_dir = os.path.join(scratch_local_dir,'YouTubeFaces/aligned_images_DB')
if not os.path.exists(base_dir): base_dir = os.path.join(public_dir,'YouTubeFaces/YouTubeFaces/aligned_images_DB')
identities = os.listdir(base_dir)
total_frames = 0
for identity in identities:
if total_frames > TOTAL_FRAMES: continue
video_numbers = os.listdir(os.path.join(base_dir,identity))
for video_number in video_numbers:
video_number_path = os.path.join(base_dir,identity,video_number)
frame_partial_names = os.listdir(video_number_path)
frame_partial_index = sorted([int(item.split('.')[1]) for item in frame_partial_names])
first_part, dummy, last_part = frame_partial_names[0].split('.')
frame_fnames = [os.path.join(video_number_path,first_part + '.' + str(img_index) + '.' + last_part) for img_index in frame_partial_index]
frames = len(frame_fnames)
video_array = np.zeros((newsize,newsize,frames),dtype=np.uint8)
for frame_index, frame_fname in enumerate(frame_fnames):
image = misc.imread(frame_fname,flatten=True)
sxy, szx = image.shape
cent = int(np.floor(np.float(szx)/2))
video_array[...,frame_index] = misc.imresize(image[(cent-pcrop):(cent+pcrop),(cent-pcrop):(cent+pcrop)],(newsize,newsize))
self.YouTubeVideos.append(video_array)
self.YouTubeInfo['video_weights'].append(frames)
total_frames += frames
print '\rtotal_frames loaded=%d'%total_frames
num_videos = len(self.YouTubeVideos)
video_weights = np.zeros((num_videos,))
for ind, val in enumerate(self.YouTubeInfo['video_weights']):
video_weights[ind] = float(val)/total_frames
self.YouTubeInfo['video_weights'] = video_weights
self.YouTubeInfo['num_videos'] = num_videos
self.YouTubeInfo['pcrop'] = pcrop
self.YouTubeInfo['newsize'] = newsize
self.YouTubeInfo['TOTAL_FRAMES'] = TOTAL_FRAMES
print '\nSaving cache file:', cache_info_name, '...',
if not os.path.exists(cache_dir): os.makedirs(cache_dir)
with open(cache_info_name,'wb') as fh:
cPickle.dump(dict(YouTubeInfo=self.YouTubeInfo),fh)
print 'Done'
for video_ind, video in enumerate(self.YouTubeVideos):
cache_video_name = cache_video_basename%(newsize,pcrop,TOTAL_FRAMES,video_ind)
print 'Saving cache file:', cache_video_name, '...',
if not os.path.exists(cache_dir): os.makedirs(cache_dir)
with open(cache_video_name, 'wb') as fh:
cPickle.dump(dict(video=video,video_ind=video_ind), fh)
print 'Done'
else:
print 'Loading cache file:', cache_info_name, '...'
with open(cache_info_name,'rb') as fh:
ldict = cPickle.load(fh)
self.YouTubeInfo = ldict['YouTubeInfo']
print 'Done'
video_index = np.where(np.random.multinomial(1,self.YouTubeInfo['video_weights']))[0][0]
cache_video_name = cache_video_basename%(self.YouTubeInfo['newsize'],
self.YouTubeInfo['pcrop'],
self.YouTubeInfo['TOTAL_FRAMES'],
video_index)
with open(cache_video_name,'rb') as fh:
ldict = cPickle.load(fh)
video_array = ldict['video']
#video_array = self.YouTubeInfo['videos'][video_index]
num_frames = video_array.shape[2]
if batchsize > num_frames:
batch = np.zeros((self.model.D,batchsize))
done = False
batch_remaining = batchsize
t0 = 0
while not done:
video_index = np.where(np.random.multinomial(1,self.YouTubeInfo['video_weights']))[0][0]
cache_video_name = cache_video_basename%(self.YouTubeInfo['newsize'],
self.YouTubeInfo['pcrop'],
self.YouTubeInfo['TOTAL_FRAMES'],
video_index)
with open(cache_video_name,'rb') as fh:
ldict = cPickle.load(fh)
video_array = ldict['video']
#video_array = self.YouTubeInfo['videos'][video_index]
num_frames = video_array.shape[2]
tsz = min(num_frames,batch_remaining)
batch0 = self.crop_single_video(video_array,tsz)
batch[:,t0:t0+tsz] = batch0.reshape(self.model.D,tsz)
t0 += tsz
batch_remaining -= tsz
if batch_remaining <= 0: done = True
else:
batch = self.crop_single_video(video_array,batchsize).reshape(self.model.D,batchsize)
return batch.astype(np.single)
elif self.datasource == 'YouTubeFaces_aligned_asymmetric':
import cPickle
from scipy import misc
from config import scratch_local_dir, scratch_dir, public_dir
patch_sz = self.model.patch_sz
if not hasattr(self,'YouTubeInfo'):
normcrop = 80
xcrop = 60 # crop pixels from center
ycrop = 100
newxsize = int(np.ceil( float(xcrop)/normcrop * patch_sz*1.5) )
newysize = int(np.ceil( float(ycrop)/normcrop * patch_sz*1.5) )
TOTAL_FRAMES = 1000000
cache_dir = os.path.join(scratch_dir,'hdl','YouTubeFaces/aligned_images_DB')
cache_name = os.path.join(cache_dir,'newxsize_%d_newysize_%d_xcrop_%d_ycrop_%d_normcrop_%d_TOTAL_FRAMES_%d.npz'%(newxsize,newysize,xcrop,ycrop,normcrop,TOTAL_FRAMES))
if not os.path.exists(cache_name):
self.YouTubeInfo = {'videos':[],'video_weights':[]}
base_dir = os.path.join(scratch_local_dir,'YouTubeFaces/aligned_images_DB')
if not os.path.exists(base_dir): base_dir = os.path.join(public_dir,'YouTubeFaces/YouTubeFaces/aligned_images_DB')
identities = os.listdir(base_dir)
total_frames = 0
for identity in identities:
if total_frames > TOTAL_FRAMES: continue
video_numbers = os.listdir(os.path.join(base_dir,identity))
for video_number in video_numbers:
video_number_path = os.path.join(base_dir,identity,video_number)
frame_partial_names = os.listdir(video_number_path)
frame_partial_index = sorted([int(item.split('.')[1]) for item in frame_partial_names])
first_part, dummy, last_part = frame_partial_names[0].split('.')
frame_fnames = [os.path.join(video_number_path,first_part + '.' + str(img_index) + '.' + last_part) for img_index in frame_partial_index]
frames = len(frame_fnames)
video_array = np.zeros((newysize,newxsize,frames),dtype=np.uint8)
for frame_index, frame_fname in enumerate(frame_fnames):
image = misc.imread(frame_fname,flatten=True)
sxy, szx = image.shape
cent = int(np.floor(np.float(szx)/2))
video_array[...,frame_index] = misc.imresize(image[(cent-ycrop):(cent+ycrop),(cent-xcrop):(cent+xcrop)],(newysize,newxsize))
self.YouTubeInfo['videos'].append(video_array)
self.YouTubeInfo['video_weights'].append(frames)
total_frames += frames
print '\rtotal_frames loaded=%d'%total_frames
num_videos = len(self.YouTubeInfo['videos'])
video_weights = np.zeros((num_videos,))
for ind, val in enumerate(self.YouTubeInfo['video_weights']):
video_weights[ind] = float(val)/total_frames
self.YouTubeInfo['video_weights'] = video_weights
self.YouTubeInfo['num_videos'] = num_videos
print '\nSaving cache file:', cache_name, '...',
if not os.path.exists(cache_dir): os.makedirs(cache_dir)
with open(cache_name,'wb') as fh:
cPickle.dump(dict(YouTubeInfo=self.YouTubeInfo),fh)
print 'Done'
else:
print 'Loading cache file:', cache_name, '...'
with open(cache_name,'rb') as fh:
ldict = cPickle.load(fh)
self.YouTubeInfo = ldict['YouTubeInfo']
print 'Done'
video_index = np.where(np.random.multinomial(1,self.YouTubeInfo['video_weights']))[0][0]
video_array = self.YouTubeInfo['videos'][video_index]
num_frames = video_array.shape[2]
if batchsize > num_frames:
batch = np.zeros((self.model.D,batchsize))
done = False
batch_remaining = batchsize
t0 = 0
while not done:
video_index = np.where(np.random.multinomial(1,self.YouTubeInfo['video_weights']))[0][0]
video_array = self.YouTubeInfo['videos'][video_index]
num_frames = video_array.shape[2]
tsz = min(num_frames,batch_remaining)
batch0 = self.crop_single_video(video_array,tsz)
batch[:,t0:t0+tsz] = batch0.reshape(self.model.D,tsz)
t0 += tsz
batch_remaining -= tsz
if batch_remaining <= 0: done = True
else:
batch = self.crop_single_video(video_array,batchsize).reshape(self.model.D,batchsize)
return batch.astype(np.single)
elif self.datasource == 'PLoS09_Cars_Planes':
if not hasattr(self,'videos'):
from kairos.data import PLoS09
#movie_size = (96,96)
movie_size = self.kargs.get('movie_size',(40,40))
self.videos, self.chunk_labels = PLoS09.load_movies(movie_size)
self.BUFF = 0
self.topmargin = 0
self.nvideos, self.videoheight, self.videowidth, self.videot = self.videos.shape
if batchsize > self.videot:
batch = np.zeros((self.model.D,batchsize))
done = False
batch_remaining = batchsize
t0 = 0
while not done:
tsz = min(self.videot,batch_remaining)
batch0 = self.crop_videos(tsz)
batch[:,t0:t0+tsz] = batch0.reshape(self.model.D,tsz)
t0 += tsz
batch_remaining -= tsz
if batch_remaining <= 0: done = True
else:
batch = self.crop_videos(batchsize).reshape(self.model.D,batchsize)
return batch.astype(np.double)
elif self.datasource == 'PLoS09_Planes':
if not hasattr(self, 'videos'):
from kairos.data import PLoS09
#movie_size = (96,96)
movie_size = self.kargs.get('movie_size', (40, 40))
temp_videos, temp_chunk_labels = PLoS09.load_movies(movie_size)
videos = []
chunk_labels = []
for chunk_ind, temp_chunk_label in enumerate(temp_chunk_labels):
if temp_chunk_label == 'p':
videos.append(temp_videos[chunk_ind,...])
chunk_labels.append(temp_chunk_label)
self.videos = np.asarray(videos)
self.chunk_labels = chunk_labels
self.BUFF = 0
self.topmargin = 0
self.nvideos, self.videoheight, self.videowidth, self.videot = self.videos.shape
if batchsize > self.videot:
batch = np.zeros((self.model.D, batchsize))
done = False
batch_remaining = batchsize
t0 = 0
while not done:
tsz = min(self.videot, batch_remaining)
batch0 = self.crop_videos(tsz)
batch[:, t0:t0 + tsz] = batch0.reshape(self.model.D, tsz)
t0 += tsz
batch_remaining -= tsz
if batch_remaining <= 0: done = True
else:
batch = self.crop_videos(batchsize).reshape(self.model.D, batchsize)
return batch.astype(np.double)
elif self.datasource == 'PLoS09_Cars':
if not hasattr(self, 'videos'):
from kairos.data import PLoS09
#movie_size = (96,96)
movie_size = self.kargs.get('movie_size', (40, 40))
temp_videos, temp_chunk_labels = PLoS09.load_movies(movie_size)
videos = []
chunk_labels = []
for chunk_ind, temp_chunk_label in enumerate(temp_chunk_labels):
if temp_chunk_label == 'c':
videos.append(temp_videos[chunk_ind, ...])
chunk_labels.append(temp_chunk_label)
self.videos = np.asarray(videos)
self.chunk_labels = chunk_labels
self.BUFF = 0
self.topmargin = 0
self.nvideos, self.videoheight, self.videowidth, self.videot = self.videos.shape
if batchsize > self.videot:
batch = np.zeros((self.model.D, batchsize))
done = False
batch_remaining = batchsize
t0 = 0
while not done:
tsz = min(self.videot, batch_remaining)
batch0 = self.crop_videos(tsz)
batch[:, t0:t0 + tsz] = batch0.reshape(self.model.D, tsz)
t0 += tsz
batch_remaining -= tsz
if batch_remaining <= 0: done = True
else:
batch = self.crop_videos(batchsize).reshape(self.model.D, batchsize)
return batch.astype(np.double)
elif self.datasource == 'PandaCarVsPlane':
if not hasattr(self, 'videos'):
from kairos.data import pandaworld
#movie_size = (96,96)
movie_size = self.kargs.get('movie_size', (40, 40))
self._databaseobject = pandaworld.PandaCarVsPlane(movie_size=movie_size)
self.labels = self._databaseobject.y
szt, szy, szx = self._databaseobject.x.shape
self.videos = np.zeros((1,szy,szx,szt),dtype=self._databaseobject.x.dtype)
for t in range(szt):
self.videos[0,:,:,t] = self._databaseobject.x[t,:,:]
self.chunk_labels = self._databaseobject.y
self.topmargin = 0
self.BUFF = 0
self.nvideos, self.videoheight, self.videowidth, self.videot = self.videos.shape
if batchsize > self.videot:
batch = np.zeros((self.model.D, batchsize))
done = False
batch_remaining = batchsize
t0 = 0
while not done:
tsz = min(self.videot, batch_remaining)
batch0 = self.crop_videos(tsz)
batch[:, t0:t0 + tsz] = batch0.reshape(self.model.D, tsz)
t0 += tsz
batch_remaining -= tsz
if batch_remaining <= 0: done = True
else:
batch = self.crop_videos(batchsize).reshape(self.model.D, batchsize)
return batch.astype(np.double)
elif self.datasource == 'TorontoFaces48':
from scipy.io import loadmat
from config import public_dir
patch_sz = self.model.patch_sz
if not hasattr(self,'images'):
loadfile = os.path.join(public_dir,'TorontoFaces','TFD_ranzato_48x48.mat')
mfile = loadmat(loadfile)
self.images = mfile['images']
batch = np.zeros((self.model.D,batchsize))
for i in xrange(batchsize):
t,x,y = self.images.shape
rt = np.random.randint(t)
rx = np.random.randint(x-patch_sz+1)
ry = np.random.randint(y-patch_sz+1)
batch[:,i] = self.images[rt,rx:rx+patch_sz,ry:ry+patch_sz].ravel()
return batch
elif self.datasource == 'TorontoFaces96':
from scipy.io import loadmat
from config import public_dir
patch_sz = self.model.patch_sz
if not hasattr(self,'images'):
loadfile = os.path.join(public_dir,'TorontoFaces','TFD_ranzato_96x96.mat')
mfile = loadmat(loadfile)
self.images = mfile['images']
batch = np.zeros((self.model.D,batchsize))
for i in xrange(batchsize):
t,x,y = self.images.shape
rt = np.random.randint(t)
rx = np.random.randint(x-patch_sz+1)
ry = np.random.randint(y-patch_sz+1)
batch[:,i] = self.images[rt,rx:rx+patch_sz,ry:ry+patch_sz].ravel()
return batch
elif self.datasource == '3Dvideo_color':
from config import data_dir
import tables
if not hasattr(self,'videos'):
print 'Loading 3Dvideo_color'
self.video_buffer = 100
self.video_reload = 200
self.video_counter = 0
self.binocular_offset = +13
videopath = os.path.join(data_dir,'3Dvideo','processed')
self.video_list = [os.path.join(videopath,item) for item in filter(lambda x: x.count('.h5'),os.listdir(videopath))]
self.videos = []
for video_ind in range(self.video_buffer):
rind = np.random.randint(len(self.video_list))
h5file = tables.openFile(self.video_list[rind])
binocvideo = np.hstack((h5file.root.left.read(),h5file.root.right.read()))
self.videos.append(binocvideo)
h5file.close()
self.video_counter += 1
if not self.video_counter%self.video_reload:
rind = np.random.randint(len(self.video_list))
print 'Load video', self.video_list[rind]
h5file = tables.openFile(self.video_list[rind])
binocvideo = np.hstack((h5file.root.left.read(),h5file.root.right.read()))
h5file.close()
self.videos.pop()
self.videos = [binocvideo] + self.videos
vidind = np.random.randint(self.video_buffer)
video = self.videos[vidind]
if batchsize > video.shape[0]:
batch = np.zeros((self.model.D,batchsize))
done = False
batch_remaining = batchsize
t0 = 0
while not done:
batch0 = None
tsz = 0
while batch0 is None:
vidind = np.random.randint(self.video_buffer)
video = self.videos[vidind]
tsz = min(video.shape[0],batch_remaining)
batch0 = self.crop_single_binoc_video(video,tsz)
batch[:,t0:t0+tsz] = batch0.reshape(tsz,self.model.D).T
t0 += tsz
batch_remaining -= tsz
if batch_remaining <= 0: done = True
else:
batch0 = None
while batch0 is None:
batch0 = self.crop_single_binoc_video(video,batchsize)
batch = np.double(batch0.reshape(batchsize,self.model.D).T)
return batch
elif self.datasource == 'randn':
return np.random.randn(self.model.D,batchsize)
elif self.datasource == 'X':
if not hasattr(self,'X'): self.X = self.kargs['input_data']
batch = np.zeros((self.model.D,batchsize))
for i in xrange(batchsize):
t,x = self.X.shape
rt = np.random.randint(t)
batch[:,i] = self.X[rt,:].ravel()
return batch
else:
assert NotImplementedError, self.datasource
