def test(files_):
global jobs
if use.drop: # dont use dropout when testing
drop.p_traj.set_value(float32(0.))
drop.p_vid.set_value(float32(0.))
drop.p_hidden.set_value(float32(0.))
ce = []
first_test_file = True
# use.aug=False
for file in files_:
if first_test_file:
augm = False
first_test_file = False
else: augm = True
load_data(file, augm,istest=True)
ce.append(_batch(test_model, False))
if use.drop: # reset dropout
drop.p_traj.set_value(drop.p_traj_val)
drop.p_vid.set_value(drop.p_vid_val)
drop.p_hidden.set_value(drop.p_hidden_val)
if use.aug:
start_load(files.train,augm=use.aug)
return _avg(ce)
def test(files_):
global jobs
if use.drop: # dont use dropout when testing
drop.p_traj.set_value(float32(0.))
drop.p_vid.set_value(float32(0.))
drop.p_hidden.set_value(float32(0.))
ce = []
first_test_file = True
# use.aug=False
for file in files_:
if first_test_file:
augm = False
first_test_file = False
else:
augm = True
load_data(file, augm, istest=True)
ce.append(_batch(test_model, False))
if use.drop: # reset dropout
drop.p_traj.set_value(drop.p_traj_val)
drop.p_vid.set_value(drop.p_vid_val)
drop.p_hidden.set_value(drop.p_hidden_val)
if use.aug:
start_load(files.train, augm=use.aug)
return _avg(ce)
def load_data(path, trans):
global rng, x_,t_,y_,first_report2
""" load data into shared variables """
# if trans and use.aug:
# transform(path) # que up the path for augmentation
# vid, traj, lbl = load_aug(path)
# else:
# vid, traj, lbl = load_normal(path)
# file = GzipFile(path, 'rb')
# vid, skel, lbl = load(file)
# file.close()
# traj,ori,pheight = skel
# print path
# import cv2
# for img in vid[0,0,0]:
# cv2.imshow("Video", img)
# cv2.waitKey(0)
# for img in vid[0,0,1]:
# cv2.imshow("Video", img)
# cv2.waitKey(0)
# new_vid = empty(in_shape,dtype="uint8")
# vid_ = vid[:,0,:2,:,::2,::2]
# vid_ = vid[:,0,:2]
# zm = 1.*90./128.
# vid_ = ndimage.zoom(vid_,(1,1,1,zm,zm),order=0)
# new_vid[:,0] = vid_
# new_vid[:,1] = vid[:,1,:2]
# print "loading..."
start_time = time()
if not trans:
start_load(files.valid,jobs,False)
vid, skel, lbl = queue.get()[0]
traj,ori,pheight = skel
print "get in",str(time()-start_time)[:3]+"s",
# shuffle data
ind = rng.permutation(batch_size)
vid, traj, lbl = vid[ind].astype(floatX), traj[ind].astype(floatX),lbl[ind].astype(floatX)
# vid = vid/(255./(scaler*2.))-scaler
# traj = traj/(255./(scaler_traj*2.))-scaler_traj
# traj = traj/(255./5.)
lbl -= 1
if first_report2:
print "data range:",vid.min(),vid.max()
print "traj range:",traj.min(),traj.max()
print "lbl range:",lbl.min(),lbl.max()
first_report2 = False
# set value
x_.set_value(vid, borrow=True)
t_.set_value(traj, borrow=True)
y_.set_value(lbl, borrow=True)
_files = glob(src+'/batch_100_*.zip')+glob(src+'/valid/batch_100_*.zip')
# _files = glob(src+'_train/batch_100_*.p')
# _files.sort()
# _files = _files[:10]
rng.shuffle(_files)
class files:
data_files = _files
n_train = int(len(data_files) * .8)
n_valid = int(len(data_files) * .2)
train = data_files[:n_train]
valid = data_files[n_train:n_train+n_valid]
if use.valid2: valid2 = data_files[n_train+n_valid:]
# valid2 = glob(src+'_valid/batch_100_*.p')
# data augmentation
jobs,queue = start_load(files.train,augm=use.aug,start=True)
# print data sizes
if use.valid2: files.n_test = len(files.valid2)
else: files.n_test = 0
write('data: total: %i train: %i valid: %i test: %i' % \
((files.n_test+files.n_train+files.n_valid)*batch_size,
files.n_train*batch_size,
files.n_valid*batch_size,
files.n_test*batch_size))
first_report2 = True
def load_data(path, trans):
global rng, x_,t_,y_,first_report2
""" load data into shared variables """
def load_data(path, trans):
global rng, x_, t_, y_, first_report2
""" load data into shared variables """
# if trans and use.aug:
# transform(path) # que up the path for augmentation
# vid, traj, lbl = load_aug(path)
# else:
# vid, traj, lbl = load_normal(path)
# file = GzipFile(path, 'rb')
# vid, skel, lbl = load(file)
# file.close()
# traj,ori,pheight = skel
# print path
# import cv2
# for img in vid[0,0,0]:
# cv2.imshow("Video", img)
# cv2.waitKey(0)
# for img in vid[0,0,1]:
# cv2.imshow("Video", img)
# cv2.waitKey(0)
# new_vid = empty(in_shape,dtype="uint8")
# vid_ = vid[:,0,:2,:,::2,::2]
# vid_ = vid[:,0,:2]
# zm = 1.*90./128.
# vid_ = ndimage.zoom(vid_,(1,1,1,zm,zm),order=0)
# new_vid[:,0] = vid_
# new_vid[:,1] = vid[:,1,:2]
# print "loading..."
start_time = time()
if not trans:
start_load(files.valid, jobs, False)
vid, skel, lbl = queue.get()[0]
traj, ori, pheight = skel
print "get in", str(time() - start_time)[:3] + "s",
# shuffle data
ind = rng.permutation(batch_size)
vid, traj, lbl = vid[ind].astype(floatX), traj[ind].astype(
floatX), lbl[ind].astype(floatX)
# vid = vid/(255./(scaler*2.))-scaler
# traj = traj/(255./(scaler_traj*2.))-scaler_traj
# traj = traj/(255./5.)
lbl -= 1
if first_report2:
print "data range:", vid.min(), vid.max()
print "traj range:", traj.min(), traj.max()
print "lbl range:", lbl.min(), lbl.max()
first_report2 = False
# set value
x_.set_value(vid, borrow=True)
t_.set_value(traj, borrow=True)
y_.set_value(lbl, borrow=True)
# _files = _files[:10]
rng.shuffle(_files)
class files:
data_files = _files
n_train = int(len(data_files) * .8)
n_valid = int(len(data_files) * .2)
train = data_files[:n_train]
valid = data_files[n_train:n_train + n_valid]
if use.valid2: valid2 = data_files[n_train + n_valid:]
# valid2 = glob(src+'_valid/batch_100_*.p')
# data augmentation
jobs, queue = start_load(files.train, augm=use.aug, start=True)
# print data sizes
if use.valid2: files.n_test = len(files.valid2)
else: files.n_test = 0
write('data: total: %i train: %i valid: %i test: %i' % \
((files.n_test+files.n_train+files.n_valid)*batch_size,
files.n_train*batch_size,
files.n_valid*batch_size,
files.n_test*batch_size))
first_report2 = True
def load_data(path, trans):
global rng, x_, t_, y_, first_report2
