def __init__(self, flatten=False, **kwargs):
im_dir = os.environ['MOZI_DATA_PATH'] + '/cifar10/'
self.label_names = ['airplane', 'automobile', 'bird', 'cat', 'deer',
'dog', 'frog','horse','ship','truck']
path = 'http://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz'
im_dir = get_file(fpath="{}/cifar-10-python.tar.gz".format(im_dir), origin=path, untar=True)
self.img_shape = (3,32,32)
self.img_size = np.prod(self.img_shape)
self.n_classes = 10
fnames = ['data_batch_%i' % i for i in range(1,6)] + ['test_batch']
X = []
y = []
for fname in fnames:
data_path = "{}/{}".format(im_dir, fname)
with open(data_path) as fin:
data_batch = cPickle.load(fin)
if flatten:
X.extend(data_batch['data'].reshape((len(data_batch['data']), self.img_size)))
else:
X.extend(data_batch['data'].reshape((len(data_batch['data']),)+self.img_shape))
y.extend(data_batch['labels'])
X_npy = np.array(X, dtype=floatX)
X_npy /= 255.0
y_npy = make_one_hot(y, onehot_size=self.n_classes)
super(Cifar10, self).__init__(X=X_npy, y=y_npy, **kwargs)
def __init__(self, resized_shape=(222, 222, 3), **kwargs):
'''
using only voc 2012 for actions classification, total 2154 images
resized_shape is of (height, width, channel)
'''
im_dir = os.environ['MOZI_DATA_PATH'] + '/voc'
path = 'http://host.robots.ox.ac.uk/pascal/VOC/voc2012/VOCtrainval_11-May-2012.tar'
im_dir = get_file(
fpath="{}/VOCtrainval_11-May-2012.tar".format(im_dir),
origin=path,
untar=True)
actls = [
'jumping', 'phoning', 'playinginstrument', 'reading', 'ridingbike',
'ridinghorse', 'running', 'takingphoto', 'usingcomputer',
'walking', 'other'
]
X_path = os.environ['MOZI_DATA_PATH'] + '/voc/X.npy'
y_path = os.environ['MOZI_DATA_PATH'] + '/voc/y.npy'
if not os.path.exists(X_path) or not os.path.exists(y_path):
print X_path + ' does not exists, generating..'
annote = im_dir + '/VOC2012/Annotations'
images = im_dir + '/VOC2012/JPEGImages'
files = glob.glob(annote + '/2012*xml')
labels = []
rimage = []
for f in files:
bname = os.path.basename(f).rstrip('.xml')
image = imread('{}/{}.jpg'.format(images, bname))
rimage.append(resize(image, resized_shape))
tree = ET.parse(f)
root = tree.getroot()
actions = root.find('object').find('actions')
for act in actions:
if act.text == '1':
labels.append(actls.index(act.tag))
# only restrict to one action per photo
break
print 'saving data'
with open(X_path, 'wb') as Xout, open(y_path, 'wb') as yout:
X = np.asarray(rimage)
y = np.asarray(labels)
np.save(Xout, X)
np.save(yout, y)
else:
print X_path + ' exists, loading..'
with open(X_path, 'rb') as Xin, open(y_path, 'rb') as yin:
X = np.load(Xin)
y = np.load(yin)
super(VOC, self).__init__(X=np.rollaxis(X, 3, 1),
y=make_one_hot(y, len(actls)),
**kwargs)
def __init__(self, nb_words=None, skip_top=0, maxlen=None, seed=113,
pad_zero=False, start_char=1, oov_char=2, index_from=3, **kwargs):
'''
adapted from keras
'''
im_dir = os.environ['MOZI_DATA_PATH'] + '/imdb/'
path = "https://s3.amazonaws.com/text-datasets/imdb.pkl"
im_dir = get_file(fpath="{}/imdb.pkl".format(im_dir), origin=path, untar=False)
with open('{}/imdb.pkl'.format(im_dir)) as fin:
X, labels = np.load(fin)
np.random.seed(seed)
np.random.shuffle(X)
np.random.seed(seed)
np.random.shuffle(labels)
if start_char is not None:
X = [[start_char] + [w + index_from for w in x] for x in X]
elif index_from:
X = [[w + index_from for w in x] for x in X]
if maxlen:
new_X = []
new_labels = []
for x, y in zip(X, labels):
if len(x) < maxlen:
new_X.append(x)
new_labels.append(y)
X = new_X
labels = new_labels
if not nb_words:
nb_words = max([max(x) for x in X])
# by convention, use 2 as OOV word
# reserve 'index_from' (=3 by default) characters: 0 (padding), 1 (start), 2 (OOV)
if oov_char is not None:
X = [[oov_char if (w >= nb_words or w < skip_top) else w for w in x] for x in X]
else:
nX = []
for x in X:
nx = []
for w in x:
if (w >= nb_words or w < skip_top):
nx.append(w)
nX.append(nx)
X = nX
if pad_zero and maxlen:
X = pad_sequences(X, maxlen=maxlen)
super(IMDB, self).__init__(X=np.asarray(X), y=np.asarray(labels).reshape((len(labels),1)), **kwargs)
def __init__(self, nb_words=None, skip_top=0, maxlen=None, seed=113,
pad_zero=False, start_char=1, oov_char=2, index_from=3, **kwargs):
'''
adapted from keras
'''
im_dir = os.environ['MOZI_DATA_PATH'] + '/imdb/'
path = "https://s3.amazonaws.com/text-datasets/imdb.pkl"
im_dir = get_file(fpath="{}/imdb.pkl".format(im_dir), origin=path, untar=False)
with open('{}/imdb.pkl'.format(im_dir)) as fin:
X, labels = np.load(fin)
np.random.seed(seed)
np.random.shuffle(X)
np.random.seed(seed)
np.random.shuffle(labels)
if start_char is not None:
X = [[start_char] + [w + index_from for w in x] for x in X]
elif index_from:
X = [[w + index_from for w in x] for x in X]
if maxlen:
new_X = []
new_labels = []
for x, y in zip(X, labels):
if len(x) < maxlen:
new_X.append(x)
new_labels.append(y)
X = new_X
labels = new_labels
if not nb_words:
nb_words = max([max(x) for x in X])
# by convention, use 2 as OOV word
# reserve 'index_from' (=3 by default) characters: 0 (padding), 1 (start), 2 (OOV)
if oov_char is not None:
X = [[oov_char if (w >= nb_words or w < skip_top) else w for w in x] for x in X]
else:
nX = []
for x in X:
nx = []
for w in x:
if (w >= nb_words or w < skip_top):
nx.append(w)
nX.append(nx)
X = nX
if pad_zero and maxlen:
X = pad_sequences(X, maxlen=maxlen)
super(IMDB, self).__init__(X=np.asarray(X), y=np.asarray(labels).reshape((len(labels),1)), **kwargs)
def __init__(self, resized_shape=(222,222,3), **kwargs):
'''
using only voc 2012 for actions classification, total 2154 images
resized_shape is of (height, width, channel)
'''
im_dir = os.environ['MOZI_DATA_PATH'] + '/voc'
path = 'http://host.robots.ox.ac.uk/pascal/VOC/voc2012/VOCtrainval_11-May-2012.tar'
im_dir = get_file(fpath="{}/VOCtrainval_11-May-2012.tar".format(im_dir), origin=path, untar=True)
actls = ['jumping', 'phoning', 'playinginstrument', 'reading', 'ridingbike',
'ridinghorse', 'running', 'takingphoto', 'usingcomputer', 'walking',
'other']
X_path = os.environ['MOZI_DATA_PATH'] + '/voc/X.npy'
y_path = os.environ['MOZI_DATA_PATH'] + '/voc/y.npy'
if not os.path.exists(X_path) or not os.path.exists(y_path):
print X_path + ' does not exists, generating..'
annote = im_dir + '/VOC2012/Annotations'
images = im_dir + '/VOC2012/JPEGImages'
files = glob.glob(annote + '/2012*xml')
labels = []
rimage = []
for f in files:
bname = os.path.basename(f).rstrip('.xml')
image = imread('{}/{}.jpg'.format(images, bname))
rimage.append(resize(image, resized_shape))
tree = ET.parse(f)
root = tree.getroot()
actions = root.find('object').find('actions')
for act in actions:
if act.text == '1':
labels.append(actls.index(act.tag))
# only restrict to one action per photo
break
print 'saving data'
with open(X_path, 'wb') as Xout, open(y_path, 'wb') as yout:
X = np.asarray(rimage)
y = np.asarray(labels)
np.save(Xout, X)
np.save(yout, y)
else:
print X_path + ' exists, loading..'
with open(X_path, 'rb') as Xin, open(y_path, 'rb') as yin:
X = np.load(Xin)
y = np.load(yin)
super(VOC, self).__init__(X=np.rollaxis(X,3,1), y=make_one_hot(y,len(actls)), **kwargs)
def __init__(self, flatten=False, fine_label=True, **kwargs):
'''
PARAM:
fine_label: True (100 classes) False (20 classes)
'''
im_dir = os.environ['MOZI_DATA_PATH'] + '/cifar100/'
path = 'http://www.cs.toronto.edu/~kriz/cifar-100-python.tar.gz'
im_dir = get_file(fpath="{}/cifar-100-python.tar.gz".format(im_dir),
origin=path,
untar=True)
self.img_shape = (3, 32, 32)
self.img_size = np.prod(self.img_shape)
fnames = ['train', 'test']
X = []
y = []
for fname in fnames:
data_path = "{}/{}".format(im_dir, fname)
with open(data_path) as fin:
data_batch = cPickle.load(fin)
if flatten:
X.extend(data_batch['data'].reshape(
(len(data_batch['data']), self.img_size)))
else:
X.extend(data_batch['data'].reshape(
(len(data_batch['data']), ) + self.img_shape))
if fine_label:
y.extend(data_batch['fine_labels'])
self.n_classes = 100
else:
y.extend(data_batch['coarse_labels'])
self.n_classes = 20
X_npy = np.array(X, dtype=floatX)
X_npy /= 255.0
y_npy = make_one_hot(y, onehot_size=self.n_classes)
super(Cifar100, self).__init__(X=X_npy, y=y_npy, **kwargs)
def __init__(self, flatten=False, **kwargs):
im_dir = os.environ['MOZI_DATA_PATH'] + '/cifar10/'
path = 'http://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz'
im_dir = get_file(fpath="{}/cifar-10-python.tar.gz".format(im_dir),
origin=path,
untar=True)
self.label_names = [
'airplane', 'automobile', 'bird', 'cat', 'deer', 'dog', 'frog',
'horse', 'ship', 'truck'
]
self.img_shape = (3, 32, 32)
self.img_size = np.prod(self.img_shape)
self.n_classes = 10
fnames = ['data_batch_%i' % i for i in range(1, 6)] + ['test_batch']
X = []
y = []
for fname in fnames:
data_path = "{}/{}".format(im_dir, fname)
with open(data_path) as fin:
data_batch = cPickle.load(fin)
if flatten:
X.extend(data_batch['data'].reshape(
(len(data_batch['data']), self.img_size)))
else:
X.extend(data_batch['data'].reshape(
(len(data_batch['data']), ) + self.img_shape))
y.extend(data_batch['labels'])
X_npy = np.array(X, dtype=floatX)
X_npy /= 255.0
y_npy = make_one_hot(y, onehot_size=self.n_classes)
ridx = np.arange(len(X_npy))
np.random.shuffle(ridx)
X_npy = X_npy[ridx]
y_npy = y_npy[ridx]
super(Cifar10, self).__init__(X=X_npy, y=y_npy, **kwargs)
def __init__(self, flatten=False, fine_label=True, **kwargs):
'''
PARAM:
fine_label: True (100 classes) False (20 classes)
'''
im_dir = os.environ['MOZI_DATA_PATH'] + '/cifar100/'
path = 'http://www.cs.toronto.edu/~kriz/cifar-100-python.tar.gz'
im_dir = get_file(fpath="{}/cifar-100-python.tar.gz".format(im_dir), origin=path, untar=True)
self.img_shape = (3,32,32)
self.img_size = np.prod(self.img_shape)
fnames = ['train', 'test']
X = []
y = []
for fname in fnames:
data_path = "{}/{}".format(im_dir, fname)
with open(data_path) as fin:
data_batch = cPickle.load(fin)
if flatten:
X.extend(data_batch['data'].reshape((len(data_batch['data']), self.img_size)))
else:
X.extend(data_batch['data'].reshape((len(data_batch['data']),)+self.img_shape))
if fine_label:
y.extend(data_batch['fine_labels'])
self.n_classes = 100
else:
y.extend(data_batch['coarse_labels'])
self.n_classes = 20
X_npy = np.array(X, dtype=floatX)
X_npy /= 255.0
y_npy = make_one_hot(y, onehot_size=self.n_classes)
super(Cifar100, self).__init__(X=X_npy, y=y_npy, **kwargs)
