# snapshot_epoch=False, run_id='googlenet_cs249_aug')
#model.save('googlenet_aug')
model.load('googlenet_aug')
test_path = '/home/ubuntu/cs249_final_project/test'
import os
f = open('test.txt', 'w')
for filename in os.listdir(test_path):
f.write(
os.path.join(test_path, filename) + ' ' +
os.path.splitext(filename)[0] + '\n')
f.close()
build_hdf5_image_dataset('test.txt',
image_shape=(300, 300),
mode='file',
categorical_labels=False,
output_path='testset.h5')
h5f = h5py.File('testset.h5', 'r')
X = h5f['X']
Y = h5f['Y']
predict = model.predict(X)
import csv
with open('submit.csv', 'w') as csvfile:
writer = csv.writer(csvfile, delimiter=',')
writer.writerow(['image_name', 'Type_1', 'Type_2', 'Type_3'])
for i in range(len(predict)):
filename = str(int(Y[i])) + '.jpg'
writer.writerow([filename] + predict[i])
from tflearn.data_utils import build_hdf5_image_dataset
import h5py
path = '/home/suger/workspace/pig-face-recognition/raw_data/txt/'
# filenum = 1
# filename = 'train_data'
# files = []
# result = []
# for i in range(0, filenum):
# files.append(path + filename + str(i) + '.txt')
# result.append(filename + str(i) + '.h5')
# build_hdf5_image_dataset(files[i], image_shape=(448, 448), mode='file', output_path=result[i], categorical_labels=True, normalize=False)
# print('Finish dataset ' + result[i])
filenum = 1
filename = 'validation_data'
files = []
result = []
for i in range(0, filenum):
files.append(path + filename + str(i) + '.txt')
result.append(filename + str(i) + '.h5')
build_hdf5_image_dataset(files[i],
image_shape=(448, 448),
mode='file',
output_path=result[i],
categorical_labels=True,
normalize=False)
print('Finish dataset ' + result[i])
@author: sanat
"""
from __future__ import division, print_function, absolute_import
import tflearn
from tflearn.layers.core import input_data, dropout, fully_connected
from tflearn.layers.conv import conv_2d, max_pool_2d
from tflearn.layers.normalization import local_response_normalization
from tflearn.layers.estimator import regression
from tflearn.data_utils import build_hdf5_image_dataset
import tflearn.datasets.oxflower17 as oxflower17
import h5py
import math
X, Y = oxflower17.load_data(one_hot=True, resize_pics=(227, 227))
"""train_dataset_file = '/home/sanat/Desktop/CDSAML/imagepaths(train).txt'
val_dataset_file = '/home/sanat/Desktop/CDSAML/imagepaths(val).txt'
build_hdf5_image_dataset(train_dataset_file, image_shape=(128, 128), mode='file', output_path='dataset.h5', categorical_labels=True, normalize=True)
h5f = h5py.File('dataset.h5', 'r')
X = h5f['X']
Y = h5f['Y']
X_val, Y_val = image_preloader(val_dataset_file, image_shape=(227, 227),
mode='file', categorical_labels=True,
normalize=True)"""
# Building 'Alexnet 2.0'
network = input_data(shape=[None, 227, 227, 3])
"""layer 1:conv+max+norm"""
"""
from __future__ import division, print_function, absolute_import
import tflearn
from tflearn.layers.core import input_data, dropout, fully_connected
from tflearn.layers.conv import conv_2d, max_pool_2d
from tflearn.layers.normalization import local_response_normalization
from tflearn.layers.estimator import regression
dataset_file = r'/home/ubuntu/cs249_final_project/image_files/train'
from tflearn.data_utils import build_hdf5_image_dataset
build_hdf5_image_dataset(dataset_file,
image_shape=(300, 300),
mode='folder',
output_path='dataset.h5',
categorical_labels=True,
normalize=True)
import h5py
h5f = h5py.File('dataset.h5', 'r')
X = h5f['X']
Y = h5f['Y']
net = tflearn.input_data(shape=[None, 300, 300, 3])
net = tflearn.conv_2d(net, 64, 3, activation='relu', bias=False)
# Residual blocks
net = tflearn.residual_bottleneck(net, 3, 16, 64)
net = tflearn.residual_bottleneck(net, 1, 32, 128, downsample=True)
net = tflearn.residual_bottleneck(net, 2, 32, 128)
net = tflearn.residual_bottleneck(net, 1, 64, 256, downsample=True)
from tqdm import tqdm
def get_label(windows_path):
fileName = (str(windows_path)).split('\\')[-1]
label = int(fileName[0:fileName.find('.')].split('_')[1]) - 1
return label
if __name__ == '__main__':
for name in ['train', 'val']:
print('{0} h5 file start to create'.format(name))
directory = pathlib.Path('data/' + name)
h5plan_name = pathlib.Path('h5plan.txt')
output_filename = pathlib.Path(name + '_128.h5')
with open(h5plan_name, "w") as text_file:
for windows_path in tqdm(list(directory.glob('*.jpg'))):
text_file.write('{0} {1}\n'.format(str(windows_path),
get_label(windows_path)))
build_hdf5_image_dataset(h5plan_name,
image_shape=(128, 128),
mode='file',
output_path=output_filename,
categorical_labels=True,
normalize=True)
print('{0} h5 file created'.format(name))
train_data = h5py.File(output_filename, 'r')
X_train, Y_train = train_data['X'], train_data['Y']
print(Y_train[:])
##import tflearn.datasets.oxflower17 as oxflower17
##X, Y = oxflower17.load_data(one_hot=True, resize_pics=(227, 227))
import tensorflow as tf
import tflearn
import h5py
from tflearn.data_utils import build_hdf5_image_dataset
##Building Dataset
root_folder = './cacoons/jpg/'
print('-----------Start making dataset!----------')
if not os.path.exists('cacoon.h5'):
build_hdf5_image_dataset(root_folder,
image_shape=(227, 227),
mode='folder',
output_path='cacoon.h5',
categorical_labels=True,
normalize=False)
print('------------Dataset is prepared!-------------- ')
h5f = h5py.File('cacoon.h5', 'r')
X = h5f['X']
Y = h5f['Y']
# Building 'AlexNet'
network = input_data(shape=[None, 227, 227, 3])
network = conv_2d(network, 48, 11, strides=4, activation='relu')
network = max_pool_2d(network, 3, strides=2)
network = local_response_normalization(network)
network = conv_2d(network, 128, 5, activation='relu')
network = max_pool_2d(network, 3, strides=2)
network = local_response_normalization(network)
def data_prep0(conf, method='random', training_size=0.7):
'''
create hdf5 files based on train and test selecting strategy
and return results as train and test sets
it augments train data
:param conf: parameters from congigure file
:param method: random / sequence; at sequence get first training_size of images as training
:param training_size: portion of images that will be used in trainig
:return: X_train, Y_train, X_test, Y_test
'''
data_folder = conf['data_folder']
test_counter = 0
counter = 0
train_file = data_folder + 'train.txt'
test_file = data_folder + 'test.txt'
with open(data_folder + 'labeling.csv') as inputFile:
df = pd.read_csv(data_folder + 'labeling.csv')
with open(train_file, 'w+') as trainFile:
with open(test_file, 'w+') as testFile:
for row in df.iterrows():
row = row[1]
counter += 1
test_counter += 1
line = data_folder + 'ercis/' + str(int(row['index'] - 1)).zfill(5) + '.tif' \
+ ' ' + str(int(row['manual_label'])) + '\n'
if method == 'random':
if random.random() < training_size:
trainFile.write(line)
else:
testFile.write(line)
trainFile.write(line)
test_counter += 1
else:
if counter < 37204 * training_size: # random.random() < train_size:
trainFile.write(line)
else:
testFile.write(line)
trainFile.write(line)
test_counter += 1
# Build a HDF5 dataset (only required once)
build_hdf5_image_dataset(train_file,
image_shape=(40, 40),
mode='file',
output_path=data_folder + 'train.h5',
categorical_labels=True,
normalize=True)
build_hdf5_image_dataset(test_file,
image_shape=(40, 40),
mode='file',
output_path=data_folder + 'test.h5',
categorical_labels=True,
normalize=True)
# Load HDF5 dataset
h5f = h5py.File('train.h5', 'w')
X_train = h5f['X']
Y_train = h5f['Y']
h5f = h5py.File('test.h5', 'w')
X_test = h5f['X']
Y_test = h5f['Y']
return X_train, Y_train, X_test, Y_test
from tflearn.data_utils import build_hdf5_image_dataset
import h5py
new_train = "/home/adoke/tf_tutorial/aircrafts_new/new_train_val/new_train.txt"
new_val = "/home/adoke/tf_tutorial/aircrafts_new/new_train_val/new_val.txt"
new_test = "/home/adoke/tf_tutorial/aircrafts_new/from_start/a3_variants_test.txt"
# image_shape option can be set to different values to create images of different sizes
build_hdf5_image_dataset(new_val,
image_shape=(224, 224),
mode='file',
output_path='new_val_224.h5',
categorical_labels=True,
normalize=False)
print 'Done creating new_val.h5'
build_hdf5_image_dataset(new_test,
image_shape=(224, 224),
mode='file',
output_path='new_test_224.h5',
categorical_labels=True,
normalize=False)
print 'Done creating new_test.h5'
build_hdf5_image_dataset(new_train,
image_shape=(488, 488),
mode='file',
output_path='new_train_488.h5',
categorical_labels=True,
normalize=False)
print 'Done creating new_train_488.h5'
import sys
import csv
import numpy as np
import matplotlib.pyplot as plt
import tflearn
import tensorflow as tf
tf.reset_default_graph()
from tflearn.data_utils import load_csv
csv.field_size_limit(sys.maxsize)
# Build a HDF5 dataset (only required once)
from tflearn.data_utils import build_hdf5_image_dataset
build_hdf5_image_dataset(dataset_file,
image_shape=(250, 250),
mode='file',
output_path='your_data_path.h5',
categorical_labels=False,
normalize=True)
# Load HDF5 dataset
import h5py
h5f = h5py.File('your_data_path.h5', 'r')
X = h5f['X']
Y = h5f['Y']
network = tflearn.input_data(shape=[None, 250, 250, 3])
network = tflearn.conv_2d(network,
64,
10,
4,
activation='relu',
import os
from PIL import Image
from tflearn.data_utils import build_hdf5_image_dataset
image_shape = (32, 32)
dataset_file = "classes.txt"
output_path = "dataset.h5"
for file_name in os.listdir("."):
if os.path.isdir(file_name):
for image_name in os.listdir(file_name):
image_path = "{}/{}".format(file_name, image_name)
image = Image.open(image_path)
image = image.resize(image_shape)
image.save(image_path)
build_hdf5_image_dataset(dataset_file,
image_shape=image_shape,
output_path=output_path,
mode="file",
categorical_labels=True,
normalize=True,
grayscale=False)
import tensorflow as tf
import tflearn
import os
from tflearn.data_preprocessing import ImagePreprocessing
from tflearn.data_utils import build_hdf5_image_dataset
import h5py
data_dir = 'TU-Berlin'
build_hdf5_image_dataset(data_dir,
image_shape=(224, 224),
mode='folder',
output_path='dataset.h5',
categorical_labels=True,
normalize=True)
h5f = h5py.File('dataset.h5', 'r')
X = h5f['X']
Y = h5f['Y']
print(X.shape)
# new_train = "/home/adoke/tf_tutorial/aircrafts_new/new_train_val/new_train.txt"
# new_val = "/home/adoke/tf_tutorial/aircrafts_new/new_train_val/new_val.txt"
# new_test = "/home/adoke/tf_tutorial/aircrafts_new/from_start/a3_variants_test.txt"
#
# # image_shape option can be set to different values to create images of different sizes
# build_hdf5_image_dataset(new_val, image_shape=(224, 224), mode='file', output_path='new_val_224.h5', categorical_labels=True, normalize=False)
# print ('Done creating new_val.h5')
# build_hdf5_image_dataset(new_test, image_shape=(224, 224), mode='file', output_path='new_test_224.h5', categorical_labels=True, normalize=False)
# print ('Done creating new_test.h5')
# build_hdf5_image_dataset(new_train, image_shape=(488, 488), mode='file', output_path='new_train_488.h5', categorical_labels=True, normalize=False)
# print ('Done creating new_train_488.h5')
from tflearn.data_utils import build_hdf5_image_dataset
trainfile = "../../data/train_data/"
validfile = "../../data/valid_data/"
totaltrain = "../../data/train/"
build_hdf5_image_dataset(trainfile, image_shape=(448, 448),
mode='folder', output_path='../data/train.h5',
categorical_labels=True, normalize=False, files_extension=['.JPEG'])
print('Done creating train.h5')
build_hdf5_image_dataset(validfile, image_shape=(224, 224),
mode='folder', output_path='../data/valid.h5',
categorical_labels=True, normalize=False, files_extension=['.jpg'])
print('Done creating valid.h5')
# build_hdf5_image_dataset(trainfile, image_shape=(448, 448),
# mode='folder', output_path='../data/train.h5',
# categorical_labels=True, normalize=False, files_extension=['.JPEG'])
from tflearn.data_utils import build_hdf5_image_dataset
import h5py
#new_train = "/home/adoke/tf_tutorial/aircrafts_new/new_train_val/new_train.txt"
#new_val = "/home/adoke/tf_tutorial/aircrafts_new/new_train_val/new_val.txt"
#new_test = "/home/adoke/tf_tutorial/aircrafts_new/from_start/a3_variants_test.txt"
new_train = "/media/goerlab/My Passport/Welder_detection/dataset/20180209/h5_middle_crop/train2.txt"
new_val = "/media/goerlab/My Passport/Welder_detection/dataset/20180209/h5_middle_crop/val.txt"
#new_test="/home/goerlab/Bilinear-CNN-TensorFlow/train_test_small/images_test.txt"
dir = "/media/goerlab/My Passport/Welder_detection/dataset/20180209/h5_middle_crop/"
#image_shape option can be set to different values to create images of different sizes
# build_hdf5_image_dataset(new_val, image_shape=(448, 448), mode='file', output_path=dir+'new_val_448.h5', categorical_labels=True, normalize=False)
#
# #
# print 'Done creating new_val.h5'
#build_hdf5_image_dataset(new_test, image_shape=(448, 448), mode='file', output_path=dir+'new_test_224.h5', categorical_labels=True, normalize=False)
#print 'Done creating new_test.h5'
build_hdf5_image_dataset(new_train,
image_shape=(448, 448),
mode='file',
output_path=dir + 'new_train_448.h5',
categorical_labels=True,
normalize=False)
print 'Done creating new_train_488.h5'
DATA_DIR = os.environ['HOME']
train = pd.read_csv(TRAIN_FILE)
# split into training and test
f = open(DATASET_FILE, 'w')
for idx, row in training.iterrows():
file_path = os.environ['HOME'] + '/' + row['file_path'].split('/')[1]
folder = os.environ['HOME'] + '/' + row['file_path'].split('/')[0]
if not os.path.isfile(file_path): continue
# create folder if needed
# crop image
# write logic to toss out poorly formatted images
line_string = '{} {}\\n'.format(file_path, row['expression'])
f.write(line_string)
f.close()
dataset_file = 'train_dataset_trim.txt'
build_hdf5_image_dataset(dataset_file,
image_shape=(IMG_SIZE, IMG_SIZE),
mode='file',
output_path=TRAIN_H5,
normalize=True,
categorical_labels=True)
build_hdf5_image_dataset(dataset_file,
image_shape=(IMG_SIZE, IMG_SIZE),
mode='file',
output_path=TRAIN_H5,
normalize=True,
categorical_labels=True)
def data_prep(conf, method='random', training_size=0.7, clean_start=False):
'''
create hdf5 files based on train and test selecting strategy
and return results as train and test sets
:param conf: parameters from congigure file
:param method: random / sequence; at sequence get first training_size of images as training
:param training_size: portion of images that will be used in trainig
:param start_clean: start from zero or work based on previous work
:return: X_train, Y_train, X_test, Y_test
'''
data_folder = conf['data_folder']
if clean_start:
os.remove(data_folder + 'train.h5')
os.remove(data_folder + 'test.h5')
#in the case files is available just read it and return
if os.path.isfile(data_folder +
'train.h5') and os.path.isfile(data_folder +
'validation.h5'):
_h5f = h5py.File(data_folder + 'train.h5', 'r')
X_train = _h5f['X']
Y_train = _h5f['Y']
h5f_ = h5py.File(data_folder + 'validation.h5', 'r')
X_test = h5f_['X']
Y_test = h5f_['Y']
return X_train, Y_train, X_test, Y_test
# if the file is not available, continue creating it
test_counter = 0
counter = 0
train_file = data_folder + 'train.txt'
test_file = data_folder + 'test.txt'
with open(data_folder + 'labeling.csv') as inputFile:
df = pd.read_csv(data_folder + 'labeling.csv')
with open(train_file, 'w+') as trainFile:
with open(test_file, 'w+') as testFile:
for row in df.iterrows():
row = row[1]
counter += 1
test_counter += 1
line = data_folder + 'ercis/' + str(int(row['index'] - 1)).zfill(5) + '.tif' \
+ ' ' + str(int(row['manual_label'])) + '\n'
if method == 'random':
if random.random() < training_size:
trainFile.write(line)
else:
testFile.write(line)
test_counter += 1
else:
if counter < 37204 * training_size: # random.random() < train_size:
trainFile.write(line)
else:
testFile.write(line)
test_counter += 1
# Build a HDF5 dataset (only required once)
build_hdf5_image_dataset(train_file,
image_shape=(40, 40),
mode='file',
output_path=data_folder + 'train.h5',
categorical_labels=True,
normalize=True)
build_hdf5_image_dataset(test_file,
image_shape=(40, 40),
mode='file',
output_path=data_folder + 'test.h5',
categorical_labels=True,
normalize=True)
# Load HDF5 dataset
_h5f = h5py.File(data_folder + 'train.h5', 'r')
X_train = _h5f['X']
Y_train = _h5f['Y']
h5f_ = h5py.File(data_folder + 'test.h5', 'r')
X_test = h5f_['X']
Y_test = h5f_['Y']
return X_train, Y_train, X_test, Y_test
dataset_file = 'my_dataset.txt'
# Build a HDF5 dataset (only required once)
from tflearn.data_utils import build_hdf5_image_dataset
TRAIN_DATA = '../train_data'
VAL_DATA = '../val_data'
build_hdf5_image_dataset(TRAIN_DATA,
image_shape=(128, 128),
mode='file',
output_path='dataset_train.h5',
categorical_labels=True,
normalize=True)
from tflearn.data_utils import build_hdf5_image_dataset import h5py new_train = "train.txt" new_val = "val.txt" new_test = "test.txt" # image_shape option can be set to different values to create images of different sizes build_hdf5_image_dataset(new_val, image_shape=(50, 50), mode='file', output_path='new_val.h5', categorical_labels=True, normalize=False) print 'Done creating new_val.h5' build_hdf5_image_dataset(new_test, image_shape=(50, 50), mode='file', output_path='new_test.h5', categorical_labels=True, normalize=False) print 'Done creating new_test.h5' build_hdf5_image_dataset(new_train, image_shape=(50, 50), mode='file', output_path='new_train.h5', categorical_labels=True, normalize=False) print 'Done creating new_train_488.h5'
#print(len(train))
print(tile_df[['cell_type_idx',
'cell_type']].sort_values('cell_type_idx').drop_duplicates())
sys.exit()
with open('path_train.data', 'w') as pf:
pf.writelines(train)
with open('path_val.data', 'w') as pf:
pf.writelines(val)
build_hdf5_image_dataset(
'path_train.data',
image_shape=(224, 224),
mode='file',
output_path='train.h5',
categorical_labels=True,
normalize=True,
grayscale=False,
)
build_hdf5_image_dataset(
'path_val.data',
image_shape=(224, 224),
mode='file',
output_path='val.h5',
categorical_labels=True,
normalize=True,
grayscale=False,
)
# Load path/class_id image file:
# dataset_origin_train = './data/original_jpg_structured_train-validation-only/'
# dataset_origin_test = './data/original_jpg_structured_test-only/'
dataset_train_hdf5 = 'coal70_train_dataset.h5'
dataset_test_hdf5 = 'coal70_test_dataset.h5'
# Build a HDF5 dataset (only required once)
from tflearn.data_utils import build_hdf5_image_dataset
dataset_hdf5_file = Path(dataset_train_hdf5)
if not dataset_hdf5_file.exists():
print("Creating", dataset_train_hdf5)
build_hdf5_image_dataset(args['dataset_origin_train'],
image_shape=(256, 256),
mode='folder',
output_path=dataset_train_hdf5,
categorical_labels=True,
normalize=True)
dataset_hdf5_file = Path(dataset_test_hdf5)
if not dataset_hdf5_file.exists():
print("Creating", dataset_test_hdf5)
build_hdf5_image_dataset(args['dataset_origin_test'],
image_shape=(256, 256),
mode='folder',
output_path=dataset_test_hdf5,
categorical_labels=True,
normalize=True)
# Load HDF5 dataset
h5f = h5py.File(dataset_train_hdf5, 'r')
from tflearn.data_utils import build_hdf5_image_dataset
import h5py
new_train = "/home/meteo/zihao.chen/fine-grained-classifi/train_test/train_data"
new_val = "/home/meteo/zihao.chen/fine-grained-classifi/train_test/validation_data.txt"
new_test = "/home/meteo/zihao.chen/fine-grained-classifi/train_test/test_data.txt"
upload_test = '/home/meteo/zihao.chen/fine-grained-classifi/TEST_FIlE'
# image_shape option can be set to different values to create images of different sizes
# build_hdf5_image_dataset(new_val, image_shape=(448, 448), mode='file', output_path='new_val_448.h5', categorical_labels=True, normalize=False)
# print 'Done creating new_val_448.h5'
# build_hdf5_image_dataset(new_test, image_shape=(448, 448), mode='file', output_path='new_test_448.h5', categorical_labels=True, normalize=False)
# print 'Done creating new_test_448bc .h5'
# build_hdf5_image_dataset(new_train+'_%d.txt'%(0), image_shape=(224, 224), mode='file', output_path='new_train_224_%d.h5'% (0), categorical_labels=True, normalize=False)
for index in range(11):
build_hdf5_image_dataset(upload_test + '/Test_%d.txt' % index,
image_shape=(448, 448),
mode='file',
output_path='TEST_FIlE/upload_test_448_%d.h5' %
index,
categorical_labels=True,
normalize=False)
print 'Done creating new_train_448.h5'
lambda x: mode + '/image_' + str(x) + '.jpg')
filenames = filenames.values.astype(str)
labels = y.values.astype(int)
data = np.zeros(filenames.size, dtype=[('var1', 'S36'), ('var2', int)])
data['var1'] = filenames
data['var2'] = labels
np.savetxt(dataset_file, data, fmt="%10s %d")
output = mode + 'dataset.h5'
build_hdf5_image_dataset(dataset_file,
image_shape=(50, 50, 1),
mode='file',
output_path=output,
categorical_labels=True,
normalize=True,
grayscale=True)
# Load HDF5 dataset
h5f = h5py.File('../data/' + mode + 'dataset.h5', 'r')
X_images = h5f['X']
Y_labels = h5f['Y'][:]
print(X_images.shape)
X_images = X_images[:, :, :].reshape([-1, 50, 50, 1])
print(X_images.shape)
h5f.close()
h5f = h5py.File('../data/' + mode + '.h5', 'w')
def data_prep_(conf, method = 'seq', training_size = 0.5, clean_start = False, hdfs=True):
'''
create hdf5 files based on train and test selecting strategy
and return results as train and test sets
:param conf: parameters from congigure file
:param method: random / sequence; at sequence get first training_size of images as training
:param training_size: portion of images that will be used in trainig
:param start_clean: start from zero or work based on previous work
:return: X_train, Y_train, X_test, Y_test
'''
data_folder = conf['data_folder']
if clean_start:
os.remove(data_folder + 'train.h5')
os.remove(data_folder + 'test.h5')
#in the case files is available just read it and return
if os.path.isfile(data_folder + 'train.h5') and os.path.isfile(data_folder + 'validation.h5'):
_h5f = h5py.File(data_folder + 'train.h5', 'r')
X_train = _h5f['X']
Y_train = _h5f['Y']
h5f_ = h5py.File(data_folder + 'validation.h5', 'r')
X_test = h5f_['X']
Y_test = h5f_['Y']
return X_train, Y_train, X_test, Y_test
# if the file is not available, continue creating it
test_counter = 0
counter = 0
train_file = data_folder + 'train.txt'
test_file = data_folder + 'test.txt'
with open(data_folder + 'labeling.csv') as inputFile:
df = pd.read_csv(data_folder + conf['data_frame'])
with open(train_file, 'w+') as trainFile:
with open(test_file, 'w+') as testFile:
for f in os.listdir(data_folder + conf['cropped_folder']):
is_train = False
is_test = False
if method == 'random':
if random.random() < training_size:
is_train = True
else:
is_test = True
else:
if counter < len(df) * training_size:
is_train = True
else:
if random.random() < 0.25:
is_train = True
is_test = True
label = int(df.loc[int(f.split('.')[0].split('_')[0]) - 1, 'manual_label'])
line = data_folder + conf['cropped_folder'] + f + ' ' + str(label) + '\n'
if is_train:
trainFile.write(line)
if is_test and not '_' in f:
testFile.write(line)
if hdfs:
# Build a HDF5 dataset (only required once)
build_hdf5_image_dataset(train_file, image_shape=(40, 40), mode='file', output_path=data_folder + 'train.h5',
categorical_labels=True, normalize=True)
build_hdf5_image_dataset(test_file, image_shape=(40, 40), mode='file', output_path=data_folder + 'test.h5',
categorical_labels=True, normalize=True)
# Load HDF5 dataset
_h5f = h5py.File(data_folder + 'train.h5', 'r')
X_train = _h5f['X']
Y_train = _h5f['Y']
h5f_ = h5py.File(data_folder + 'test.h5', 'r')
X_test = h5f_['X']
Y_test = h5f_['Y']
else:
X_train, Y_train = image_preloader(data_folder +'train.txt', image_shape=(40, 40), mode='file',
categorical_labels=True, normalize=True)
X_test, Y_test = image_preloader(data_folder +'test.txt', image_shape=(40, 40), mode='file',
categorical_labels=True, normalize=True)
return X_train, Y_train, X_test, Y_test
from tflearn.data_utils import shuffle
from tflearn.layers.core import input_data, fully_connected
from tflearn.layers.conv import conv_2d, max_pool_2d
from tflearn.layers.estimator import regression
from tflearn.data_preprocessing import ImagePreprocessing
from tflearn.data_augmentation import ImageAugmentation
from tflearn.layers.normalization import local_response_normalization
import h5py
from tflearn.data_utils import build_hdf5_image_dataset
dataset_file = 'train_data.txt'
validation_file = 'cval_data.txt'
build_hdf5_image_dataset(dataset_file,
image_shape=[144, 144],
mode='file',
output_path='dataset.h5',
categorical_labels=True)
h5f = h5py.File('dataset.h5', 'r')
X = h5f['X']
Y = h5f['Y']
build_hdf5_image_dataset(validation_file,
image_shape=[144, 144],
mode='file',
output_path='validation.h5',
categorical_labels=True)
h5f = h5py.File('validation.h5', 'r')
X_val = h5f['X']
Y_val = h5f['Y']
def get_data(data_dir, hdf5):
"""This function loads in the data, either by loading images on the fly or by creating and
loading from a hdf5 database.
Args:
data_dir: Root directory of the dataset.
hdf5: Boolean. If true, (create and) load data from a hdf5 database.
Returns:
X: training images.
Y: training labels.
X_test: validation images.
Y_test: validation labels."""
# Get the filenames of the lists containing image paths and labels.
train_file, val_file = build_dataset_index(data_dir)
# Check if (creating and) loading from hdf5 database is desired.
if hdf5:
# Create folder to store dataset.
if not os.path.exists('hdf5'):
os.makedirs('hdf5')
# Check if hdf5 databases already exist and create them if not.
if not os.path.exists('hdf5/tiny-imagenet_train.h5'):
from tflearn.data_utils import build_hdf5_image_dataset
print ' Creating hdf5 train dataset.'
build_hdf5_image_dataset(train_file,
image_shape=(64, 64),
mode='file',
output_path='hdf5/tiny-imagenet_train.h5',
categorical_labels=True,
normalize=True)
if not os.path.exists('hdf5/tiny-imagenet_val.h5'):
from tflearn.data_utils import build_hdf5_image_dataset
print ' Creating hdf5 val dataset.'
build_hdf5_image_dataset(val_file,
image_shape=(64, 64),
mode='file',
output_path='hdf5/tiny-imagenet_val.h5',
categorical_labels=True,
normalize=True)
# Load training data from hdf5 dataset.
h5f = h5py.File('hdf5/tiny-imagenet_train.h5', 'r')
X = h5f['X']
Y = h5f['Y']
# Load validation data.
h5f = h5py.File('hdf5/tiny-imagenet_val.h5', 'r')
X_test = h5f['X']
Y_test = h5f['Y']
# Load images directly from disk when they are required.
else:
from tflearn.data_utils import image_preloader
X, Y = image_preloader(train_file,
image_shape=(64, 64),
mode='file',
categorical_labels=True,
normalize=True,
filter_channel=True)
X_test, Y_test = image_preloader(val_file,
image_shape=(64, 64),
mode='file',
categorical_labels=True,
normalize=True,
filter_channel=True)
# Randomly shuffle the dataset.
X, Y = shuffle(X, Y)
return X, Y, X_test, Y_test
from tflearn.data_utils import build_hdf5_image_dataset
import h5py
path = '/home/smie/zhengjx/Res_Bilinear_cnns/raw_data/txt/'
filenum = 50;
filename = 'train_data'
files = [];
result = [];
for i in range(0, filenum):
files.append(path + filename + str(i) + '.txt');
result.append(filename + str(i) + '.h5')
build_hdf5_image_dataset(files[i], image_shape=(488, 488), mode='file', output_path=result[i], categorical_labels=True, normalize=False)
print('Finish dataset ' + result[i]);
import tflearn
import tensorflow as tf
tf.reset_default_graph()
from tflearn.data_utils import load_csv
csv.field_size_limit(sys.maxsize)
# Load path/class_id image file:
#dataset_file ="your file path"
# Build a HDF5 dataset (only required once)
from tflearn.data_utils import build_hdf5_image_dataset
build_hdf5_image_dataset(
dataset_file,
image_shape=(250, 250),
mode='file',
output_path='/Users/coco/Desktop/CG_data/dataset_test.h5',
categorical_labels=False,
normalize=True)
# Load HDF5 dataset
import h5py
h5f = h5py.File('your file path.h5', 'r')
X = h5f['X']
Y = h5f['Y']
network = tflearn.input_data(shape=[None, 250, 250, 3])
network = tflearn.conv_2d(network,
64,
10,
4,
import tflearn
from tflearn.data_utils import shuffle
from tflearn.layers.core import input_data, fully_connected
from tflearn.layers.conv import conv_2d, max_pool_2d
from tflearn.layers.estimator import regression
from tflearn.data_preprocessing import ImagePreprocessing
from tflearn.data_augmentation import ImageAugmentation
from tflearn.layers.normalization import local_response_normalization
import h5py
from tflearn.data_utils import build_hdf5_image_dataset
dataset_file = 'train_data.txt'
validation_file = 'cval_data.txt'
build_hdf5_image_dataset(dataset_file, image_shape=[144, 144], mode='file', output_path='dataset.h5', categorical_labels=True)
h5f = h5py.File('dataset.h5', 'r')
X = h5f['X']
Y = h5f['Y']
build_hdf5_image_dataset(validation_file, image_shape=[144, 144], mode='file', output_path='validation.h5', categorical_labels=True)
h5f = h5py.File('validation.h5', 'r')
X_val = h5f['X']
Y_val = h5f['Y']
X, Y = shuffle(X, Y)
# Make sure the data is normalized
img_prep = ImagePreprocessing()
img_prep.add_featurewise_zero_center()
# # image_shape option can be set to different values to create images of different sizes
# build_hdf5_image_dataset(new_val, image_shape=(224, 224), mode='file', output_path='new_val_224.h5', categorical_labels=True, normalize=False)
# print ('Done creating new_val.h5')
# build_hdf5_image_dataset(new_test, image_shape=(224, 224), mode='file', output_path='new_test_224.h5', categorical_labels=True, normalize=False)
# print ('Done creating new_test.h5')
# build_hdf5_image_dataset(new_train, image_shape=(488, 488), mode='file', output_path='new_train_488.h5', categorical_labels=True, normalize=False)
# print ('Done creating new_train_488.h5')
from tflearn.data_utils import build_hdf5_image_dataset
trainfile = "../data/train_data_crop/"
validfile = "../data/valid_data_crop/"
totaltrain = "../data/total_train_crop/"
# build_hdf5_image_dataset(trainfile, image_shape=(224, 224),
# mode='folder', output_path='../data/train.h5',
# categorical_labels=True, normalize=False, files_extension=['.jpg'])
# print('Done creating train.h5')
# build_hdf5_image_dataset(validfile, image_shape=(224, 224),
# mode='folder', output_path='../data/valid.h5',
# categorical_labels=True, normalize=False, files_extension=['.jpg'])
# print('Done creating valid.h5')
build_hdf5_image_dataset(totaltrain,
image_shape=(224, 224),
mode='folder',
output_path='../data/totaltrain_crop.h5',
categorical_labels=True,
normalize=False,
files_extension=['.jpg'])
print('Done creating totaltrain.h5')
from tflearn.data_utils import image_preloader
from tflearn.data_utils import build_hdf5_image_dataset
# Cross subject dataset
dataset_train = 'D:\PythonProj\DataSet\Hand4/train'
dataset_test = 'D:\PythonProj\DataSet\Hand4/test'
# build_hdf5_image_dataset(dataset_train, image_shape=(32, 32), mode='folder', output_path='hand_train_32.h5', categorical_labels=True, normalize=True)
# build_hdf5_image_dataset(dataset_test, image_shape=(32, 32), mode='folder', output_path='hand_test_32.h5', categorical_labels=True, normalize=True)
# testing
# X, Y = image_preloader(dataset_file, image_shape=(32, 32), mode='folder', categorical_labels=True, normalize=True)
# # Load HDF5 dataset
# h5f = h5py.File('hand_train.h5', 'r')
# CrossValidation dataset
dataset_train = 'D:\PythonProj\DataSet\Hand4\CrossValidation/4/train'
dataset_test = 'D:\PythonProj\DataSet\Hand4\CrossValidation/4/test'
build_hdf5_image_dataset(dataset_train,
image_shape=(32, 32),
mode='folder',
output_path='hand_train_32_S4.h5',
categorical_labels=True,
normalize=True)
build_hdf5_image_dataset(dataset_test,
image_shape=(32, 32),
mode='folder',
output_path='hand_test_32_S4.h5',
categorical_labels=True,
normalize=True)
from __future__ import division, print_function, absolute_import
import tflearn
from tflearn.layers.core import input_data, dropout, fully_connected
from tflearn.layers.conv import conv_2d, max_pool_2d
from tflearn.layers.normalization import local_response_normalization
from tflearn.layers.estimator import regression
from tflearn.data_utils import build_hdf5_image_dataset
import h5py
dataset_train = 'train'
dataset_evaluate = 'test-boat\primotest.txt'
build_hdf5_image_dataset(dataset_train,
image_shape=(128, 38),
mode='folder',
output_path='dataset.h5',
categorical_labels=True,
normalize=True)
build_hdf5_image_dataset(dataset_evaluate,
image_shape=(128, 38),
mode='file',
output_path='evaluate.h5',
categorical_labels=True,
normalize=True)
h5f_t = h5py.File('dataset.h5', 'r')
X = h5f_t['X']
Y = h5f_t['Y']
h5f_e = h5py.File('evaluate.h5', 'r')
Xe = h5f_e['X']
Ye = h5f_e['Y']
from tflearn.data_utils import build_hdf5_image_dataset
import h5py
trainset = "train"
testset = "test"
build_hdf5_image_dataset(testset,
image_shape=(224, 224),
mode='folder',
output_path='new_test.h5',
categorical_labels=True,
normalize=False)
build_hdf5_image_dataset(testset,
image_shape=(224, 224),
mode='folder',
output_path='new_val.h5',
categorical_labels=True,
normalize=False)
print('Done creating new_test.h5')
build_hdf5_image_dataset(trainset,
image_shape=(224, 224),
mode='folder',
output_path='new_train.h5',
categorical_labels=True,
normalize=False)
print('Done creating new_train.h5')
