def extract_items(self, indices, deepcopy=True, verbose=False):
""" Given a list of indices, it returns a new DataObj that is a subset
of this one, that includes only items at those indices.
By default it creates a deep copy of the data object.
Args:
indices: (list) the indices you want to keep
deepcopy: (bool) creates a deep copy of the data
verbose: (bool)
Returns:
DataObj
"""
verbose_print("Extracting a slice of items", verbose, end="")
# CREATE NEW DATA OBJ WITH EXTRACTED DATA
new_data = DataObj()
new_data.X = self.X[indices]
new_data.Y = self.Y[indices]
new_data.N = self.N[indices]
new_data.BBOX = self.BBOX[indices]
verbose_print_done(verbose)
if deepcopy:
return copy.deepcopy(new_data)
else:
return new_data
def steal_n_samples(self, n=1024, random=False, verbose=False):
""" Removes n samples from the data and returns those removed samples
as a new data object.
You can optionaly select to steal the first n items (default) or
a random sample of n items (random=True).
"""
verbose_print("Stealing {} samples. ".format(n), verbose, end="")
# SELECT WHICH INDICES TO STEAL
if random:
indices = np.random.choice(np.arange(self.n_samples),
size=n,
replace=False)
else:
indices = np.arange(n)
# CREATE NEW DATA OBJ WITH EXTRACTED DATA
new_data = DataObj()
new_data.X = self.X[indices]
new_data.Y = self.Y[indices]
new_data.N = self.N[indices]
new_data.BBOX = self.BBOX[indices]
# REMOVE DATA USED
self.X = np.delete(self.X, indices, axis=0)
self.Y = np.delete(self.Y, indices, axis=0)
self.N = np.delete(self.N, indices, axis=0)
self.BBOX = np.delete(self.BBOX, indices, axis=0)
verbose_print_done(verbose)
return new_data
def create_batch(self, batch_n, augment=False, verbose=False):
""" Given the n'th batch number, it creates a new data object with
that batch of data.
Optionally allows you to do data augmentation on that batch.
"""
verbose_print("Creating a Batch of Data", verbose, end="")
# CREATE NEW DATA OBJ WITH EXTRACTED DATA
i, j = self.batch_indices(batch_n)
batch_data = DataObj(batchsize=self.batchsize)
batch_data.X = self.X[i:j]
batch_data.Y = self.Y[i:j]
batch_data.N = self.N[i:j]
batch_data.BBOX = self.BBOX[i:j]
if augment:
batch_data.do_random_transforms()
# batch_data.X, batch_data.BBOX = batch_augmentation(
# X=batch_data.X,
# bboxes=batch_data.BBOX,
# N=batch_data.N,
# crop=54,
# random_crop=True,
# brightness=0.3,
# contrast=0.3,
# blur=1,
# noise=10,
# seed=None)
verbose_print_done(verbose)
return batch_data
def get_n_samples(self, n=1024, random=False, indices=None, verbose=False):
""" Gets n number of samples and returns those samples
as a new data object.
You can optionaly select to get the first n items (default) or
a random sample of n items (random=True).
"""
verbose_print("Creating a sample of {} items".format(n),
verbose,
end="")
# SELECT WHICH INDICES TO USE
if random:
indices = np.random.choice(np.arange(self.n_samples),
size=n,
replace=False)
else:
indices = np.arange(n)
new_data = self.extract_items(indices=indices,
deepcopy=True,
verbose=False)
verbose_print_done(verbose)
# # CREATE NEW DATA OBJ WITH EXTRACTED DATA
# new_data = DataObj()
# new_data.X = self.X[indices]
# new_data.Y = self.Y[indices]
# new_data.N = self.N[indices]
# new_data.BBOX = self.BBOX[indices]
#
# verbose_print_done(verbose)
# return copy.deepcopy(new_data)
return new_data
def shuffle(self, verbose=False):
verbose_print("Shuffling the Data", verbose, end="")
i_shuffled = np.random.permutation(self.n_samples)
self.X = self.X[i_shuffled]
self.Y = self.Y[i_shuffled]
self.N = self.N[i_shuffled]
self.BBOX = self.BBOX[i_shuffled]
verbose_print_done(verbose)
def set_valid_data(self,
n=1024,
random=True,
batchsize=128,
verbose=False):
verbose_print("Creating Validation Data", verbose, end="")
self.valid = self.train.steal_n_samples(n=n, random=random)
self.valid.do_center_crops()
self.valid.set_batchsize(n=batchsize)
verbose_print_done(verbose)
def set_train_eval_data(self,
n=1024,
random=False,
random_transforms=True,
batchsize=128,
verbose=False):
verbose_print("Creating Train Evaluation Data", verbose, end="")
self.train_eval = self.train.get_n_samples(n=n, random=random)
if random_transforms:
self.train_eval.do_random_transforms()
self.train_eval.set_batchsize(n=batchsize)
verbose_print_done(verbose)
def __init__(self, d=None, pickle=None, verbose=False):
""" Creates an Evals object to store evaluation metrics for each epoch.
Args:
d: (dict or None)(optional) - initialize Evals object from
a dictionary
pickle: (str or None) (optional) path to a pickle file of a
dictionary to initialize the Evals object.
verbose: (bool)
"""
self.stuff = dict()
self._initialized = True
# INITIAL BLANK VALUES
self.pda_train = []
self.pda_valid = []
self.wna_train = []
self.wna_valid = []
self.iou_train = []
self.iou_valid = []
self.time_pred = []
self.time_train = []
self.loss = []
self.alpha = []
# LOAD EVALS FROM DICTIONARY
if d is not None:
verbose_print("Loading evals from a dictionary",
verbose=verbose,
end="")
self.stuff.update(copy.deepcopy(d))
# LOAD EVALS FROM PICKLE FILE (of a dictionary)
elif pickle is not None:
short_path = limit_string(pickle, tail=PRINT_WIDTH - 32)
verbose_print("Loading evals from " + short_path, verbose, end="")
if os.path.exists(pickle):
d = pickle2obj(pickle)
self.stuff.update(copy.deepcopy(d))
else:
verbose_print("\n-- file does not exist. Creating blank Evals",
verbose,
end="")
else:
verbose_print("Creating blank Evals", verbose, end="")
verbose_print_done(verbose)
def do_random_transforms(self, verbose=False):
""" Modifies the X data by applying random transforms on the image data,
such as cropping, shifting, brightness, contrast, blur and noise.
Also updates the bounding box labels to account for these
transformations.
NOTE: the images are rescaled to 54x54
"""
verbose_print("Randomly transform the images", verbose, end="")
self.X, self.BBOX = batch_augmentation(X=self.X,
bboxes=self.BBOX,
N=self.N,
crop=54,
random_crop=True,
brightness=0.3,
contrast=0.3,
blur=1,
noise=10,
seed=None)
verbose_print_done(verbose)
settings = Settings()
settings.conv_dropout = 0.1
settings.fc_dropout = 0.1
settings.image_chanels = 1
settings.image_size = (54, 54)
# ==============================================================================
# DATA
# ==============================================================================
data_dir = "data"
X_file = os.path.join(data_dir, "X_test_cropped64.pickle")
Y_file = os.path.join(data_dir, "Y_test.pickle")
verbose_print("Loading data", verbose=verbose, end="")
data = DataObj(X=pickle2obj(X_file), Y=pickle2obj(Y_file), batchsize=128)
verbose_print_done(verbose)
data.limit_samples(n=limit, verbose=verbose)
verbose_print("Performing center crops", verbose=verbose, end="")
data.do_center_crops()
verbose_print_done(verbose)
# ==============================================================================
# GRAPH AND SESSION
# ==============================================================================
model = model_a
checkpoint_dir = "results/A_02/checkpoints/"
checkpoint_file = os.path.join(checkpoint_dir, "checkpoint_max.chk")
# CREATE TENSORFLOW GRAPH
def save_dict_pickle(self, f, verbose=False):
short_path = limit_string(f, front=10, tail=31)
verbose_print("Saving Evals to " + short_path, verbose, end="")
obj2pickle(self.stuff, file=f)
verbose_print_done(verbose)