def __init__(self,
dataset,
img_ids,
p_transform,
data_prep_fun,
label_prep_fun,
rng,
random,
duplicate_label=True,
labels=app.get_labels_array(),
pretext='',
**kwargs):
self.dataset = dataset
self.img_ids = img_ids
self.nsamples = len(img_ids)
self.batch_size = p_transform["num_aug"]
self.p_transform = p_transform
self.data_prep_fun = data_prep_fun
self.label_prep_fun = label_prep_fun
self.rng = rng
self.random = random
self.labels = labels
self.duplicate_label = duplicate_label
self.pretext = pretext
def __init__(self,
dataset,
batch_size,
img_ids,
p_transform,
data_prep_fun,
label_prep_fun,
rng,
random,
infinite,
full_batch,
override_patch_size=None,
version=1,
**kwargs):
self.dataset = dataset
self.img_ids = img_ids
self.nsamples = len(img_ids)
self.batch_size = batch_size
self.p_transform = p_transform
self.data_prep_fun = data_prep_fun
self.label_prep_fun = label_prep_fun
self.rng = rng
self.random = random
self.infinite = infinite
self.full_batch = full_batch
self.override_patch_size = override_patch_size
if override_patch_size:
self.patch_size = override_patch_size
else:
self.patch_size = self.p_transform['vector_size']
self.labels = app.get_labels_array(version=version)
def __init__(self,
dataset,
tta,
img_ids,
p_transform,
data_prep_fun,
label_prep_fun,
rng,
random,
duplicate_label=True,
labels=app.get_labels_array(),
**kwargs):
self.dataset = dataset
self.img_ids = img_ids
self.nsamples = len(img_ids)
self.tta = tta
self.batch_size = tta.n_augmentations
self.p_transform = p_transform
self.data_prep_fun = data_prep_fun
self.label_prep_fun = label_prep_fun
self.rng = rng
self.random = random
self.labels = labels
self.duplicate_label = duplicate_label
def __init__(self, dataset, batch_size, label_id, img_ids, p_transform,
data_prep_fun, label_prep_fun, rng, random, infinite,
full_batch, **kwargs):
self.dataset = dataset
self.img_ids = img_ids
self.nsamples = len(img_ids)
self.batch_size = batch_size
self.p_transform = p_transform
self.data_prep_fun = data_prep_fun
self.label_prep_fun = label_prep_fun
self.rng = rng
self.random = random
self.infinite = infinite
self.full_batch = full_batch
self.labels = app.get_labels_array()
self.label_id = label_id
all_pos_ids, all_neg_ids = app.get_pos_neg_ids(label_id)
pos_ids = []
neg_ids = []
for img_id in img_ids:
if img_id in all_pos_ids:
pos_ids.append(img_id)
elif img_id in all_neg_ids:
neg_ids.append(img_id)
else:
raise
self.pos_ids = pos_ids
self.neg_ids = neg_ids
self.min_samples = min(len(pos_ids), len(neg_ids))
for img_id in pos_ids:
if img_id in neg_ids:
raise
def __init__(self, dataset, batch_size, pos_batch_size, label_id, img_ids,
p_transform, data_prep_fun, label_prep_fun, rng, random,
infinite, full_batch, **kwargs):
self.dataset = dataset
self.img_ids = img_ids
self.nsamples = len(img_ids)
self.batch_size = batch_size
self.p_transform = p_transform
self.data_prep_fun = data_prep_fun
self.label_prep_fun = label_prep_fun
self.rng = rng
self.random = random
self.infinite = infinite
self.full_batch = full_batch
self.labels = app.get_labels_array()
self.label_id = label_id
self.pos_batch_size = pos_batch_size
self.neg_batch_size = batch_size - pos_batch_size
print np.amax(self.img_ids), np.amin(self.img_ids)
selection_labels = self.labels[self.img_ids, :]
self.pos_ids = np.where(selection_labels[:, label_id] > 0)[0]
self.neg_ids = np.where(selection_labels[:, label_id] < 1)[0]
print label_id
print self.pos_ids, self.neg_ids
print len(self.pos_ids), len(self.neg_ids), self.labels.shape[0]
assert ((len(self.pos_ids) + len(self.neg_ids)) == len(img_ids))
def __init__(self, batch_size, img_paths, labeled_img_paths, p_transform,
data_prep_fun, label_prep_fun, rng, random, infinite,
full_batch, **kwargs):
self.img_paths = img_paths
self.labeled_img_paths = labeled_img_paths
self.nsamples = len(img_paths)
self.batch_size = batch_size
self.p_transform = p_transform
self.data_prep_fun = data_prep_fun
self.label_prep_fun = label_prep_fun
self.rng = rng
self.random = random
self.infinite = infinite
self.full_batch = full_batch
self.labels = app.get_labels_array()
def build_joint_feature_vector(config_names, ids, nr_of_augmentations=1):
labels = app.get_labels_array()
targets = dict(zip(ids, labels[ids]))
print(len(ids))
vectors = {}
error_ids = []
for valid_id in ids:
augmentations = []
for aug in range(nr_of_augmentations):
predictions = []
error = False
for config_name in config_names:
try:
file = open(
os.path.join("/data/plnt/model-predictions/fgodin/",
config_name, "features",
str(valid_id) + "_" + str(aug) + ".npy"),
"rb")
predictions.append(np.load(file))
file.close()
except IOError as e:
#print(valid_id)
error = True
error_ids.append(valid_id)
#print(np.concatenate(predictions).shape)
if not error:
augmentations.append(np.concatenate(predictions))
vectors[valid_id] = augmentations
for id in list(set(error_ids)):
print(str(id) + ",")
return vectors, targets
folds = app.make_stratified_split(no_folds=5)
print len(folds)
train_ids = folds[0] + folds[1] + folds[2] + folds[3]
valid_ids = folds[4]
all_ids = folds[0] + folds[1] + folds[2] + folds[3] + folds[4]
bad_ids = []
train_ids = [x for x in train_ids if x not in bad_ids]
valid_ids = [x for x in valid_ids if x not in bad_ids]
test_ids = np.arange(40669)
test2_ids = np.arange(20522)
labels = app.get_labels_array()
print 'np.sum(labels)', np.sum(labels)
print app.read_img_ids_from_lst_file(
'manual_labeling/fn_cultivation_wrongly_labeled.lst')
d_l2i = {
12:
app.read_img_ids_from_lst_file(
'manual_labeling/fn_cultivation_wrongly_labeled.lst')
}
app.remove_tags(labels, d_l2i)
print 'should be lower after removing tags'
print 'np.sum(labels)', np.sum(labels)
train_data_iterator = data_iterators.DataGenerator(
tps = [np.sum(qpreds[:,i]*targs[:,i]) for i in range(17)]
fps = [np.sum(qpreds[:,i]*(1-targs[:,i])) for i in range(17)]
fns = [np.sum((1-qpreds[:,i])*targs[:,i]) for i in range(17)]
print 'TP'
print np.int32(tps)
print 'FP'
print np.int32(fps)
print 'FN'
print np.int32(fns)
print 'worst classes'
print app.get_headers()
print 4*np.array(fps)+np.array(fns)
label_arr = app.get_labels_array()
for i in range(17):
fn = (1-qpreds[:,i])*targs[:,i]
indices = np.int32(np.where(fn==1)[0])
fn_img_ids = [ids[j] for j in indices]
for img_id in fn_img_ids:
print label_arr[img_id,i],
if label_arr[img_id,i] != 1:
print 'Warning ', img_id, 'does not have the correct label'
print
print
print i
print
for iid in fn_img_ids:
print str(iid)+',',
print