negative_val_loss = 0.0
negative_val_accuracy = 0.0
for pbi in range(0, val_num, import_batch_size):
print("validation process:%d/%d" % (pbi, val_num))
for pti in range(pbi, min(pbi + import_batch_size, val_num)):
data_index = val_indices[pti]
if data_index < num_positive:
pfile = pfiles[data_index]
isnodule = True
else:
pfile = nfiles[data_index - num_positive]
isnodule = False
data_volume = np.load(pfile)
if isnodule:
if pfile.split('/')[0].find("luna") >= 0:
patient_uid, nodule_diameter = mt.get_annotation_informations(
pfile, annotation)
elif pfile.split('/')[0].find("tianchi") >= 0:
patient_uid, nodule_diameter = mt.get_annotation_informations(
pfile, annotation2)
else:
patient_uid = mt.get_volume_informations(pfile)[0]
nodule_diameter = 0
data_volume = da.extract_volumes(
data_volume,
np.int_([REGION_SIZE, REGION_SIZE, REGION_SIZE]),
nodule_diameter=nodule_diameter,
scale_augment=AUGMENTATION,
translation_augment=AUGMENTATION,
rotation_augment=AUGMENTATION)
else:
data_volume = mt.local_crop(
positive_batch_size = (1 - int(ALL_NEGATIVE)) * (
int(BATCH_SIZE / 2 / aug_proportion) - 1
) + 1 #if augmentatiion implemented then set batch size to 1 or half of BATCH_SIZE
#negative_batch_size = int(positive_batch_size * np_proportion)
for pbi in tqdm(range(0, positive_train_num, positive_batch_size)):
#if AUGMENTATION:
# print("training process:%d/%d" %(pbi, positive_train_num))
posbatchend = min(pbi + positive_batch_size,
positive_train_num)
for pti in range(pbi, posbatchend):
data_index = positive_train_indices[pti]
pfile = tpfiles[data_index]
#pfile = 'luna_cubes_56_overbound/subset5/npy/1.3.6.1.4.1.14519.5.2.1.6279.6001.112740418331256326754121315800_34_ob_annotations.npy'
if pfile.split('/')[0].find("luna") >= 0:
patient_uid, nodule_diameter = mt.get_annotation_informations(
pfile,
"luna_cubes_56_overbound/luna_annotations.csv")
elif pfile.split('/')[0].find("tianchi") >= 0:
patient_uid, nodule_diameter = mt.get_annotation_informations(
pfile,
"tianchi_cubes_56_overbound/tianchi_annotations.csv"
)
else:
patient_uid = mt.get_volume_informations(pfile)[0]
nodule_diameter = 0
positive_data = np.load(pfile)
if "positive_batch" not in dir():
#positive_batch = mt.extract_volumes(positive_data, centering=CENTERING, nodule_diameter=nodule_diameter, scale_augment=SCALE_AUGMENTATION, translation_augment=TRANSLATION_AUGMENTATION, rotation_augment=ROTATION_AUGMENTATION, flip_augment=FLIP_AUGMENTATION)
positive_batch = extract_volumes(
positive_data, nodule_diameter=nodule_diameter)
else:
else:
negative_probabilities[:] = negative_importances
# the number of negative samples is in default larger than the number of positive samples
train_num = 0
train_loss = 0.0
train_accuracy = 0.0
positive_batch_size = int(BATCH_SIZE / 2)
for pbi in range(0, positive_train_num, positive_batch_size):
#print("training process:%d/%d" %(pbi, positive_train_num))
for pti in range(
pbi, min(pbi + positive_batch_size,
positive_train_num)):
data_index = positive_train_indices[pti]
pfile = ptfiles[data_index]
patient_uid, nodule_diameter = mt.get_annotation_informations(
pfile, "LUNA16/csvfiles/annotations.csv")
positive_data = np.load(pfile)
if "positive_batch" not in dir():
positive_batch = mt.extract_volumes(
positive_data, nodule_diameter=nodule_diameter)
else:
positive_batch = np.concatenate(
(positive_batch,
mt.extract_volumes(
positive_data,
nodule_diameter=nodule_diameter)),
axis=0)
negative_batch_size = min(
positive_batch.shape[0],
negative_probabilities.nonzero()[0].size)
if negative_batch_size > 0: