negative_val_loss = 0.0 negative_val_accuracy = 0.0 for pbi in range(0, val_num, import_batch_size): print("validation process:%d/%d" % (pbi, val_num)) for pti in range(pbi, min(pbi + import_batch_size, val_num)): data_index = val_indices[pti] if data_index < num_positive: pfile = pfiles[data_index] isnodule = True else: pfile = nfiles[data_index - num_positive] isnodule = False data_volume = np.load(pfile) if isnodule: if pfile.split('/')[0].find("luna") >= 0: patient_uid, nodule_diameter = mt.get_annotation_informations( pfile, annotation) elif pfile.split('/')[0].find("tianchi") >= 0: patient_uid, nodule_diameter = mt.get_annotation_informations( pfile, annotation2) else: patient_uid = mt.get_volume_informations(pfile)[0] nodule_diameter = 0 data_volume = da.extract_volumes( data_volume, np.int_([REGION_SIZE, REGION_SIZE, REGION_SIZE]), nodule_diameter=nodule_diameter, scale_augment=AUGMENTATION, translation_augment=AUGMENTATION, rotation_augment=AUGMENTATION) else: data_volume = mt.local_crop(
positive_batch_size = (1 - int(ALL_NEGATIVE)) * ( int(BATCH_SIZE / 2 / aug_proportion) - 1 ) + 1 #if augmentatiion implemented then set batch size to 1 or half of BATCH_SIZE #negative_batch_size = int(positive_batch_size * np_proportion) for pbi in tqdm(range(0, positive_train_num, positive_batch_size)): #if AUGMENTATION: # print("training process:%d/%d" %(pbi, positive_train_num)) posbatchend = min(pbi + positive_batch_size, positive_train_num) for pti in range(pbi, posbatchend): data_index = positive_train_indices[pti] pfile = tpfiles[data_index] #pfile = 'luna_cubes_56_overbound/subset5/npy/1.3.6.1.4.1.14519.5.2.1.6279.6001.112740418331256326754121315800_34_ob_annotations.npy' if pfile.split('/')[0].find("luna") >= 0: patient_uid, nodule_diameter = mt.get_annotation_informations( pfile, "luna_cubes_56_overbound/luna_annotations.csv") elif pfile.split('/')[0].find("tianchi") >= 0: patient_uid, nodule_diameter = mt.get_annotation_informations( pfile, "tianchi_cubes_56_overbound/tianchi_annotations.csv" ) else: patient_uid = mt.get_volume_informations(pfile)[0] nodule_diameter = 0 positive_data = np.load(pfile) if "positive_batch" not in dir(): #positive_batch = mt.extract_volumes(positive_data, centering=CENTERING, nodule_diameter=nodule_diameter, scale_augment=SCALE_AUGMENTATION, translation_augment=TRANSLATION_AUGMENTATION, rotation_augment=ROTATION_AUGMENTATION, flip_augment=FLIP_AUGMENTATION) positive_batch = extract_volumes( positive_data, nodule_diameter=nodule_diameter) else:
else: negative_probabilities[:] = negative_importances # the number of negative samples is in default larger than the number of positive samples train_num = 0 train_loss = 0.0 train_accuracy = 0.0 positive_batch_size = int(BATCH_SIZE / 2) for pbi in range(0, positive_train_num, positive_batch_size): #print("training process:%d/%d" %(pbi, positive_train_num)) for pti in range( pbi, min(pbi + positive_batch_size, positive_train_num)): data_index = positive_train_indices[pti] pfile = ptfiles[data_index] patient_uid, nodule_diameter = mt.get_annotation_informations( pfile, "LUNA16/csvfiles/annotations.csv") positive_data = np.load(pfile) if "positive_batch" not in dir(): positive_batch = mt.extract_volumes( positive_data, nodule_diameter=nodule_diameter) else: positive_batch = np.concatenate( (positive_batch, mt.extract_volumes( positive_data, nodule_diameter=nodule_diameter)), axis=0) negative_batch_size = min( positive_batch.shape[0], negative_probabilities.nonzero()[0].size) if negative_batch_size > 0: