def from_retinanet(annotations_csv):
model = deepforest.deepforest()
model.use_release()
"""use the keras retinanet source to create detections"""
### Keras retinanet
# Format args for CSV generator
classes_file = utilities.create_classes(annotations_csv)
arg_list = utilities.format_args(annotations_csv, classes_file,
model.config)
args = parse_args(arg_list)
# create generator
validation_generator = csv_generator.CSVGenerator(
args.annotations,
args.classes,
image_min_side=args.image_min_side,
image_max_side=args.image_max_side,
config=args.config,
shuffle_groups=False,
)
all_detections = _get_detections(validation_generator,
model.prediction_model,
score_threshold=args.score_threshold,
max_detections=100)
all_annotations = _get_annotations(validation_generator)
return all_detections, all_annotations, validation_generator
def test_lengths(config):
"""Assert that a csv generator and tfrecords create
the same number of images in a epoch"""
created_records = tfrecords.create_tfrecords(
annotations_file="tests/output/testfile_tfrecords.csv",
class_file="tests/output/classes.csv",
image_min_side=config["image-min-side"],
backbone_model=config["backbone"],
size=100,
savedir="tests/output/")
# tfdata
tf_filenames = find_tf_filenames(path="tests/output/*.tfrecord")
# keras generator
backbone = models.backbone(config["backbone"])
generator = csv_generator.CSVGenerator(
csv_data_file="tests/output/testfile_tfrecords.csv",
csv_class_file="tests/output/classes.csv",
image_min_side=config["image-min-side"],
preprocess_image=backbone.preprocess_image,
)
fit_genertor_length = generator.size()
assert len(tf_filenames) == fit_genertor_length
def create_test_generator():
test_generator = csv_generator.CSVGenerator(test_annotations,
classes,
shuffle_groups=False,
auto_augment=None,
rand_augment=None,
config=None)
return test_generator
def create_train_generator(N, M):
train_generator = csv_generator.CSVGenerator(train_annotations,
classes,
transform_generator=None,
visual_effect_generator=None,
image_min_side=800,
image_max_side=1333,
auto_augment=None,
rand_augment=(N, M),
config=None)
return train_generator
def test_equivalence(config, setup_create_tensors):
# unpack created tensors
tf_inputs, tf_targets = setup_create_tensors
# the image going in to tensorflow should be equivalent
# to the image from the fit_generator
backbone = models.backbone(config["backbone"])
# CSV generator
generator = csv_generator.CSVGenerator(
csv_data_file="tests/output/testfile_tfrecords.csv",
csv_class_file="tests/data/classes.csv",
image_min_side=config["image-min-side"],
preprocess_image=backbone.preprocess_image,
)
# find file in randomize generator group
first_file = generator.groups[0][0]
gen_filename = os.path.join(generator.base_dir, generator.image_names[first_file])
original_image = generator.load_image(first_file)
inputs, targets = generator.__getitem__(0)
image = inputs[0, ...]
targets = targets[0][0, ...]
with tf.Session() as sess:
# seek the randomized image to match
tf_inputs, tf_targets = sess.run([tf_inputs, tf_targets])
# assert filename is the same as generator
# assert gen_filename == filename
# tf_image = tf_image[0,...]
tf_inputs = tf_inputs[0, ...]
tf_targets = tf_targets[0][0, ...]
# Same shape
# assert tf_image.shape == image.shape
assert tf_inputs.shape == image.shape
assert tf_targets.shape == targets.shape
def test_equivalence(config, setup_create_tensors):
#unpack created tensors
tf_inputs, tf_targets = setup_create_tensors
#the image going in to tensorflow should be equivalent to the image from the fit_generator
backbone = models.backbone(config["backbone"])
#CSV generator
generator = csv_generator.CSVGenerator(
csv_data_file="tests/data/testfile_tfrecords.csv",
csv_class_file="tests/data/classes.csv",
image_min_side=config["image-min-side"],
preprocess_image=backbone.preprocess_image,
)
#find file in randomize generator group
first_file = generator.groups[0][0]
gen_filename = os.path.join(generator.base_dir, generator.image_names[first_file])
original_image = generator.load_image(first_file)
inputs, targets = generator.__getitem__(0)
image = inputs[0,...]
targets = targets[0][0,...]
with tf.Session() as sess:
#seek the randomized image to match
tf_inputs, tf_targets = sess.run([tf_inputs,tf_targets])
#assert filename is the same as generator
#assert gen_filename == filename
#tf_image = tf_image[0,...]
tf_inputs = tf_inputs[0,...]
tf_targets = tf_targets[0][0,...]
#Same shape
#assert tf_image.shape == image.shape
assert tf_inputs.shape == image.shape
assert tf_targets.shape == targets.shape
#Same values, slightly duplicitious with above, but useful for debugging.
#Saved array is the same as generator image
#np.testing.assert_array_equal(image, tf_image)
#Loaded array is the same as generator, this is not true currently, the opencv and the tensorflow interpolation method is slightly different, waiting for tf. 2.0
#np.testing.assert_array_equal(tf_loaded, tf_image)
##Useful for debug to plot
#fig = plt.figure()
#ax1 = fig.add_subplot(1,4,1)
#ax1.title.set_text('Fit Gen Original')
#plt.imshow(original_image[...,::-1])
#ax1 = fig.add_subplot(1,4,2)
#ax1.title.set_text('Fit Generator')
#plt.imshow(image)
#ax2 = fig.add_subplot(1,4,3)
#ax2 = fig.add_subplot(1,4,4)
#ax2.title.set_text('Loaded Image')
#plt.imshow(tf_inputs)
#plt.show()
#Check for bad file types
#@pytest.fixture()
#def bad_annotations():
#annotations = utilities.xml_to_annotations(get_data("OSBS_029.xml"))
#f = "tests/data/testfile_error_deepforest.csv"
#annotations.to_csv(f,index=False,header=False)
#return f
#def test_tfdataset_error(bad_annotations):
#with pytest.raises(ValueError):
#records_created = tfrecords.create_tfrecords(annotations_file=bad_annotations, class_file=get_data("classes.csv"), image_min_side=800, backbone_model="resnet50", size=100, savedir="tests/data/")
# set the modified tf session as backend in keras
keras.backend.tensorflow_backend.set_session(get_session())
# In[2]:
import pandas as pd
test_csv_file = 'test_hpc_RMmin2_corrected.csv'
class_csv_file = 'classes.csv'
train_ds = pd.read_csv(test_csv_file)
test_generator = csv_generator.CSVGenerator(test_csv_file,
csv_class_file=class_csv_file)
# In[ ]:
model_path = os.path.join('training-files/retinanet_100/models_for_testing/resnet50_csv_99_1.h5')
# load retinanet model
model = models.load_model(model_path, backbone_name='resnet50')
model = models.convert_model(model)
average_precisions,recall,precision,true_positives,false_positives = evaluate(
test_generator,
model,
save_path='./save/test/')