def test_max_results_option(self):
"""Test the max_results option."""
option = ImageClassifierOptions(max_results=_MAX_RESULTS)
classifier = ImageClassifier(_MODEL_FILE, options=option)
categories = classifier.classify(self.image)
self.assertLessEqual(
len(categories), _MAX_RESULTS, 'Too many results returned.')
def test_deny_list(self):
"""Test the label_deny_list option."""
option = ImageClassifierOptions(label_deny_list=_DENY_LIST)
classifier = ImageClassifier(_MODEL_FILE, options=option)
categories = classifier.classify(self.image)
for category in categories:
label = category.label
self.assertNotIn(label, _DENY_LIST,
'Label "{0}" found but in deny list.'.format(label))
def test_identify_image(self):
config = configure.read_config()
config['Algorithm']['semantic_categories_no'] = 10
config['Algorithm']['scene_attributes_no'] = 5
config['Algorithm']['formatting_precision'] = 3
config['GPS']['reverse_location'] = False
classifier = ImageClassifier(config)
res = classifier.identify_image('test/images/27302080E.jpg')
self.maxDiff = None # print full diff on mismatch
self.assertDictEqual(res, classification_27302080E)
def test_allow_list(self):
"""Test the label_allow_list option."""
option = ImageClassifierOptions(label_allow_list=_ALLOW_LIST)
classifier = ImageClassifier(_MODEL_FILE, options=option)
categories = classifier.classify(self.image)
for category in categories:
label = category.label
self.assertIn(
label, _ALLOW_LIST,
'Label "{0}" found but not in label allow list'.format(label))
def test_score_threshold_option(self):
"""Test the score_threshold option."""
option = ImageClassifierOptions(score_threshold=_SCORE_THRESHOLD)
classifier = ImageClassifier(_MODEL_FILE, options=option)
categories = classifier.classify(self.image)
for category in categories:
score = category.score
self.assertGreaterEqual(
score, _SCORE_THRESHOLD,
'Classification with score lower than threshold found. {0}'.format(
category))
def train_classifier_on_hyperparameters(learning_rate_init=learning_rate_init,
gamma=gamma,
seed_parameter=seed_parameter,
track=track):
hyperparameter_dict = {
"lr_init": learning_rate_init,
"lr_gamma": gamma,
"seed_parameter": seed_parameter,
}
wandb_kwargs.update({"config": hyperparameter_dict})
wandb_kwargs["name"] = f"lr {learning_rate_init:.3f}, gamma{gamma:.4f}"
make_data_loader = MakeDataLoader(N_sample=N_sample)
classifier = Classifier(
seed=seed_parameter,
gamma=gamma,
sample_variance_threshold=sample_variance_threshold,
optimizer=optimizer,
optimizer_kwargs=optimizer_kwargs,
learning_rate_init=learning_rate_init,
weight_loss_sample_variance=weight_loss_sample_variance,
evaluation_steps=evaluation_steps,
considered_groups=considered_groups,
N_batches_test=N_batches_test,
).to(device)
if N_gpus > 1 and device.type == "cuda":
classifier = torch.nn.DataParallel(classifier)
if reload:
classifier.load()
classifier.use_label_hierarchy()
# with torch.autograd.detect_anomaly():
if track:
train_classifier_tracked(classifier,
make_data_loader,
epochs=epochs,
save=True,
batch_size=batch_size,
wandb_kwargs=wandb_kwargs,
track=True)
else:
train_classifier(classifier,
make_data_loader,
epochs=epochs,
save=True,
batch_size=batch_size)
def test_augmentation_handling(self):
""" test whether multiple views of input image are recombined correctly"""
classifier = ImageClassifier()
images_test = torch.zeros(2, 3, 64, 64)
images_test[0, 0, 20] = 1
images_test[0, 1, :, 20] = 1
images_augmented = classifier.augment(images_test)
self.assertEqual(images_augmented.shape, (32, 3, 45, 45))
classifier.N_conv_outputs = 1
results = torch.tensor([[i] for img in images_augmented
for i, color in enumerate(img)
if torch.any(color)])
recombined = classifier.recombine_augmentation(results)
results = [torch.all(rec == i) for i, rec in enumerate(recombined)]
self.assertTrue(results[0] and results[1])
def test_default_option(self):
"""Check if the default option works correctly."""
classifier = ImageClassifier(_MODEL_FILE)
categories = classifier.classify(self.image)
# Check if all ground truth classification is found.
for gt_classification in self._ground_truth_classifications:
is_gt_found = False
for real_classification in categories:
is_label_match = real_classification.label == gt_classification.label
is_score_match = abs(real_classification.score -
gt_classification.score) < _ACCEPTABLE_ERROR_RANGE
# If a matching classification is found, stop the loop.
if is_label_match and is_score_match:
is_gt_found = True
break
# If no matching classification found, fail the test.
self.assertTrue(is_gt_found, '{0} not found.'.format(gt_classification))
def _create_ground_truth_csv(self, output_file=_GROUND_TRUTH_FILE):
"""A util function to regenerate the ground truth result.
This function is not used in the test but it exists to make adding more
images and ground truth data to the test easier in the future.
Args:
output_file: Filename to write the ground truth CSV.
"""
classifier = ImageClassifier(_MODEL_FILE)
categories = classifier.classify(self.image)
with open(output_file, 'w') as f:
header = ['label', 'score']
writer = csv.DictWriter(f, fieldnames=header)
writer.writeheader()
for category in categories:
writer.writerow({
'label': category.label,
'score': category.score,
})
def train_autoencoder(epochs: int = 250, batch_size: int = 64, num_workers=4, track=False, plot_images: int = 0):
""" perform training loop for the cVAE """
if track:
wandb.login(key="834835ffb309d5b1618c537d20d23794b271a208")
wandb.init(**wandb_kwargs)
encoder = ConditionalEncoder().cuda()
decoder = ConditionalDecoder().cuda()
classifier = ImageClassifier().cuda()
classifier.load()
classifier.eval()
classifier.use_label_hierarchy()
make_data_loader = MakeDataLoader(augmented=True)
data_loader = make_data_loader.get_data_loader_train(batch_size=batch_size,
shuffle=True,
num_workers=num_workers)
for epoch in trange(epochs, desc="epochs"):
for images, labels in data_loader:
images, labels = images.cuda(), labels.cuda()
images = images*2 - 1 # rescale (0,1) to (-1,1)
latent_mu, latent_sigma = encoder(images, labels)
latent = gaussian_sampler(latent_mu, latent_sigma)
generated_images = decoder(latent, labels)
generated_labels = classifier(generated_images)
decoder.zero_grad()
encoder.zero_grad()
loss_recon_ = loss_reconstruction(images, generated_images)
loss_kl_ = alpha*loss_kl([latent_mu, latent_sigma])
loss_class_ = beta*loss_class(labels, generated_labels)
loss = loss_recon_ + loss_kl_ + loss_class_
loss.backward()
encoder.optimizer.step()
decoder.optimizer.step()
encoder.save()
decoder.save()
if track:
log = {
"loss reconstruction" : loss_recon_.item(),
"loss KL": loss_kl_.item(),
"loss class": loss_class_.item(),
"loss": loss.item()
}
wandb.log(log)
if plot_images and not epoch % plot_images:
width = min(8, len(generated_images))
write_generated_galaxy_images_iteration(iteration=epoch, images=generated_images, width=width, height=len(generated_images)//width, file_prefix="generated_cVAE")
wandb.finish()
def __init__(self, job, log_dir, finished_queue, cv):
self.log_dir = log_dir
self.job = job
self.finished_queue = finished_queue
if self.job.type == 'image_classification':
self.network = ImageClassifier(self.job, self.log_dir,
self.finished_queue, cv)
elif self.job.type == 'object_detection':
self.network = ObjectDetector(self.job, log_dir,
self.finished_queue, cv)
elif self.job.type == 'structured_prediction':
# TODO: Structured data predictor should be built
pass
elif self.job.type == 'structured_classification':
# TODO: Structured classifier should be built
pass
elif self.job.type == 'custom':
# TODO: Custom built nn should be built
pass
else:
raise Exception()
def train_classifier(classifier: Classifier,
make_data_loader,
*,
epochs: int = 5,
batch_size: int = 32,
save: bool = False,
track: bool = False):
schedule = {
# epoch : performed change
1: classifier.use_label_hierarchy,
}
classifier.use_label_hierarchy()
data_loader_train = make_data_loader.get_data_loader_train(
batch_size=batch_size, num_workers=num_workers)
data_loader_valid = make_data_loader.get_data_loader_valid(
batch_size=batch_size, num_workers=num_workers)
for epoch in trange(epochs, desc=f"epochs"):
if classifier.epoch in schedule.keys():
schedule[classifier.epoch]()
classifier.train_epoch(data_loader_train,
data_loader_valid,
track=track)
classifier.plot_losses(save=save)
classifier.plot_accuracy(save=save)
classifier.plot_test_accuracy(save=save)
classifier.plot_sample_variances(save=save)
import torch
from image_classifier import ImageClassifier
from dataset import MakeDataLoader
classifier = ImageClassifier()
classifier.load()
classifier.eval()
classifier.use_label_hierarchy()
make_data_loader = MakeDataLoader()
data_loader = make_data_loader.get_data_loader_valid(batch_size=64,
shuffle=True,
num_workers=4)
for images, labels in data_loader:
predicted_labels = classifier(images)
mse = torch.mean((labels - predicted_labels)**2)
print("rmse", torch.sqrt(mse))
print()
print("L1", torch.mean(torch.abs(labels - predicted_labels)))
break
import matplotlib.pyplot as plt
from image_classifier import ImageClassifier
from dataset import MakeDataLoader
from helpful_functions import write_RGB_image
from file_system import folder_results
## load augmentation procedure
cls = ImageClassifier()
augment = cls.augment
## load single image
make_data_loader = MakeDataLoader()
data_loader = make_data_loader.get_data_loader_train(batch_size=1, )
for image, _ in data_loader:
break
## produce augmentations
augmentations = augment(image)
## plot single image and augmented images
write_RGB_image(image=image[0].numpy(), filename="original_image.png")
for i, a in enumerate(augmentations):
write_RGB_image(image=a.numpy(), filename=f"augmentation{i}.png")
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import base64
import json
from io import BytesIO
from PIL import Image
from image_classifier import ImageClassifier
# create an Image Classifier instance
image_classifier = ImageClassifier()
def lambda_handler(event, context):
if event['isBase64Encoded'] is not True:
return {
'statusCode': 400 # Bad Request!
}
# The event body will have a base64 string containing the image bytes,
# Deocde the base64 string into bytes
image_bytes = base64.b64decode(event['body'])
# Create a Pillow Image from the image bytes
image = Image.open(BytesIO(image_bytes))
# DATA
# ##############################################################################
# Load data and create validation split
data = pickle2obj(pickle_file)
create_valid_split(data, n_valid=10000)
# Printout of shapes
for key in data.keys():
print("{} = {}".format(key, data[key].shape))
img_dims = data["X_train"].shape[1:3]
img_shape = data["X_train"].shape[1:4]
train_gen = data_generator(data["X_train"], data["Y_train"], preprocess_func=preprocess_images, shuffle=True)
valid_gen = data_generator(data["X_valid"], data["Y_valid"], preprocess_func=preprocess_images, shuffle=True)
# ##############################################################################
# MODEL
# ##############################################################################
from image_classifier import ImageClassifier
from architectures import torch, Variable
from architectures import BasicNet
net = BasicNet(img_shape=[28,28,1], n_classes=10)
model = ImageClassifier(net=net, n_classes=10)
model.set_optimizer(lr=1e-3)
model.fit(train_gen=train_gen, valid_gen=valid_gen, n_epochs=2, steps_per_epoch=500)
# TODO: Save and load model
# Save model
def run(model: str, max_results: int, num_threads: int, enable_edgetpu: bool,
camera_id: int, width: int, height: int) -> None:
"""Continuously run inference on images acquired from the camera.
Args:
model: Name of the TFLite image classification model.
max_results: Max of classification results.
num_threads: Number of CPU threads to run the model.
enable_edgetpu: Whether to run the model on EdgeTPU.
camera_id: The camera id to be passed to OpenCV.
width: The width of the frame captured from the camera.
height: The height of the frame captured from the camera.
"""
# Initialize the image classification model
options = ImageClassifierOptions(num_threads=num_threads,
max_results=max_results,
enable_edgetpu=enable_edgetpu)
classifier = ImageClassifier(model, options)
# Variables to calculate FPS
counter, fps = 0, 0
start_time = time.time()
# Start capturing video input from the camera
cap = cv2.VideoCapture(camera_id)
cap.set(cv2.CAP_PROP_FRAME_WIDTH, width)
cap.set(cv2.CAP_PROP_FRAME_HEIGHT, height)
# Continuously capture images from the camera and run inference
while cap.isOpened():
success, image = cap.read()
if not success:
sys.exit(
'ERROR: Unable to read from webcam. Please verify your webcam settings.'
)
counter += 1
image = cv2.flip(image, 1)
# List classification results
categories = classifier.classify(image)
# Show classification results on the image
for idx, category in enumerate(categories):
class_name = category.label
score = round(category.score, 2)
result_text = class_name + ' (' + str(score) + ')'
text_location = (_LEFT_MARGIN, (idx + 2) * _ROW_SIZE)
cv2.putText(image, result_text, text_location,
cv2.FONT_HERSHEY_PLAIN, _FONT_SIZE, _TEXT_COLOR,
_FONT_THICKNESS)
# Calculate the FPS
if counter % _FPS_AVERAGE_FRAME_COUNT == 0:
end_time = time.time()
fps = _FPS_AVERAGE_FRAME_COUNT / (end_time - start_time)
start_time = time.time()
# Show the FPS
fps_text = 'FPS = ' + str(int(fps))
text_location = (_LEFT_MARGIN, _ROW_SIZE)
cv2.putText(image, fps_text, text_location, cv2.FONT_HERSHEY_PLAIN,
_FONT_SIZE, _TEXT_COLOR, _FONT_THICKNESS)
# Stop the program if the ESC key is pressed.
if cv2.waitKey(1) == 27:
break
cv2.imshow('image_classification', image)
cap.release()
cv2.destroyAllWindows()
import json
from image_classifier import ImageClassifier
config = configure.read_config()
parser = argparse.ArgumentParser(description='S3 Connector',
prog='Citizen Sensor')
parser.add_argument('-b',
'--bucket',
help='Bucket name on AWS S3',
required=True)
args = parser.parse_args()
bucket = BucketWrapper(args.bucket).load()
classifier = ImageClassifier(config)
error_file = open('errors.log', 'w')
while bucket.keys:
try:
key = bucket.pop()
keyfile = KeyFile(key)
print('Processing: {}'.format(keyfile.name))
result = classifier.identify_image(keyfile)
with open(result['filename'] + '.json', 'w') as f:
res = json.dump(result,
f,
sort_keys=True,
indent=4,
separators=(',', ': '))
if not os.path.exists(ARGS.tf_rec):
ARG_PARSER.error("TF Records file does not exist!")
# check if metadata file (JSON) exists
JSON_PATH = ARGS.tf_rec.replace(const.TF_REC_EXT, '.json')
if not os.path.exists(JSON_PATH):
ARG_PARSER.error(
"Metadata (JSON) of TF Records file does not exist!")
TF_REC = ARGS.tf_rec
# get NUM_DATA, NUM_CLASSES from metadata
NUM_DATA, NUM_CLASSES = ut.read_metadata(JSON_PATH)
# instantiate model
IMG_MODEL = ImageClassifier(ARGS.arch, ARGS.mode, NUM_CLASSES)
# perform training
if const.TRN_MODE in ARGS.mode:
IMG_MODEL.train(ARGS.alias, TF_REC, NUM_DATA)
# perform classification
elif const.TST_MODE in ARGS.mode:
if ARGS.model_dir is None:
ARGS.model_dir = os.path.join(const.MODELS_FOLDER, ARGS.arch)
elif not os.path.isdir(ARGS.model_dir):
ARG_PARSER.error("Model checkpoint directory does not exist!")
# wildcards should always follow arrangement in ut.save_model
"tags" : ["training", "parameter search"], # tags for organizing tasks
"name" : f"lr_G {lr_generator}, lr_D {lr_discriminator}", # bottom level identifier, label of graph in UI
"config" : hyperparameter_dict, # dictionary of used hyperparameters
}
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
#device = torch.device("cpu")
generator = Generator(latent_dim=latent_dim, labels_dim=labels_dim, G_lr=lr_generator).to(device)
discriminator = Discriminator(labels_dim=labels_dim, D_lr=lr_discriminator).to(device)
if reload:
generator.load()
# discriminator.load()
if conditional:
classifier = ImageClassifier().to(device)
classifier.load() # use pretrained classifier
classifier.eval() # classifier is not trained, but always in evaluation mode
classifier.use_label_hierarchy()
loss_class = torch.nn.MSELoss()
def train_discriminator(images: torch.Tensor,
labels: torch.Tensor,
optimizer_step: bool = True):
latent = generate_latent(labels.shape[0], latent_dim, sigma=False)
labels_fake = generate_labels(labels.shape[0])
if not conditional:
labels_fake[:] = 0
generated_images = generator(latent, labels_fake)
f.write(',')
else:
# raise KeyError('id {} not found'.format(prediction_id))
[f.write(',') for i in self.val_idxs]
f.write(','.join(precision % number for number in array))
f.write('\n')
def write(self, prediction):
self.write_single(self.output_filename_semantic, prediction.id, prediction.semantic_scores, "%.2f")
self.write_single(self.output_filename_scene, prediction.id, prediction.scene_scores, "%.2f")
self.write_single(self.output_filename_fc7, prediction.id, prediction.fc7[(0,9),:].flatten(), "%.4f" )
# self.write_single(self.output_filename_fc7, prediction.id, prediction.fc7[0], "%.4f" )
self.files_processed += 1
if self.rotate:
self.rotate_output_filename()
if __name__ == '__main__':
import configure
from image_classifier import ImageClassifier
config = configure.read_config()
writer = Writer(config, 'test', filename_with_extra_fields='data/evaluation_3k_set.csv', key_idx=1, val_idxs=[0,3,4])
classifier = ImageClassifier(config)
writer.write_headers()
with open('test/images/3221567431_a58ffbd628.jpg', 'rb') as testimage_f:
p = classifier.get_prediction(testimage_f)
writer.write(p)