def test_sequential(opt, epochs=200):
"""
This method tests sequential method using MSE loss and takes optimizer as a parameter
"""
# train data
TRAIN_FEATURES = torch.empty(1000, 2).uniform_(0, 1)
# get labels depending if in or outside of the circle
TRAIN_LABELS = h.get_labels(TRAIN_FEATURES, torch.empty(1, 2).fill_(0.5))
# test data
TEST_FEATURES = torch.empty(1000, 2).uniform_(0, 1)
# get labels for the test set
TEST_LABELS = h.get_labels(TEST_FEATURES, torch.empty(1, 2).fill_(0.5))
plt.figure()
h.plot_points(TRAIN_FEATURES, TRAIN_LABELS, "Training Data with Labels")
# create a sequential model with 3 Linear layers, firts 2 with Relu and tha last with
# Tanh activation function
model = Module.Sequential(Module.Linear(2, 25),
Module.ReLU(),
Module.Linear(25, 25),
Module.ReLU(),
Module.Linear(25, 25),
Module.ReLU(),
Module.Linear(25, 2),
Module.Tanh())
# use MSE Loss
loss_fun = Loss.MSE()
# train model fr 200 epochs for the given optimizer with batch size of 10
model.train(TRAIN_FEATURES, TRAIN_LABELS, epochs=epochs, batch_size=10, opt=opt,
loss=loss_fun, verbose=True, accuracy=True,
validation_set=(TEST_FEATURES, TEST_LABELS))
# predict labels on a test set
test_labels = model.predict(TEST_FEATURES)
plt.figure()
h.plot_points(TEST_FEATURES, test_labels, "Test points and predictions")
return model
def predict(self, X, features_names=None):
"""
Run a model in order to predict what's
:param X : base64, input image
:return : list, classes with probabilities
"""
# covert image to base64
input_image = io.BytesIO(base64.b64decode(X))
# Disable scientific notation for clarity
np.set_printoptions(suppress=True)
# Load the model
model = tensorflow.keras.models.load_model(model_path)
# Create the array of the right shape to feed into the keras model
# The 'length' or number of images you can put into the array is
# determined by the first position in the shape tuple, in this case 1.
data = np.ndarray(shape=(1, 224, 224, 3), dtype=np.float32)
# Replace this with the path to your image
image = Image.open(input_image)
# read labels
labels = get_labels(labels_path)
#resize the image to a 224x224 with the same strategy as in TM2:
#resizing the image to be at least 224x224 and then cropping from the center
size = (224, 224)
image = ImageOps.fit(image, size, Image.ANTIALIAS)
#turn the image into a numpy array
image_array = np.asarray(image)
# Normalize the image
normalized_image_array = (image_array.astype(np.float32) / 127.0) - 1
# Load the image into the array
data[0] = normalized_image_array
# run the inference
prediction = model.predict(data).tolist()[0]
# combine output with provided labels,
# and convert prediciton to percentage
output = {}
for x in range(0, len(prediction)):
output[labels[x]] = '{:.2f}%'.format(prediction[x] * 100)
return [output]
def show_image_data_ground_truth(self,
data_df,
image_id,
is_colab,
figsize=(40, 40)):
# Get the an image id given in the training set for visualization
vis_df = data_df[data_df['ImageId'] == image_id]
vis_df = vis_df.reset_index(drop=True)
class_ids = helpers.get_labels(vis_df)
masks = helpers.get_masks(vis_df, target_dim=self.target_dim)
bounding_boxes = helpers.get_bounding_boxes(vis_df, masks)
class_ids, masks, bounding_boxes = helpers.remove_empty_masks(
class_ids, masks, bounding_boxes)
img = Image.open(
common.get_image_path(self.main_folder_path, image_id,
is_colab)).convert("RGB")
img = helpers.rescale(img, target_dim=self.target_dim)
self.show_image_data(img,
class_ids,
masks,
bounding_boxes,
figsize=figsize)
from tensorflow.keras.callbacks import EarlyStopping, ModelCheckpoint
import helpers
import rpn
import faster_rcnn
args = helpers.handle_args()
if args.handle_gpu:
helpers.handle_gpu_compatibility()
batch_size = 1
# If you have trained faster rcnn model you can load weights from faster rcnn model
load_weights_from_frcnn = False
hyper_params = helpers.get_hyper_params(nms_topn=10)
VOC_test_data, VOC_info = helpers.get_dataset("voc/2007", "test")
labels = helpers.get_labels(VOC_info)
# We add 1 class for background
hyper_params["total_labels"] = len(labels) + 1
# If you want to use different dataset and don't know max height and width values
# You can use calculate_max_height_width method in helpers
max_height, max_width = helpers.VOC["max_height"], helpers.VOC["max_width"]
VOC_test_data = VOC_test_data.map(lambda x : helpers.preprocessing(x, max_height, max_width))
padded_shapes, padding_values = helpers.get_padded_batch_params()
VOC_test_data = VOC_test_data.padded_batch(batch_size, padded_shapes=padded_shapes, padding_values=padding_values)
base_model = VGG16(include_top=False)
if hyper_params["stride"] == 16:
base_model = Sequential(base_model.layers[:-1])
rpn_model = rpn.get_model(base_model, hyper_params)
def test_softmax(opt, dropout=False, ReLu=True, nb_neuron=25, epochs=200):
"""
Learns sequential model with softmax layer and cross entropy loss
:param opt: optimizer to be used
:param dropout: whether to use dropouts
:param ReLu: whether to use ReLU (otherwise Tanh)
:param nb_neuron: number of neurons for hidden layer
:return: trained model
"""
# train data
TRAIN_FEATURES = torch.empty(1000, 2).uniform_(0, 1)
# get training labels depending if in or outside of the circle
TRAIN_LABELS = h.get_labels(TRAIN_FEATURES, torch.empty(1, 2).fill_(0.5))
# test data
TEST_FEATURES = torch.empty(1000, 2).uniform_(0, 1)
# get labels for the test set
TEST_LABELS = h.get_labels(TEST_FEATURES, torch.empty(1, 2).fill_(0.5))
plt.figure()
h.plot_points(TRAIN_FEATURES, TRAIN_LABELS, "Training points and labels")
# create a sequential model
model = Module.Sequential()
# add 1st linear layer with ReLu or Tanh activation depending on a ReLU parameter
model.add(Module.Linear(2, nb_neuron))
model.add(get_activation(ReLu))
if dropout:
# add dropouts in case dropout=True with p value of 0.5
model.add(Module.Dropout(p=.5))
# add hidden layer with nb_neuron neurons followed by an activation
model.add(Module.Linear(nb_neuron, nb_neuron))
model.add(get_activation(ReLu))
if dropout:
model.add(Module.Dropout(p=.5))
# add hidden layer with nb_neuron neurons followed by an activation
model.add(Module.Linear(nb_neuron, nb_neuron))
model.add(get_activation(ReLu))
if dropout:
model.add(Module.Dropout(p=.5))
# add final linear layer
model.add(Module.Linear(nb_neuron, 2))
# crossentropy loss combines with the final softmax layer
loss_ce = Loss.CrossEntropy();
# train the model for the given parameters + 200 epochs, 10 batch size
model.train(TRAIN_FEATURES, TRAIN_LABELS, epochs=epochs, batch_size=10, opt=opt,
loss=loss_ce, verbose=True, accuracy=True,
validation_set=(TEST_FEATURES, TEST_LABELS))
# predict outcome of the test data
test_labels = model.predict(TEST_FEATURES)
h.plot_points(TEST_FEATURES, test_labels, "Test points and predictions")
return model
def test_get_labels(self):
target = ['A', 'A', 'B']
labels = helpers.get_labels(target)
assert len(labels) == 2
assert 'A' in labels
assert 'B' in labels
import torch
import numpy as np
from transformers import BertTokenizer
from layoutlm import LayoutlmForTokenClassification
from torch.nn import CrossEntropyLoss
from PIL import Image
#Imports from this repository
from preprocess import preprocess
from helpers import get_labels, set_seed
MODEL_PATH = "model/"
LABEL_LIST_PATH = "data/labels.txt"
set_seed(42)
label_list = get_labels(LABEL_LIST_PATH)
pad_token_label_id = CrossEntropyLoss().ignore_index
def predict(image_file):
"""
Predict token-level classification given the words and bounding boxes
Parameter:
image_file: list
Contains necessary information, more specifically the words, their bounding boxes, receipt image dimension, and the file names
Return:
dict
Contains the extracted information(company, address, date, total) with the corresponding image file name
"""
def preprocess(image_file):
"""
Preprocess the words and bounding boxes inputs into InputFeatures to be put into the model
Parameter:
image_file: list
Contains necessary information, more specifically the words, their bounding boxes, receipt image dimension, and the file names
Return:
list
list of InputFeatures object containing the processed data to be input to the model
"""
actual_bboxes = []
words = []
boxes = []
page_size = []
labels = []
examples = []
file_name = ""
pad_token_label_id = CrossEntropyLoss().ignore_index
for i in image_file:
if (i == ""):
if words:
examples.append(
InputExample(words=words,
boxes=boxes,
actual_bboxes=actual_bboxes,
page_size=page_size,
labels=labels,
file_name=file_name))
words = []
boxes = []
actual_bboxes = []
file_name = None
page_size = []
labels = []
else:
i_split = i.split("\t")
file_name = i_split[3].strip()
actual_bbox = [int(b) for b in i_split[1].split()]
page_size = [int(i) for i in i_split[2].split()]
width, length = page_size
box = [
int(1000 * (actual_bbox[0] / width)),
int(1000 * (actual_bbox[1] / length)),
int(1000 * (actual_bbox[2] / width)),
int(1000 * (actual_bbox[3] / length))
]
actual_bboxes.append(actual_bbox)
boxes.append(box)
words.append(i_split[0])
labels.append("O")
if words:
examples.append(
InputExample(
words=words,
labels=labels,
boxes=boxes,
actual_bboxes=actual_bboxes,
file_name=file_name,
page_size=page_size,
))
label_list = get_labels(LABEL_LIST_PATH)
tokenizer = BertTokenizer.from_pretrained(
MODEL_PATH,
do_lower_case=True,
cache_dir=None,
)
features = convert_examples_to_features(
examples,
label_list,
MAX_SEQ_LENGTH,
tokenizer,
cls_token_at_end=False,
# xlnet has a cls token at the end
cls_token=tokenizer.cls_token,
cls_token_segment_id=0,
sep_token=tokenizer.sep_token,
sep_token_extra=False,
pad_on_left=False,
pad_token=tokenizer.convert_tokens_to_ids([tokenizer.pad_token])[0],
pad_token_segment_id=0,
pad_token_label_id=pad_token_label_id,
)
return features
import sys
import pickle
import os
import helpers
""" Train data and store model to file """
args = sys.argv
train_dataset_path = os.path.abspath(args[1])
print('Dataset path: {}'.format(train_dataset_path))
print('Loading dataset...')
train_target, train_data = helpers.load_dataset(train_dataset_path)
bags_of_words = helpers.create_bags_of_words(train_data)
words = helpers.get_words(bags_of_words)
labels = helpers.get_labels(train_target)
model_path = os.path.join(os.getcwd(), 'model')
if not os.path.exists(model_path):
os.mkdir(model_path)
print('Training data...')
label_probs, probs_per_label = helpers.train(
bags_of_words, words, train_target, labels)
print('Storing model...')
with open(os.path.join(model_path, 'train.pickle'), 'wb') as f:
pickle.dump((label_probs, probs_per_label, words, labels), f)
print('Training done.')
print('============== INFO ===============')
Xn_orig = Xn_orig / np.sqrt((Xn_orig**2).sum(axis=-1, keepdims=True))
# Remove corrupted if exist
drop_idx = np.unique(np.argwhere(np.isnan(Xn_orig))[:, 0])
Xn = np.delete(Xn_orig, drop_idx, axis=0)
df_meta = df_meta.drop(drop_idx).reset_index(drop=True)
# Add missing info
df_meta['date'] = pd.to_datetime(df_meta.date)
df_meta['lat'] = df_meta.apply(lambda rec: geohash.decode(rec['geohash'])[1],
axis=1)
df_meta['lon'] = df_meta.apply(lambda rec: geohash.decode(rec['geohash'])[0],
axis=1)
df_meta = df_meta[['path', 'date', 'geohash', 'lat', 'lon']]
df_label = get_labels(df_meta, df_hoppers, 'hoppers', n_neighbor=1)
y = df_label['hoppers'].values
img_paths = glob(os.path.join(args.climate_data_path, '*.zip'))
img_paths = [get_paths(img) for img in img_paths]
df_climate = pd.DataFrame(img_paths)
df_climate['date'] = pd.to_datetime(df_climate.date)
df_climate['climate_idx'] = df_climate.index
merge_idx = df_label.merge(df_climate,
left_on=['geohash', 'date'],
right_on=['geohash', 'date']).climate_idx
jobs = []
for img_path in img_paths:
job = delayed(zip2numpy_gldas)(img_path)
def __getitem__(self, idx):
if self.gather_statistics:
start = time.time()
image_id = self.image_ids[idx]
vis_df = self.data_df[self.data_df['ImageId'] == image_id]
vis_df = vis_df.reset_index(drop=True)
labels = helpers.get_labels(vis_df)
mask_start_ts = time.time()
try:
masks = helpers.get_masks(vis_df, target_dim=self.target_dim)
for mask in masks:
assert not torch.any(torch.isnan(mask))
assert torch.where(mask > 0)[0].shape[0] == torch.sum(
mask) # check only ones and zeros
except Exception as e:
self.skipped_images.append(image_id)
print(
"ERROR: Skipped image with id [{}] due to a mask exception [{}]"
.format(image_id, e))
return
if self.gather_statistics:
self.inc_by(self.lock, self.total_mask_time,
time.time() - mask_start_ts)
box_start_ts = time.time()
boxes = helpers.get_bounding_boxes(vis_df, masks)
try:
for box in boxes:
assert not torch.any(torch.isnan(box))
except Exception as e:
self.skipped_images.append(image_id)
print(
"ERROR: Skipped image with id [{}] due to a BB exception [{}]".
format(image_id, e))
return
if self.gather_statistics:
self.inc_by(self.lock, self.total_box_time,
time.time() - box_start_ts)
num_objs = len(labels)
image_id_idx = idx
area = (boxes[:, 3] - boxes[:, 1]) * (boxes[:, 2] - boxes[:, 0])
# suppose all instances are not crowd
iscrowd = torch.zeros((num_objs, ), dtype=torch.int64)
labels, masks, boxes = helpers.remove_empty_masks(labels, masks, boxes)
target = {}
if "faster" in self.model_name:
target["labels"] = labels
assert torch.min(target["labels"]) >= 0
assert torch.max(target["labels"]) <= self.num_classes - 1
else:
# we only need the correction for the modified model
target["labels"] = torch.add(
labels,
1) # refer to fast_collate, this is needed for efficient det
assert torch.min(target["labels"]) >= 1
assert torch.max(target["labels"]) <= self.num_classes
target["masks"] = masks
target["boxes"] = boxes
target["image_id"] = image_id_idx
target["area"] = area
target["iscrowd"] = iscrowd
# target["image_id"] = torch.tensor(image_id_idx)
# target["area"] = torch.tensor(area)
# target["iscrowd"] = torch.tensor(iscrowd)
image_load_start_ts = time.time()
image_orig = Image.open(
common.get_image_path(self.main_folder_path, image_id,
self.is_colab)).convert("RGB")
image = helpers.rescale(image_orig, target_dim=self.target_dim)
if self.gather_statistics:
self.inc_by(self.lock, self.total_image_load_time,
time.time() - image_load_start_ts)
# TODO(ofekp): check what happens here when the image is < self.target_dim. What will helpers.py scale method do to the image in this case?
target["img_size"] = image_orig.size[
-2:] if self.target_dim is None else (self.target_dim,
self.target_dim)
image_orig_max_dim = max(target["img_size"])
img_scale = self.target_dim / image_orig_max_dim
target["img_scale"] = 1. / img_scale # back to original size
if self.gather_statistics:
transform_start_ts = time.time()
if self.transforms is not None:
image, target = self.transforms(image, target)
if self.gather_statistics:
self.inc_by(self.lock, self.total_transform_time,
time.time() - transform_start_ts)
self.inc_by(self.lock, self.images_processed, 1)
self.inc_by(self.lock, self.total_process_time,
time.time() - start)
assert image.shape[0] <= self.target_dim and image.shape[
1] <= self.target_dim and image.shape[2] <= self.target_dim
return image, target
