def __init__(self, is_training):
self.reader = Reader(is_training)
self.is_training = is_training
self.learning_rate = cfg.LEARNING_RATE
self.batch_size = cfg.BATCH_SIZE
self.class_num = len(cfg.CLASSES)
self.blocks = cfg.BLOCKS
self.ratios = cfg.RATIOS
self.Sk = cfg.Sk
self.x = [tf.placeholder(tf.float32, [None, None, 3])
] * self.batch_size
self.true_labels = [tf.placeholder(tf.float32, [None])
] * self.batch_size
self.true_boxes = [tf.placeholder(tf.float32, [None, 4])
] * self.batch_size
self.pred_loc, self.pred_cls = self.ssd_net(self.x)
self.saver = tf.train.Saver()
def __init__(self, is_training):
self.is_training = is_training
self.epoches = cfg.EPOCHES
self.learning_rate = cfg.LEARNING_RATE
self.num_classes = len(cfg.CLASSES)
self.model_path = cfg.MODEL_PATH
self.batch_size = cfg.BATCH_SIZE
self.target_size = cfg.TARGET_SIZE
self.keep_rate = cfg.KEEP_RATE
self.reader = Reader(is_training=is_training)
self.x = tf.placeholder(tf.float32,
[None, self.target_size, self.target_size, 3])
self.y = tf.placeholder(tf.int32,
[None, self.target_size, self.target_size])
self.loss_weght = tf.placeholder(
tf.float32, [None, self.target_size, self.target_size])
self.y_hat = self.network(self.x)
self.loss = self.sample_loss(self.y, self.y_hat, self.loss_weght)
self.saver = tf.train.Saver()
def __init__(self, is_training=True):
self.is_training = is_training
if self.is_training:
self.reader = Reader()
self.batch_size = 16
self.lr = 2e-4
self.wd = 5e-3
self.epoches = 100
self.batches = 64
self.size = 96
self.label_num = 30
self.x = tf.placeholder(tf.float32, [None, self.size, self.size, 1])
self.y = tf.placeholder(tf.float32, [None, self.label_num])
self.y_hat = self.network(self.x)
self.model_path = './model'
self.ckpt_path = os.path.join(self.model_path, 'model.ckpt')
self.saver = tf.train.Saver()
def __init__(self, is_training):
self.is_training = is_training
self.epoches = cfg.EPOCHES
self.learning_rate = 0
self.num_classes = len(cfg.CLASSES)
self.model_path = cfg.MODEL_PATH
self.batch_size = cfg.BATCH_SIZE
self.target_size = cfg.TARGET_SIZE
self.keep_rate = cfg.KEEP_RATE
self.reader = Reader(is_training=is_training)
self.x = tf.placeholder(
tf.float32, [None, self.target_size, self.target_size, 3])
self.y_hat = self.network(self.x)
with open('var.txt', 'r') as f:
variables = tf.contrib.framework.get_variables_to_restore()
var_s = f.read().splitlines()
variables_to_restore = [
v for v in variables if v.name in var_s]
self.saver = tf.train.Saver(variables_to_restore)
def concat_data():
"""
To concat raw data to one csv, used for training, validation, and testing.
:return: None
"""
with open(os.path.join(os.path.dirname(__file__), 'config.json'), 'r') as json_file:
config = json.load(json_file)
features = config['features_data']
path = config['directory']
reader = Reader()
for year_used_for, years in config['years'].items():
temp_list = []
for year in years:
temp = reader.read_many(year, features)
temp_list.append(temp)
temp_frame = pd.concat(temp_list)
temp_frame.to_csv((os.path.join(path, '{}.csv').format(year_used_for)), index=False)
def make_dataset():
"""
To make tf.data.Dataset for training and test_x, test_y for testing and evaluation.
:return: train_dataset, val_dataset, test_features, test_labels
"""
reader = Reader()
with open(os.path.join(os.path.dirname(__file__), 'config.json'),
'r') as json_file:
config = json.load(json_file)
features = config['features']
train = reader.read_many('train_resample', features)
val = reader.read_many('val_resample', features)
test = reader.read_many('test_resample', features)
train_mean = train.mean(axis=0)['RPH']
train_std = train.std(axis=0)['RPH']
def normalization(dataframe):
return (dataframe - train.mean(axis=0)) / train.std(axis=0)
def dataframe_to_dataset(dataframe,
shuffle=True,
repeat=True,
batch_size=32):
dataframe = dataframe.copy()
labels = dataframe.pop('RPH')
dataset = tf.data.Dataset.from_tensor_slices((dataframe, labels))
if shuffle:
dataset = dataset.shuffle(buffer_size=len(dataframe))
dataset = dataset.batch(batch_size)
if repeat:
dataset = dataset.repeat()
return dataset
train, val, test = normalization(train), normalization(val), normalization(
test)
train_dataset = dataframe_to_dataset(train)
val_dataset = dataframe_to_dataset(val, shuffle=False)
test_labels = test.pop('RPH')
test_features = test
return train_dataset, val_dataset, test_features, test_labels, train_mean, train_std
def __init__(self, is_training):
self.reader = Reader(is_training)
self.is_training = is_training
self.learning_rate = cfg.LEARNING_RATE
self.class_num = len(cfg.CLASSES)
self.weight_decay = cfg.WEIGHT_DECAY
self.blocks = cfg.BLOCKS
self.ratios = cfg.RATIOS
self.keep_rate = cfg.KEEP_RATE
self.model_path = cfg.MODEL_PATH
self.momentum = cfg.MOMENTUM
self.Sk = cfg.Sk
self.x = tf.placeholder(tf.float32, [None, None, 3])
self.true_labels = tf.placeholder(tf.float32, [None])
self.true_boxes = tf.placeholder(tf.float32, [None, 4])
self.output = self.ssd_net(
tf.expand_dims(self.x, axis=0)
)
self.anchors = tf.numpy_function(
ssd_anchor_all_layers, [self.x],
[tf.float32]*7
)
self.saver = tf.train.Saver()
def __init__(self, is_training=True):
self.size = cfg.TARGET_SIZE
self.data_path = cfg.DATA_PATH
self.model_path = cfg.MODEL_PATH
self.epoches = cfg.EPOCHES
self.batches = cfg.BATCHES
self.lr = cfg.LEARNING_RATE
self.batch_size = cfg.BATCH_SIZE
self.cls_num = cfg.N_CLASSES
self.reader = Reader()
self.keep_rate = cfg.KEEP_RATE
self.is_training = is_training
self.model_name = cfg.MODEL_NAME
self.x = tf.placeholder(tf.float32, [None, self.size, self.size, 3])
self.y = tf.placeholder(tf.float32, [None, self.cls_num])
self.y_hat = self.resnet(self.x)
self.loss = tf.reduce_mean(
tf.nn.softmax_cross_entropy_with_logits_v2(labels=self.y,
logits=self.y_hat))
self.saver = tf.train.Saver()
self.acc = self.calculate_acc(self.y, self.y_hat)
if pos_index.shape[0]:
pos_targets = encode_targets(pos_boxes, pos_anchors)
target_loc[pos_index, :] = pos_targets
return target_labels, target_scores, target_loc
if __name__ == "__main__":
from anchors import ssd_anchor_all_layers
import matplotlib.pyplot as plt
from read_data import Reader
import cv2
reader = Reader(is_training=True)
while (True):
value = reader.generate()
image = value['image'].astype(np.int)
image_copy = image.copy()
true_labels = value['classes']
true_boxes = value['boxes']
image_shape = image.shape
layers_anchors = ssd_anchor_all_layers(image)
target_labels = []
def batch_generator(config):
reader = Reader(config)
while True:
batch_features, batch_labels = reader.batch()
yield batch_features, batch_labels