def train():
train, valid = load_annotations(0.20)
x_train_set, y_train_set = generate_sets(train)
x_valid_set, y_valid_set = generate_sets(valid)
train_generator = BatchGenerator(x_train_set, y_train_set, 16)
valid_generator = BatchGenerator(x_valid_set, y_valid_set, 16)
print(len(train_generator))
print(len(valid_generator))
#with tf.device('/cpu:0'):
classes= 230
base_model = MobileNet(
include_top=False,
weights="imagenet",
# input_shape=(299, 299, 3),
#pooling='avg',
classes=230,
alpha=0.75
)
x = base_model.output
x = GlobalAveragePooling2D()(x)
shape = (1, 1, int(1024 * 0.75))
x = Reshape(shape, name='reshape_1')(x)
x = Dropout(1e-3, name='dropout')(x)
x = Conv2D(classes, (1, 1), padding='same', name='conv_preds')(x)
x = Activation('softmax', name='act_softmax')(x)
x = GlobalAveragePooling2D()(x)
#pred = Dense(230, activation='softmax')(x)
x = Reshape((classes,), name='reshape_2')(x)
model = Model(inputs=base_model.input, outputs=x)
#model.load_weights("../models/model.h5")
# Replicates the model on 8 GPUs.
# This assumes that your machine has 8 available GPUs.
# parallel_model = multi_gpu_model(model,
# gpus=2,
# cpu_merge=False)
adam = Adam(lr=0.001)
model.compile(loss='categorical_crossentropy',
optimizer=adam,
metrics=['accuracy', f_score])
# This `fit` call will be distributed on 8 GPUs.
# Since the batch size is 256, each GPU will process 32 samples.
model.fit_generator(train_generator,
steps_per_epoch=500,
epochs=100,
verbose=1,
validation_data=valid_generator,
validation_steps=1000,
max_queue_size=100,
workers=4,
use_multiprocessing=True)
model.save("../models/mobilenet.h5", overwrite=True)
def test_load_annotations(filename, zero_based, prefix):
"""test VOC/aergia format annotations"""
data = load_annotations(filename, zero_based)
n_data = len(data)
for i in xrange(n_data):
output_fname = os.path.join(OUTPUT_DIR, "{}_{}.jpg".format(prefix, i))
img, bboxes, attrs_list = data[i]
draw_bbox(img, bboxes, map(lambda x: x[0], attrs_list))
write_successful = cv2.imwrite(output_fname, img)
assert write_successful, \
'fail to write image {} to file'.format(output_fname)
def load_dataset():
annotations, classes = load_annotations()
targets = np.array(annotations)[:, 1]
print(targets)
cls_mapping, cls_list = create_cls_mapping(targets)
annotations = shuffle(annotations, random_state=52)
# train, valid = train_test_split(annotations,
# test_size=0.10,
# random_state=52)
return annotations
def train():
classes = 230
with tf.device('/cpu:0'):
base_model = VGG16(
include_top=False,
weights=None,
input_shape=(192, 192, 3),
classes=classes,
)
x = base_model.output
x = Flatten()(x)
x = Dense(4096, activation='relu', name='fc1')(x)
x = Dense(4096, activation='relu', name='fc2')(x)
x = Dense(230, activation='sigmoid')(x)
model = Model(inputs=base_model.input, outputs=x)
#model.load_weights("../models/vgg16_all_da_epoch_34.h5")
parallel_model = multi_gpu_model(model, gpus=2)
adam = Adam(lr=0.0001)
parallel_model.compile(loss='categorical_crossentropy',
optimizer=adam,
metrics=['accuracy', f_score])
train, _ = load_annotations("train", 0.00)
#valid, _ = load_annotations("validation", 0.00)
x_train_set, y_train_set = generate_sets('train', train)
# x_valid_set, y_valid_set = generate_sets('train', valid)
train_generator = BatchGenerator(x_train_set, y_train_set, 32, (192, 192))
# valid_generator = BatchGenerator(x_valid_set, y_valid_set, 128, (96, 96))
print(len(train_generator))
# print(len(valid_generator))
for i in range(0, 100):
parallel_model.fit_generator(
train_generator,
steps_per_epoch=31705,
# steps_per_epoch=30120,
epochs=1,
verbose=1,
# validation_data=valid_generator,
# validation_steps=397,
# validation_steps=1586,
max_queue_size=200,
workers=10,
use_multiprocessing=True)
model.save_weights("../models/vgg16_all_da_epoch_" + str(i) + ".h5",
overwrite=True)
def train():
classes= 230
with tf.device('/cpu:0'):
base_model = ResNet50(
include_top=False,
weights=None,
input_shape=(224, 224, 3),
classes=classes,
)
x = base_model.output
x = Flatten()(x)
x = Dense(classes, activation='sigmoid', name='predictions')(x)
model = Model(inputs=base_model.input, outputs=x)
# model.load_weights("../models/ResNet50_all_epoch_6.h5")
parallel_model = multi_gpu_model(model, gpus=2)
adam = Adam(lr=0.0001)
parallel_model.compile(loss='categorical_crossentropy',
optimizer=adam,
metrics=['accuracy', f_score])
train, valid = load_annotations("train", 0.00)
# valid, _ = load_annotations("validation", 0.00)
x_train_set, y_train_set = generate_sets('train', train)
# x_valid_set, y_valid_set = generate_sets('train', valid)
train_generator = BatchGenerator(x_train_set, y_train_set, 32, (224,224))
# valid_generator = BatchGenerator(x_valid_set, y_valid_set, 32, (224,224))
print(len(train_generator))
for i in range(1, 20):
parallel_model.fit_generator(train_generator,
steps_per_epoch=31705,
epochs=1,
verbose=1,
# validation_data=valid_generator,
# validation_steps=1586,
max_queue_size=200,
workers=10,
use_multiprocessing=True)
model.save_weights("../models/ResNet50_all_DA_epoch_"+str(6+i)+".h5", overwrite=True)
def train():
# Preprocess
annotations, classes = load_annotations()
targets = np.array(annotations)[:, 1]
print(targets)
cls_mapping, cls_list = create_cls_mapping(targets)
annotations = shuffle(annotations, random_state=52)
train, valid = train_test_split(annotations,
test_size=0.10,
random_state=52)
# Setup train data
y_train = np.array(train)[:, 1]
x_train = np.array(train)[:, 0]
Y_train = convert_to_categorical(y_train, len(classes), cls_mapping)
train_generator = BatchGenerator(x_train, Y_train, 16, (500, 100))
print("length of train train_generator", len(train_generator))
# Setup validation data
X_valid, Y_valid = load_validation_data(annotations, shape, cls_mapping)
# Set up model for training
with tf.device('/cpu:0'):
base_model = ResNet50(
include_top=False,
#weights='imagenet',
input_shape=(100, 500, 3),
classes=len(classes),
)
x = base_model.output
x = Flatten()(x)
x = Dense(len(classes), activation='softmax', name='predictions')(x)
model = Model(inputs=base_model.input, outputs=x)
# model.load_weights("../models/ResNet50_all_epoch_6.h5")
parallel_model = multi_gpu_model(model, gpus=2)
#parallel_model = model
adam = Adam(lr=0.0001, amsgrad=True)
#parallel_model.load_weights("weights/22.h5")
parallel_model.compile(loss="categorical_crossentropy",
optimizer=adam,
metrics=['accuracy', f_macro_score])
print("tart training")
for i in range(1, 200):
print(i)
parallel_model.fit_generator(
train_generator,
steps_per_epoch=2236,
epochs=1,
verbose=1,
validation_data=valid_generator,
# validation_steps=1586,
max_queue_size=200, #
workers=6,
use_multiprocessing=True)
pred = parallel_model.predict_generator(valid_generator)
np.save("pred", pred)
np.save("y_valid", Y_valid)
pred[pred >= 0.5] = 1
pred[pred < 0.5] = 0
print(
classification_report(np.array(Y_valid),
pred,
target_names=cls_list,
digits=3))
model.save_weights("../weights/" + str(i) + ".h5", overwrite=True)