model.compile(loss='binary_crossentropy',
optimizer='nadam',
metrics=['acc'])
hist = model.fit([Q1_train, Q2_train],
train['label'].values[tr],
validation_data=([Q1_test,
Q2_test], train['label'].values[va]),
epochs=50,
batch_size=1024,
shuffle=True,
callbacks=[
EarlyStopping(monitor='val_loss',
min_delta=0.0001,
patience=5,
mode='min')
])
pred = model.predict([q1_data_te, q2_data_te], batch_size=1024)
avg = [v[0] for v in pred]
re.append(avg)
avg = np.mean(re, axis=0)
preds = []
for p in avg:
if p >= 0.5:
preds.append(1)
else:
preds.append(0)
test['label'] = preds
test[['qid1', 'qid2', 'label']].to_csv('result/03_word_cnn_lstm.csv',
columns=['qid1', 'qid2', 'label'],
index=None)
preds = model.predict(batch_images)
conf = list(np.amax(preds, axis=1))
conf_list.extend(conf)
y_pred = list(np.argmax(preds, axis=1))
y_pred_list.extend(y_pred)
final_preds.extend(y_pred_list)
final_conf.extend(conf_list)
final_ids.extend(tar_ids)
shutil.rmtree("imagesfolder")
print("time", time.time() - tm)
print(len(final_preds))
# --------------------------------------- submission -------------------------------------- #
out = []
for i in range(len(final_preds)):
idx = final_preds[i]
out.append(str(idx_to_landmark[idx]) + " " + str(round(final_conf[i], 10)))
print(out[:5])
outdf = pd.DataFrame({"id": final_ids, "landmarks": out})
print(outdf.head())
outdf.to_csv("submissions.csv", index=False)
# ---------------------------------------- the end ----------------------------------------- #
# compute quantities required for featurewise normalization
# (std, mean, and principal components if ZCA whitening is applied)
train_gen.fit(x_train)
valid_gen.fit(x_valid)
filename = "./cancer_classification.h5"
# Save the model according to the conditions
checkpoint = ModelCheckpoint(filename, monitor='val_acc', verbose=1, save_best_only=True, save_weights_only=False,
mode='auto', period=1)
early = EarlyStopping(monitor='val_acc', min_delta=0, patience=10, verbose=1, mode='auto')
# fits the model on batches with real-time data augmentation:
model_final.fit_generator(train_gen.flow(x_train, y_train, batch_size=32),
epochs=epochs,
validation_data=valid_gen.flow(x_valid, y_valid),
nb_val_samples=nb_validation_samples,
callbacks=[checkpoint, early])
model_final.load_weights(filename)
predictions = []
for feature in x_test:
pred = model_final.predict(feature)
predictions.append(pred)
print(pred)
predictions = np.asarray(predictions)
print(predictions.shape)
print(predictions[0])
save_best_only=True)
csv_logger = CSVLogger("dep_resnet_history.csv", separator=',', append=False)
reduce_lr = ReduceLROnPlateau(monitor='val_acc',
factor=0.5,
patience=4,
verbose=1,
mode='max',
min_lr=0.00001)
tensorboard = TensorBoard(log_dir="logs/scalars/" +
datetime.now().strftime("%Y%m%d-%H%M%S"))
history = model.fit(X_train,
y_train,
validation_data=(X_test, y_test),
epochs=25,
batch_size=1,
callbacks=[checkpoint, csv_logger, reduce_lr, tensorboard])
model = load_model("dep_resnet.hdf5")
y_pred = model.predict(X_test)
predictions, actuals = [], []
for i in range(len(y_pred)):
predictions.append(np.where(y_pred[i] == np.max(y_pred[i]))[0][0])
actuals.append(np.where(y_test[i] == np.max(y_test[i]))[0][0])
acc = str(round(sm.accuracy_score(predictions, actuals) * 100, 3))
kappa = str(round(sm.cohen_kappa_score(predictions, actuals), 3))
print(acc)
print(kappa)
def main():
parser = argparse.ArgumentParser()
parser.add_argument("expt_dir",
type=str,
help="Path containing (train, val, test).json")
parser.add_argument("out_prefix",
type=str,
help="Prefix for writing model and prediction")
parser.add_argument("-e",
"--epochs",
type=int,
default=10,
help="# Epochs")
parser.add_argument("-i",
"--image_size",
type=int,
default=250,
help="Image Size NxN")
parser.add_argument("-m",
"--model",
type=str,
choices=['inception', 'resnet'],
default='resnet',
help="Image Model")
parser.add_argument("-b",
"--batch_size",
type=int,
default=64,
help="Batch size")
parser.add_argument("-d",
"--device",
type=int,
default=0,
help="GPU device")
parser.add_argument("-r",
"--retrain",
default=False,
action='store_true',
help="Retrains all layers (instead of last few layers")
parser.add_argument(
"-a",
"--augment",
type=int,
default=0,
help=
"Augment data s.t. each attribute appears at least these many times")
parser.add_argument("-c",
"--class_weight",
default=False,
action='store_true',
help="Use class weights during training")
parser.add_argument("-v",
"--vocab_size",
default=10000,
type=int,
help="Size of vocabulary")
args = parser.parse_args()
params = vars(args)
os.environ['CUDA_VISIBLE_DEVICES'] = str(params['device'])
# Load Data --------------------------------------------------------------------------------------------------------
train_path = osp.join(SEG_ROOT, 'annotations', params['expt_dir'],
'train2017.json')
val_path = osp.join(SEG_ROOT, 'annotations', params['expt_dir'],
'val2017.json')
train_anno_full = json.load(open(train_path))
train_anno = train_anno_full['annotations']
val_anno_full = json.load(open(val_path))
val_anno = val_anno_full['annotations']
anno_full = dict(train_anno.items() + val_anno.items())
print '# Train images = ', len(train_anno)
print '# Val images = ', len(val_anno)
print '# ALL images = ', len(anno_full)
# Helpers ---------------------------------------------------------------------------------------------------------
image_index = get_image_id_info_index()
attr_id_to_name = load_attributes_shorthand()
image_to_text_index = load_image_id_to_text()
image_ids = sorted(train_anno.keys())
attr_ids_set = set()
for img_id, entry in anno_full.iteritems():
for attr_entry in entry['attributes']:
attr_ids_set.add(attr_entry['attr_id'])
attr_ids = sorted(attr_ids_set)
attr_names = [attr_id_to_name[attr_id] for attr_id in attr_ids]
n_img = len(image_ids)
n_attr = len(attr_ids)
attr_id_to_idx = dict(zip(attr_ids, range(n_attr)))
idx_to_attr_id = {v: k for k, v in attr_id_to_idx.iteritems()}
print '# Images = ', n_img
print '# Attributes = ', n_attr
print 'Attributes: '
print attr_ids
target_image_size = (params['image_size'], params['image_size'])
print 'Loading Train Data'
(x_train, y_train, image_id_train) = anno_to_data(train_anno,
attr_id_to_idx,
target_image_size)
print '\nLoading Val Data'
(x_val, y_val, image_id_val) = anno_to_data(val_anno, attr_id_to_idx,
target_image_size)
# Data Augmentation ------------------------------------------------------------------------------------------------
if params['augment'] > 0:
# Current Stats
counter = Counter()
anno_dct = train_anno
for idx, (image_id, entry) in enumerate(anno_dct.iteritems()):
this_attr_ids = set()
for attr_entry in entry['attributes']:
this_attr_ids.add(attr_entry['attr_id'])
for attr_id in this_attr_ids:
counter[attr_id] += 1
min_count = params[
'augment'] # Each attribute should appear at least these many times
x_train_aug, y_train_aug, image_id_train_aug = [], [], []
# Which attributes do we need to augment for?
attr_ids_aug = filter(lambda x: counter[x] < min_count, attr_ids)
print 'Augmenting data to {} for {} attributes:\n{}'.format(
min_count, len(attr_ids_aug), attr_ids_aug)
for attr_id in attr_ids_aug:
num_remaining = min_count - counter[attr_id]
attr_idx = attr_id_to_idx[attr_id]
all_labels = y_train[:, attr_idx] # Get labels for all images
train_idxs = np.where(
all_labels > 0)[0] # Images which contains this attribute
for i in range(num_remaining):
# Select a random image from x_train
row_idx = np.random.choice(train_idxs)
# Add to augmented set
x_train_aug.append(x_train[row_idx])
y_train_aug.append(y_train[row_idx])
image_id_train_aug.append(image_id_train[row_idx])
x_train_aug = np.asarray(x_train_aug)
y_train_aug = np.asarray(y_train_aug)
print '# Augmented Rows = ', x_train_aug.shape[0]
x_train = np.concatenate((x_train, x_train_aug), axis=0)
y_train = np.concatenate((y_train, y_train_aug), axis=0)
image_id_train += image_id_train_aug
# Class Weights ----------------------------------------------------------------------------------------------------
class_weight = np.ones((n_attr, ))
if params['class_weight']:
n_train_samples, n_classes = y_train.shape
for i in range(n_attr):
class_weight[i] = n_train_samples / (n_classes *
np.sum(y_train[:, i]))
# Text Preprocessing ----------------------------------------------------------------------------------------------
# ---- 1. Tokenize words in each image
tokenizer = nltk.word_tokenize
for anno in [train_anno, val_anno]:
for image_id in anno:
if image_id in image_to_text_index:
this_text = image_to_text_index[image_id].lower()
this_tokens = tokenizer(this_text)
anno[image_id]['tokens'] = this_tokens
else:
anno[image_id]['tokens'] = []
# ---- 2. Set Vocabulary (using only train)
word_counter = Counter()
for image_id, entry in train_anno.iteritems():
for tok in entry['tokens']:
word_counter[tok] += 1
print
print 'Complete Vocab size = ', len(word_counter)
print '# >= 2 occurences = ', len(
filter(lambda x: x >= 2, word_counter.values()))
print '# >= 3 occurences = ', len(
filter(lambda x: x >= 3, word_counter.values()))
print '# >= 4 occurences = ', len(
filter(lambda x: x >= 4, word_counter.values()))
vocab_size = params['vocab_size']
word_to_idx = dict()
ukn_token = 'ukn'
ukn_idx = 0
word_to_idx[ukn_token] = ukn_idx
for idx, (word, word_count) in enumerate(word_counter.most_common()):
if idx >= vocab_size - 1:
break
else:
word_to_idx[word] = idx + 1
print 'Size of current vocab = ', len(word_to_idx)
# ---- 3. Tokens -> vec
# Train
n_train_samples = x_train.shape[0]
x_train_text = np.zeros((n_train_samples, vocab_size))
for row_idx, image_id in enumerate(image_id_train):
for tok in train_anno[image_id]['tokens']:
tok_idx = word_to_idx.get(tok, ukn_idx)
x_train_text[row_idx, tok_idx] = 1
# Val
n_val_samples = x_val.shape[0]
x_val_text = np.zeros((n_val_samples, vocab_size))
for row_idx, image_id in enumerate(image_id_val):
for tok in val_anno[image_id]['tokens']:
tok_idx = word_to_idx.get(tok, ukn_idx)
x_val_text[row_idx, tok_idx] = 1
# Image Preprocessing ---------------------------------------------------------------------------------------------
seed = 42
img_datagen = ImageDataGenerator(featurewise_center=True,
featurewise_std_normalization=True,
rotation_range=20,
width_shift_range=0.2,
height_shift_range=0.2,
horizontal_flip=True)
img_datagen.fit(x_train, seed=seed)
# Model ------------------------------------------------------------------------------------------------------------
if params['model'] == 'inception':
base_model = InceptionV3(weights='imagenet', include_top=False)
elif params['model'] == 'resnet':
base_model = ResNet50(weights='imagenet', include_top=False)
else:
raise ValueError('Unrecognized model')
# add a global spatial average pooling layer
x = base_model.output
x = GlobalAveragePooling2D()(x)
# let's add a fully-connected layer
x = Dense(1024, activation='relu')(x)
predictions = Dense(n_attr, activation='sigmoid')(x)
# this is the model we will train
model = Model(inputs=base_model.input, outputs=predictions)
# first: train only the top layers (which were randomly initialized)
# i.e. freeze all convolutional InceptionV3 layers
if not params['retrain']:
for layer in base_model.layers:
layer.trainable = False
# compile the model (should be done *after* setting layers to non-trainable)
model.compile(optimizer='rmsprop', loss='binary_crossentropy')
# train the model on the new data for a few epochs
epochs = params['epochs']
batch_size = params['batch_size']
print 'Training with #Epochs = {}, Batch size = {}'.format(
epochs, batch_size)
model.fit_generator(img_datagen.flow(x_train,
y_train,
batch_size=batch_size),
steps_per_epoch=len(x_train) / batch_size,
epochs=epochs,
class_weight=class_weight)
model_out_path = params['out_prefix'] + '.h5'
model.save(model_out_path)
# Evaluate ---------------------------------------------------------------------------------------------------------
print
print 'Evaluating Model...'
batch_size = params['batch_size']
n_val_rows = x_val.shape[0]
preds = model.predict_generator(img_datagen.flow(x_val,
batch_size=batch_size,
shuffle=False),
steps=(n_val_rows / batch_size) + 1,
verbose=1)
# preds = model.predict(x_val, verbose=1)
ap_scores = average_precision_score(y_val, preds, average=None)
attr_id_score = zip(attr_ids, ap_scores)
for attr_id, attr_score in sorted(attr_id_score, key=lambda x: -x[1]):
print '{:>20s}: {:.3f}'.format(attr_id_to_name[attr_id], attr_score)
print 'C-MAP = ', np.mean(ap_scores)
# Write Predictions ------------------------------------------------------------------------------------------------
out_path = params['out_prefix'] + '.json'
predictions = []
n_val_rows = preds.shape[0]
thresh = 0.1
for row_idx in range(n_val_rows):
this_region_probs = preds[row_idx]
pred_idxs = np.where(this_region_probs >= thresh)[0]
image_id = image_id_val[row_idx]
h, w = anno_full[image_id]['image_height'], anno_full[image_id][
'image_width']
bimask = np.ones((h, w), order='F', dtype='uint8')
rle = mask_utils.encode(bimask)
del bimask
for pred_idx in pred_idxs:
attr_id = idx_to_attr_id[pred_idx]
predictions.append({
'image_id':
image_id_val[row_idx],
'attr_id':
attr_id,
'segmentation':
rle,
'score':
this_region_probs[pred_idx].astype(float),
})
print 'Writing {} predictions'.format(len(predictions))
json.dump(predictions, open(out_path, 'w'), indent=2)
# exit(0)
# model = load_model('test.h5')
path = '/Users/abdulrehman/Desktop/Project/data'
classes=['male','female']
x=[]
y=[]
for fol in classes:
imgfiles=[file for file in os.listdir(path+'/'+fol) if file.endswith(".jpg")]
for img in imgfiles:
im=Image.open(path+'/'+fol+'/'+img)
im=np.asarray(im, dtype="float32")
x.append(im)
y.append(fol)
x=np.array(x)
y=np.array(y)
print(x.shape)
print(y.shape)
batch_size=32
nb_classes=len(classes)
nb_epoch=10
learning_rate = 0.1
decay_rate = learning_rate / nb_epoch
momentum = 0.8
sgd = SGD(lr=learning_rate, momentum=momentum, decay=decay_rate, nesterov=False)
def main():
parser = argparse.ArgumentParser()
parser.add_argument("outfile", type=str, help="Path to write predictions")
parser.add_argument("-t",
"--train_mode",
type=str,
choices=('train', 'trainval'),
default='trainval',
help="train/val or train+val/test")
parser.add_argument("-d",
"--device_id",
type=str,
help="GPU ID",
default='0')
args = parser.parse_args()
params = vars(args)
os.environ["CUDA_VISIBLE_DEVICES"] = params['device_id']
train_mode = params['train_mode']
# --- Load some necessary helpers ----------------------------------------------------------------------------------
image_index = get_image_id_info_index()
attr_id_to_name = load_attributes_shorthand()
image_to_text = load_image_id_to_text()
# --- Load data ----------------------------------------------------------------------------------------------------
if train_mode == 'trainval':
print 'train+val -> test'
train_path = Paths.TRAINVAL_ANNO_PATH
test_path = Paths.TEST_ANNO_PATH
elif train_mode == 'train':
print 'train -> val'
train_path = Paths.TRAIN_ANNO_PATH
test_path = Paths.VAL_ANNO_PATH
else:
raise ValueError('Unrecognized split')
train_anno = json.load(open(train_path))['annotations']
test_anno = json.load(open(test_path))['annotations']
# ------ Use only multimodal attributes
attr_ids = MODE_TO_ATTR_ID['multimodal']
# Include an ignore attribute
SAFE_ATTR = 'a0_safe'
attr_ids.append(SAFE_ATTR)
n_attr = len(attr_ids)
attr_id_to_idx = dict(zip(attr_ids, range(n_attr)))
idx_to_attr_id = {v: k for k, v in attr_id_to_idx.iteritems()}
print '# Attributes = ', n_attr
print 'Attributes: '
print attr_ids
# --- Data -> Tensors ----------------------------------------------------------------------------------------------
print 'Loading Train Data'
(x_trainval, y_trainval, image_id_trainval) = anno_to_data(train_anno,
attr_id_to_idx,
sample_frac=1.0,
mirror=False)
print x_trainval.shape, y_trainval.shape
print '\nLoading Val Data'
(x_test, y_test, image_id_test) = anno_to_data(test_anno, attr_id_to_idx)
print x_test.shape, y_test.shape
# --- Set up Language part -----------------------------------------------------------------------------------------
# ----------- Names
for name_path in [FNAMES_PATH, LNAMES_PATH]:
for _name in open(name_path):
NAMES_SET.add(_name.lower().strip())
print 'Loaded {} names...'.format(len(NAMES_SET))
# ----------- Locations
loc_path = WIKI_LOC_PATH
# For more information: http://download.geonames.org/export/dump/
with open(loc_path) as rf:
for line in rf:
loc = line.strip()
LOCATION_SET.add(loc.lower())
print 'Loaded {} locations'.format(len(LOCATION_SET))
# ----------- Names & Locations
NAME_AND_LOC = NAMES_SET & LOCATION_SET
print 'Name & Loc = ', len(NAME_AND_LOC)
x_text_trainval_tokens = get_token_list(image_id_trainval, image_to_text)
x_text_test_tokens = get_token_list(image_id_test, image_to_text)
# --- Tokenize -----------------------------------------------------------------------------------------------------
VOCAB_SIZE = 1751
word_counter = Counter()
for row in x_text_trainval_tokens:
for token in row:
if len(token.strip()) >= 1:
word_counter[token] += 1
print 'Most common words found: ', word_counter.most_common(n=10)
print 'Vocab size = ', len(word_counter)
print '# >= 2 occurences = ', len(
filter(lambda x: x >= 2, word_counter.values()))
print '# >= 3 occurences = ', len(
filter(lambda x: x >= 3, word_counter.values()))
print '# >= 4 occurences = ', len(
filter(lambda x: x >= 4, word_counter.values()))
print 'Reducing vocab to size = ', VOCAB_SIZE
# Create dict: word -> idx
WORD_TO_IDX[UNKNOWN_TOKEN] = UNKNOWN_IDX
for idx, (word, word_count) in enumerate(word_counter.most_common()):
if idx >= VOCAB_SIZE - 1:
break
else:
WORD_TO_IDX[word] = idx + 1
print 'Vocab look-up dict size = ', len(WORD_TO_IDX)
KNOWN_WORDS = set(WORD_TO_IDX.keys())
x_text_trainval = to_idx_rep(x_text_trainval_tokens, WORD_TO_IDX)
x_text_test = to_idx_rep(x_text_test_tokens, WORD_TO_IDX)
print x_text_trainval.shape
print x_text_test.shape
# --- Extract Image Features ---------------------------------------------------------------------------------------
base_resnet = ResNet50(include_top=False, weights='imagenet')
resnet = Model(inputs=base_resnet.input,
outputs=base_resnet.get_layer('avg_pool').output)
x_trainval_img_feat = img_to_features(x_trainval, resnet)
print
x_test_img_feat = img_to_features(x_test, resnet)
print
print x_trainval_img_feat.shape
print x_test_img_feat.shape
# --- Model --------------------------------------------------------------------------------------------------------
# ------ Define
# -------- Vision Model
n_feat = x_trainval_img_feat.shape[1]
image_input = Input(shape=(n_feat, ))
encoded_image = Dense(1024, activation='relu')(image_input)
# encoded_image = Dropout(0.2)(encoded_image)
# encoded_image = Dense(1024, activation='relu')(encoded_image)
encoded_image = Dropout(0.2)(encoded_image)
# -------- Language Model
text_input = Input(shape=(VOCAB_SIZE, ), )
x2 = Dense(512, activation='relu', input_shape=(VOCAB_SIZE, ))(text_input)
# x2 = Dropout(0.2)(x2)
# x2 = Dense(512, activation='relu')(x2)
encoded_text = Dropout(0.2)(x2)
# -------- Merge Models
merged = keras.layers.concatenate([encoded_image, encoded_text])
merged = Dense(512, activation='relu')(merged)
predictions = Dense(n_attr, activation='sigmoid')(merged)
# this is the model we will train
model = Model(inputs=[image_input, text_input], outputs=predictions)
model.summary()
# compile the model (should be done *after* setting layers to non-trainable)
model.compile(optimizer='adam', loss='binary_crossentropy')
# ------ Train
epochs = 50
batch_size = 128
hist = model.fit([x_trainval_img_feat, x_text_trainval],
y_trainval,
epochs=epochs,
batch_size=batch_size,
verbose=1)
# ------ Predict
preds = model.predict([x_test_img_feat, x_text_test], verbose=1)
ap_scores = average_precision_score(y_test, preds, average=None)
attr_id_score = zip(attr_ids, ap_scores)
print
for attr_id, attr_score in sorted(attr_id_score, key=lambda x: -x[1]):
print '{:>20s}: {:.3f}'.format(attr_id_to_name[attr_id], attr_score)
print 'C-MAP = ', np.mean(ap_scores)
# --- Write Output -------------------------------------------------------------------------------------------------
out_path = params['outfile']
predictions = []
n_test_rows = preds.shape[0]
thresh = 0.0
for row_idx in range(n_test_rows):
this_region_probs = preds[row_idx]
pred_idxs = np.where(this_region_probs >= thresh)[0]
image_id = image_id_test[row_idx]
h, w = test_anno[image_id]['image_height'], test_anno[image_id][
'image_width']
bimask = np.ones((h, w), order='F', dtype='uint8')
rle = mask_utils.encode(bimask)
del bimask
for pred_idx in pred_idxs:
attr_id = idx_to_attr_id[pred_idx]
if attr_id is not SAFE_ATTR:
predictions.append({
'image_id':
image_id_test[row_idx],
'attr_id':
attr_id,
'segmentation':
rle,
'score':
this_region_probs[pred_idx].astype(float),
})
print
print 'Writing {} predictions to {}'.format(len(predictions), out_path)
json.dump(predictions, open(out_path, 'wb'), indent=2)
train = np.copy(l)
count = 0
for z in range(0, 60):
print("Here:", count)
np.random.shuffle(train)
for i in train:
count += 1
model.fit([
np.array(train_descriptions[i])[np.newaxis, :],
np.array(train_categories[i])[np.newaxis, :]
], [np.array([train_labels[i]])[:, np.newaxis]],
epochs=1,
batch_size=1,
shuffle=True)
print("\n\n\n\n\n\n")
output = []
for i in l[160:179]:
count += 1
o = model.predict([
np.array(train_descriptions[i])[np.newaxis, :],
np.array(train_categories[i + 1])[np.newaxis, :]
])
print(o, i)
output.append(o)
pickle.dump(l, open('./pkl/target.p', 'wb'))
pickle.dump(o, open('./pkl/predicted.p', 'wb'))
