def vaudenay():
with open("c17.dat") as f:
line = choice([b64decode(line) for line in f.readlines()])
a = AES.new(key, AES.MODE_CBC, IV)
ciphertxt = a.encrypt(pad(line))
chunks = make_chunks(ciphertxt)
assert chunks[-1] == ciphertxt[-BLOCK_SIZE:]
return "".join([decrypt_chunk(c1, c2) for (c1, c2) in zip(chunks[:-1], chunks[1:]) ])
def download(self):
chunks = utils.make_chunks(os.listdir(self.path), 8)
batch = len(chunks)
current = 1
for chunk in chunks:
print('Batch {}/{} started'.format(current, batch))
pool = []
for show_id in chunk:
thread = DownloadThread('{}{}'.format(self.path, show_id))
thread.start()
pool.append(thread)
print('Show ID {} started'.format(show_id))
for thread in pool:
thread.join()
current += 1
print('Finished')
def writeImage(epoch, step):
unet3D.eval()
print("writing image.............")
datadir = "IXI-T1/Actual_Images"
scans = os.listdir(datadir)
img = nib.load(os.path.join(datadir, scans[-1]))
original = img.get_fdata()
original = original / np.max(original)
reduced = FFT_compression(original)
reduced_chunks = make_chunks(reduced)
predicted = []
for chunk in reduced_chunks:
chunk = torch.from_numpy(np.expand_dims(
chunk, 0).astype("float32")).to(device)
predicted_chunk = unet3D(torch.unsqueeze(chunk, 0))
predicted.append(torch.squeeze(predicted_chunk).detach().cpu().numpy())
pred = unmake_chunks(predicted)
slice_original = (original[:, :, int(original.shape[2] / 2)])
slice_reduced = (reduced[:, :, int(reduced.shape[2] / 2)])
slice_pred = (pred[:, :, int(pred.shape[2] / 2)])
slices_list = [slice_original.T, slice_reduced.T, slice_pred.T]
show(slices_list, epoch, step)
unet3D.train()
stream = "fusion"
filter_type = "avg_fovea"
filename = "thresholds/context_fusion/predictions_" + stream + "_" + filter_type + "_1809281055.pickle"
with open(filename, 'rb') as handle:
predictions = pickle.load(handle)
root_dir = '../../data/AVA/files/'
# Load groundtruth (test or val set)
# Load list of action classes and separate them (from utils_stream)
classes = utils.get_AVA_classes(root_dir + 'ava_action_list_custom.csv')
partition = get_AVA_set(classes=classes,
filename=root_dir + "AVA_Val_Custom_Corrected.csv",
soft_sigmoid=True)
test_splits = utils.make_chunks(original_list=partition,
size=len(partition),
chunk_size=2**11)
# Voting
thresh_values = [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9]
for thresh in thresh_values:
print("Testing threshold " + str(thresh) + " ")
pose_votes = {}
obj_votes = {}
human_votes = {}
for row in partition:
row = row.split("@")
i = row[0] + "@" + row[1] + "@" + str(row[2]) + "@" + str(
row[3]) + "@" + str(row[4]) + "@" + str(row[5])
pose_votes[i] = np.zeros(utils.POSE_CLASSES)
この文は「始める」と「見る」の２つの動詞を含み，「始める」に係る文節は「ここで」，
「見る」に係る文節は「吾輩は」と「ものを」と解析された場合は，次のような出力になるはずである．
始める で
見る は を
このプログラムの出力をファイルに保存し，以下の事項をUNIXコマンドを用いて確認せよ．
コーパス中で頻出する述語と格パターンの組み合わせ
「する」「見る」「与える」という動詞の格パターン（コーパス中で出現頻度の高い順に並べよ）
"""
from utils import make_chunks
from constants import FNAME_PARSED
if __name__ == '__main__':
with open('result_n45.txt', mode='w') as out_file:
for chunks in make_chunks(FNAME_PARSED):
for chunk in chunks:
# 文節内に動詞があるか確認する
verb_info = chunk.get_morphlist_by_pos('動詞')
if len(verb_info) < 1:
continue # 文節内に動詞が存在しない場合は次の文節へ
# 文節内(chunk)に動詞がある状態
particles = []
for src_number in chunk.srcs:
particle_info = chunks[src_number].get_morphlist_by_pos(
'助詞')
if len(particle_info) > 0:
particles.append(particle_info[0].surface)
if len(particles) < 1:
"""
:param PageDownloader srv:
:param show_id:
"""
threading.Thread.__init__(self)
self.service = srv
self.id = show_id
def run(self):
self.service.download(self.id)
if __name__ == '__main__':
service = PageDownloader(utils.requests_retry_session(),
int(input('Event ID:')))
start = int(input('Start ID:'))
end = int(input('End ID:'))
if start > end or start <= 0:
print('Start - End is invalid')
exit(1)
chunks = utils.make_chunks(range(start, end + 1), 10)
for chunk in chunks:
pool = []
for show_id in chunk:
thread = CollectThread(service, show_id)
thread.start()
pool.append(thread)
for thread in pool:
thread.join()
def main():
# root_dir = '../../../AVA2.1/' # root_dir for the files
root_dir = '../../data/AVA/files/'
# Erase previous models from GPU memory
K.clear_session()
# Load list of action classes and separate them
classes = utils.get_AVA_classes(root_dir + 'ava_action_list_custom.csv')
# Parameters for training
params = {
'dim': (224, 224),
'batch_size': 32,
'n_classes': len(classes['label_id']),
'n_channels': 3,
'shuffle': False,
'nb_epochs': 200,
'model': 'inceptionv3',
'email': True,
'freeze_all': True,
'conv_fusion': False,
'train_chunk_size': 2**12,
'validation_chunk_size': 2**12
}
soft_sigmoid = True
minValLoss = 9999990.0
# Get ID's and labels from the actual dataset
partition = {}
partition['train'] = get_AVA_set(
classes=classes,
filename=root_dir + "AVA_Train_Custom_Corrected.csv",
soft_sigmoid=soft_sigmoid) # IDs for training
partition['validation'] = get_AVA_set(
classes=classes,
filename=root_dir + "AVA_Val_Custom_Corrected.csv",
soft_sigmoid=soft_sigmoid) # IDs for validation
# Labels
labels_train = get_AVA_labels(classes,
partition,
"train",
filename=root_dir +
"AVA_Train_Custom_Corrected.csv",
soft_sigmoid=soft_sigmoid)
labels_val = get_AVA_labels(classes,
partition,
"validation",
filename=root_dir +
"AVA_Val_Custom_Corrected.csv",
soft_sigmoid=soft_sigmoid)
# Create + compile model, load saved weights if they exist
saved_weights = None
# saved_weights = "../models/rgbextra_gauss_resnet50_1807250030.hdf5"
model, keras_layer_names = rgb_create_model(
classes=classes['label_id'],
soft_sigmoid=soft_sigmoid,
model_name=params['model'],
freeze_all=params['freeze_all'],
conv_fusion=params['conv_fusion'])
model = compile_model(model, soft_sigmoid=soft_sigmoid)
# TODO Experiment: 1. no initialization, 2. ucf initialization 3. kinetics initialization
initialization = True # Set to True to use initialization
kinetics_weights = None
ucf_weights = "a"
if saved_weights is not None:
model.load_weights(saved_weights)
else:
if initialization is True:
if ucf_weights is None:
print("Loading MConvNet weights: ")
if params['model'] == "resnet50":
ucf_weights = utils.loadmat(
"../models/ucf_matconvnet/ucf101-img-resnet-50-split1.mat"
)
utils.convert_resnet(model, ucf_weights)
model.save(
"../models/ucf_keras/keras-ucf101-rgb-resnet50-newsplit.hdf5"
)
if kinetics_weights is None:
if params['model'] == "inceptionv3":
print("Loading Keras weights: ")
keras_weights = [
"../models/kinetics_keras/tsn_rgb_params_names.pkl",
"../models/kinetics_keras/tsn_rgb_params.pkl"
]
utils.convert_inceptionv3(model, keras_weights,
keras_layer_names)
model.save(
"../models/kinetics_keras/keras-kinetics-rgb-inceptionv3.hdf5"
)
# Try to train on more than 1 GPU if possible
# try:
# print("Trying MULTI-GPU")
# model = multi_gpu_model(model)
print("Training set size: " + str(len(partition['train'])))
# Make spltis
train_splits = utils.make_chunks(original_list=partition['train'],
size=len(partition['train']),
chunk_size=params['train_chunk_size'])
val_splits = utils.make_chunks(original_list=partition['validation'],
size=len(partition['validation']),
chunk_size=params['validation_chunk_size'])
time_str = time.strftime("%y%m%d%H%M", time.localtime())
# TODO Don't forget to change your names :)
filter_type = "gauss"
bestModelPath = "../models/rgb_kininit_" + filter_type + \
"_" + params['model'] + "_" + time_str + ".hdf5"
traincsvPath = "../loss_acc_plots/rgb_kininit_train_" + filter_type + \
"_plot_" + params['model'] + "_" + time_str + ".csv"
valcsvPath = "../loss_acc_plots/rgb_kininit_val_" + filter_type + \
"_plot_" + params['model'] + "_" + time_str + ".csv"
with tf.device('/gpu:0'): # NOTE Not using multi gpu
for epoch in range(params['nb_epochs']):
epoch_chunks_count = 0
for trainIDS in train_splits:
# Load and train
start_time = timeit.default_timer()
# -----------------------------------------------------------
x_val = y_val_pose = y_val_object = y_val_human = x_train = y_train_pose = y_train_object = y_train_human = None
x_train, y_train_pose, y_train_object, y_train_human = load_split(
trainIDS,
labels_train,
params['dim'],
params['n_channels'],
"train",
filter_type,
soft_sigmoid=soft_sigmoid)
y_t = []
y_t.append(
to_categorical(y_train_pose,
num_classes=utils.POSE_CLASSES))
y_t.append(
utils.to_binary_vector(y_train_object,
size=utils.OBJ_HUMAN_CLASSES,
labeltype='object-human'))
y_t.append(
utils.to_binary_vector(y_train_human,
size=utils.HUMAN_HUMAN_CLASSES,
labeltype='human-human'))
history = model.fit(x_train,
y_t,
batch_size=params['batch_size'],
epochs=1,
verbose=0)
utils.learning_rate_schedule(model, epoch, params['nb_epochs'])
# TODO Repeat samples of unrepresented classes?
# ------------------------------------------------------------
elapsed = timeit.default_timer() - start_time
print("Epoch " + str(epoch) + " chunk " +
str(epoch_chunks_count) + " (" + str(elapsed) +
") acc[pose,obj,human] = [" +
str(history.history['pred_pose_categorical_accuracy']) +
"," +
str(history.
history['pred_obj_human_categorical_accuracy']) +
"," +
str(history.
history['pred_human_human_categorical_accuracy']) +
"] loss: " + str(history.history['loss']))
with open(traincsvPath, 'a') as f:
writer = csv.writer(f)
avg_acc = (
history.history['pred_pose_categorical_accuracy'][0] +
history.history['pred_obj_human_categorical_accuracy']
[0] + history.history[
'pred_human_human_categorical_accuracy'][0]) / 3
writer.writerow([
str(avg_acc),
history.history['pred_pose_categorical_accuracy'],
history.history['pred_obj_human_categorical_accuracy'],
history.
history['pred_human_human_categorical_accuracy'],
history.history['loss']
])
epoch_chunks_count += 1
# Load val_data
print("Validating data: ")
# global_loss, pose_loss, object_loss, human_loss, pose_acc, object_acc, human_acc
loss_acc_list = [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]
for valIDS in val_splits:
x_val = y_val_pose = y_val_object = y_val_human = x_train = y_train_pose = y_train_object = y_train_human = None
x_val, y_val_pose, y_val_object, y_val_human = load_split(
valIDS,
labels_val,
params['dim'],
params['n_channels'],
"val",
filter_type,
soft_sigmoid=soft_sigmoid)
y_v = []
y_v.append(
to_categorical(y_val_pose, num_classes=utils.POSE_CLASSES))
y_v.append(
utils.to_binary_vector(y_val_object,
size=utils.OBJ_HUMAN_CLASSES,
labeltype='object-human'))
y_v.append(
utils.to_binary_vector(y_val_human,
size=utils.HUMAN_HUMAN_CLASSES,
labeltype='human-human'))
vglobal_loss, vpose_loss, vobject_loss, vhuman_loss, vpose_acc, vobject_acc, vhuman_acc = model.evaluate(
x_val, y_v, batch_size=params['batch_size'])
loss_acc_list[0] += vglobal_loss
loss_acc_list[1] += vpose_loss
loss_acc_list[2] += vobject_loss
loss_acc_list[3] += vhuman_loss
loss_acc_list[4] += vpose_acc
loss_acc_list[5] += vobject_acc
loss_acc_list[6] += vhuman_acc
# Average over all validation chunks
loss_acc_list = [x / len(val_splits) for x in loss_acc_list]
with open(valcsvPath, 'a') as f:
writer = csv.writer(f)
# We consider accuracy as the average accuracy over the three
# types of accuracy
acc = (loss_acc_list[4] + loss_acc_list[5] +
loss_acc_list[6]) / 3
writer.writerow([
str(acc), loss_acc_list[4], loss_acc_list[5],
loss_acc_list[6], loss_acc_list[0], loss_acc_list[1],
loss_acc_list[2], loss_acc_list[3]
])
if loss_acc_list[0] < minValLoss:
print("New best loss " + str(loss_acc_list[0]))
model.save(bestModelPath)
minValLoss = loss_acc_list[0]
if params['email']:
utils.sendemail(from_addr='*****@*****.**',
to_addr_list=['*****@*****.**'],
subject='Finished training RGB-stream ',
message='Training RGB with following params: ' +
str(params),
login='******',
password='******')
def main():
# root_dir = '../../../AVA2.1/' # root_dir for the files
root_dir = '../../data/AVA/files/'
# Erase previous models from GPU memory
K.clear_session()
# Load list of action classes and separate them
classes = utils.get_AVA_classes(root_dir + 'ava_action_list_custom.csv')
# Parameters for training
params = {
'dim': (224, 224),
'batch_size': 32,
'n_classes': len(classes['label_id']),
'n_channels': 3,
'shuffle': False,
'nb_epochs': 200,
'model': 'resnet50',
'email': True,
'freeze_all': True,
'conv_fusion': False,
'train_chunk_size': 2**12,
'validation_chunk_size': 2**12
}
soft_sigmoid = True
minValLoss = 9999990.0
# Get ID's and labels from the actual dataset
partition = {}
partition['train'] = get_AVA_set(
classes=classes,
filename=root_dir + "AVA_Train_Custom_Corrected.csv",
soft_sigmoid=soft_sigmoid) # IDs for training
partition['validation'] = get_AVA_set(
classes=classes,
filename=root_dir + "AVA_Val_Custom_Corrected.csv",
soft_sigmoid=soft_sigmoid) # IDs for validation
# Labels
labels_train = get_AVA_labels(classes,
partition,
"train",
filename=root_dir +
"AVA_Train_Custom_Corrected.csv",
soft_sigmoid=soft_sigmoid)
labels_val = get_AVA_labels(classes,
partition,
"validation",
filename=root_dir +
"AVA_Val_Custom_Corrected.csv",
soft_sigmoid=soft_sigmoid)
# http://scikit-learn.org/stable/modules/generated/sklearn.utils.class_weight.compute_class_weight.html
# Logistic Regression in Rare Events Data, Gary King, Langche Zeng
# Keras needs a dict though
# From documentation: class_weight: Optional dictionary mapping class indices (integers) to a weight (float) value, used for weighting the loss function (during training only).
# This can be useful to tell the model to "pay more attention" to samples from an under-represented class.
y = labels_to_numpy(labels_train)
penalizing_method = 'balanced'
mu = 0.7
# penalizing_method = 'weighted_log'
class_weights = np.zeros(30)
for i in y:
class_weights[i] += 1
max_class = max(class_weights)
for i in range(len(class_weights)):
if class_weights[i] != 0.0:
if penalizing_method == 'balanced':
print(
str(i) + " " + str(class_weights[i]) + " " +
str(len(y) / (class_weights[i])))
#class_weights[i] = len(y) / (class_weights[i])
class_weights[i] = max_class / (class_weights[i])
elif penalizing_method == 'weighted_log':
print(
str(i) + " " + str(class_weights[i]) + " " +
str(math.log(mu * len(y) / (class_weights[i]))))
class_weights[i] = math.log(mu * len(y) / (class_weights[i]))
else:
print(str(i) + " " + str(class_weights[i]) + " inf ")
class_weights[i] = 0.0
g = sns.barplot(x=[str(i) for i in range(len(class_weights))],
y=class_weights)
plt.xticks(rotation=-90)
plt.title("Class weights " + penalizing_method)
plt.grid(True)
plt.show()
class_dictionary = {}
print(len(class_weights))
for i in range(len(class_weights)):
class_dictionary[i] = class_weights[i]
print(class_dictionary)
it = iter(class_weights)
seclist = [
utils.POSE_CLASSES, utils.OBJ_HUMAN_CLASSES, utils.HUMAN_HUMAN_CLASSES
]
class_lists = [list(islice(it, 0, i)) for i in seclist]
print(class_lists)
# Create + compile model, load saved weights if they exist
saved_weights = None
# saved_weights = "../models/rgbextra_gauss_resnet50_1807250030.hdf5"
model, keras_layer_names = rgb_create_model(
classes=classes['label_id'],
soft_sigmoid=soft_sigmoid,
model_name=params['model'],
freeze_all=params['freeze_all'],
conv_fusion=params['conv_fusion'])
model = compile_model(model, soft_sigmoid=soft_sigmoid)
# TODO Experiment: 1. no initialization, 2. ucf initialization 3. kinetics initialization
initialization = True # Set to True to use initialization
kinetics_weights = ""
ucf_weights = "../models/ucf_keras/keras-ucf101-rgb-resnet50-newsplit.hdf5"
if saved_weights is not None:
model.load_weights(saved_weights)
else:
if initialization is True:
if ucf_weights is None:
print("Loading MConvNet weights: ")
if params['model'] == "resnet50":
ucf_weights = utils.loadmat(
"../models/ucf_matconvnet/ucf101-img-resnet-50-split1.mat"
)
utils.convert_resnet(model, ucf_weights)
model.save(
"../models/ucf_keras/keras-ucf101-rgb-resnet50-newsplit.hdf5"
)
if kinetics_weights is None:
if params['model'] == "inceptionv3":
print("Loading Keras weights: ")
keras_weights = [
"../models/kinetics_keras/tsn_rgb_params_names.pkl",
"../models/kinetics_keras/tsn_rgb_params.pkl"
]
utils.convert_inceptionv3(model, keras_weights,
keras_layer_names)
model.save(
"../models/kinetics_keras/keras-kinetics-rgb-inceptionv3.hdf5"
)
# Try to train on more than 1 GPU if possible
# try:
# print("Trying MULTI-GPU")
# model = multi_gpu_model(model)
print("Training set size: " + str(len(partition['train'])))
# Make spltis
train_splits = utils.make_chunks(original_list=partition['train'],
size=len(partition['train']),
chunk_size=params['train_chunk_size'])
val_splits = utils.make_chunks(original_list=partition['validation'],
size=len(partition['validation']),
chunk_size=params['validation_chunk_size'])
time_str = time.strftime("%y%m%d%H%M", time.localtime())
# TODO Don't forget to change your names :)
filter_type = "gauss"
bestModelPath = "../models/rgb_weightsfinal_" + filter_type + "_" + params[
'model'] + "_" + time_str + ".hdf5"
traincsvPath = "../loss_acc_plots/rgb_weightsfinal_train_" + filter_type + "_plot_" + params[
'model'] + "_" + time_str + ".csv"
valcsvPath = "../loss_acc_plots/rgb_weightsfinal_val_" + filter_type + "_plot_" + params[
'model'] + "_" + time_str + ".csv"
for epoch in range(params['nb_epochs']):
epoch_chunks_count = 0
for trainIDS in train_splits:
# Load and train
start_time = timeit.default_timer()
# -----------------------------------------------------------
x_val = y_val_pose = y_val_object = y_val_human = x_train = y_train_pose = y_train_object = y_train_human = None
x_train, y_train_pose, y_train_object, y_train_human = load_split(
trainIDS,
labels_train,
params['dim'],
params['n_channels'],
"train",
filter_type,
soft_sigmoid=soft_sigmoid)
y_t = []
y_t.append(
to_categorical(y_train_pose, num_classes=utils.POSE_CLASSES))
y_t.append(
utils.to_binary_vector(y_train_object,
size=utils.OBJ_HUMAN_CLASSES,
labeltype='object-human'))
y_t.append(
utils.to_binary_vector(y_train_human,
size=utils.HUMAN_HUMAN_CLASSES,
labeltype='human-human'))
history = model.fit(x_train,
y_t,
class_weight=class_lists,
batch_size=params['batch_size'],
epochs=1,
verbose=0)
utils.learning_rate_schedule(model, epoch, params['nb_epochs'])
# ------------------------------------------------------------
elapsed = timeit.default_timer() - start_time
print(
"Epoch " + str(epoch) + " chunk " + str(epoch_chunks_count) +
" (" + str(elapsed) + ") acc[pose,obj,human] = [" +
str(history.history['pred_pose_categorical_accuracy']) + "," +
str(history.history['pred_obj_human_categorical_accuracy']) +
"," +
str(history.history['pred_human_human_categorical_accuracy']) +
"] loss: " + str(history.history['loss']))
with open(traincsvPath, 'a') as f:
writer = csv.writer(f)
avg_acc = (
history.history['pred_pose_categorical_accuracy'][0] +
history.history['pred_obj_human_categorical_accuracy'][0] +
history.history['pred_human_human_categorical_accuracy'][0]
) / 3
writer.writerow([
str(avg_acc),
history.history['pred_pose_categorical_accuracy'],
history.history['pred_obj_human_categorical_accuracy'],
history.history['pred_human_human_categorical_accuracy'],
history.history['loss']
])
epoch_chunks_count += 1
# Load val_data
print("Validating data: ")
# global_loss, pose_loss, object_loss, human_loss, pose_acc, object_acc, human_acc
loss_acc_list = [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]
for valIDS in val_splits:
x_val = y_val_pose = y_val_object = y_val_human = x_train = y_train_pose = y_train_object = y_train_human = None
x_val, y_val_pose, y_val_object, y_val_human = load_split(
valIDS,
labels_val,
params['dim'],
params['n_channels'],
"val",
filter_type,
soft_sigmoid=soft_sigmoid)
y_v = []
y_v.append(
to_categorical(y_val_pose, num_classes=utils.POSE_CLASSES))
y_v.append(
utils.to_binary_vector(y_val_object,
size=utils.OBJ_HUMAN_CLASSES,
labeltype='object-human'))
y_v.append(
utils.to_binary_vector(y_val_human,
size=utils.HUMAN_HUMAN_CLASSES,
labeltype='human-human'))
# NOTE Can't have weights on validation
vglobal_loss, vpose_loss, vobject_loss, vhuman_loss, vpose_acc, vobject_acc, vhuman_acc = model.evaluate(
x_val, y_v, batch_size=params['batch_size'])
loss_acc_list[0] += vglobal_loss
loss_acc_list[1] += vpose_loss
loss_acc_list[2] += vobject_loss
loss_acc_list[3] += vhuman_loss
loss_acc_list[4] += vpose_acc
loss_acc_list[5] += vobject_acc
loss_acc_list[6] += vhuman_acc
# Average over all validation chunks
loss_acc_list = [x / len(val_splits) for x in loss_acc_list]
with open(valcsvPath, 'a') as f:
writer = csv.writer(f)
# We consider accuracy as the average accuracy over the three
# types of accuracy
acc = (loss_acc_list[4] + loss_acc_list[5] + loss_acc_list[6]) / 3
writer.writerow([
str(acc), loss_acc_list[4], loss_acc_list[5], loss_acc_list[6],
loss_acc_list[0], loss_acc_list[1], loss_acc_list[2],
loss_acc_list[3]
])
if loss_acc_list[0] < minValLoss:
print("New best loss " + str(loss_acc_list[0]))
model.save(bestModelPath)
minValLoss = loss_acc_list[0]
if params['email']:
utils.sendemail(
from_addr='*****@*****.**',
to_addr_list=['*****@*****.**'],
subject='Finished training RGB-stream with class_weights ' +
penalizing_method,
message='Training RGB with following params: ' + str(params),
login='******',
password='******')
def main():
K.clear_session()
root_dir = '../../data/AVA/files/'
# K.clear_session()
# Load list of action classes and separate them (from utils_stream)
classes = utils.get_AVA_classes(root_dir + 'ava_action_list_custom.csv')
# Parameters for training (batch size 32 is supposed to be the best?)
params = {
'dim': (224, 224),
'batch_size': 64,
'n_classes': len(classes['label_id']),
'n_channels': 3,
'shuffle': False,
'nb_epochs': 150,
'model': 'resnet50',
'sendmail': True,
'gen_type': 'test'
}
# Get validation set from directory
partition = {}
partition['test'] = get_AVA_set(classes=classes,
filename=root_dir +
"AVA_Test_Custom_Corrected.csv",
train=False)
filter_type = "gauss"
flowcrop = True
time_str = time.strftime("%y%m%d%H%M", time.localtime())
if flowcrop:
result_csv = "test_outputs/two-streams/output_" + params[
'gen_type'] + "_2stream_flowcrop_" + filter_type + "_" + time_str + ".csv"
else:
result_csv = "test_outputs/two-streams/output_" + params[
'gen_type'] + "_2stream_" + filter_type + "_" + time_str + ".csv"
# Load trained model
# rgb_weights = "../models/rgb_fovea_resnet50_1806301953.hdf5"
rgb_weights = "../models/rgb_gauss_resnet50_1806290918.hdf5"
# rgb_weights = "../models/rgb_crop_resnet50_1806300210.hdf5"
# rgb_weights = "../models/rgb_rgb_resnet50_1807060914.hdf5"
# flow_weights = "../models/flow_resnet50_1806281901.hdf5"
flow_weights = "../models/flowcrop_resnet50_1807180022.hdf5"
nsmodel = TwoStreamModel(classes['label_id'], rgb_weights, flow_weights)
nsmodel.compile_model(soft_sigmoid=True)
model = nsmodel.model
# two_stream_weights = "../models/two_stream_fusion_elfovresnet50_1807030015.hdf5"
# two_stream_weights = "../models/two_stream_fusion_rgbbaseline_rgb_resnet50_1809051930.hdf5"
# two_stream_weights = "../models/two_stream_fusion_flowcrop_crop_crop_resnet50_1809162007.hdf5"
# two_stream_weights = "../models/two_stream_fusion_flowcrop_fovea_fovea_resnet50_1809091554.hdf5"
two_stream_weights = "../models/two_stream_fusion_flowcrop_gauss_resnet50_1809012354.hdf5"
model.load_weights(two_stream_weights)
print("Test set size: " + str(len(partition['test'])))
# Load chunks
test_splits = utils.make_chunks(original_list=partition['test'],
size=len(partition['test']),
chunk_size=2**10)
# Test directories where pre-processed test files are
rgb_dir = "/media/pedro/actv-ssd/" + filter_type + "_"
if flowcrop is False:
flow_dir = "/media/pedro/actv-ssd/flow_"
else:
flow_dir = "/media/pedro/actv-ssd/flowcrop_"
test_chunks_count = 0
pose_votes = {}
obj_votes = {}
human_votes = {}
for row in partition['test']:
row = row.split("@")
i = row[0] + "@" + row[1] + "@" + str(row[2]) + "@" + str(
row[3]) + "@" + str(row[4]) + "@" + str(row[5])
pose_votes[i] = np.zeros(utils.POSE_CLASSES)
obj_votes[i] = np.zeros(utils.OBJ_HUMAN_CLASSES)
human_votes[i] = np.zeros(utils.HUMAN_HUMAN_CLASSES)
for testIDS in test_splits:
x_test_rgb, x_test_flow, y_test_pose, y_test_object, y_test_human = load_split(
testIDS,
None,
params['dim'],
params['n_channels'],
10,
rgb_dir,
flow_dir,
"test",
"rgb",
train=False,
crop=flowcrop)
print("Predicting on chunk " + str(test_chunks_count) + "/" +
str(len(test_splits)) + ":")
# Convert predictions to readable output and perform majority voting
predictions = model.predict([x_test_rgb, x_test_flow],
batch_size=params['batch_size'],
verbose=1)
voting.pred2classes(testIDS,
predictions,
pose_votes,
obj_votes,
human_votes,
thresh=0.4)
x_test_rgb = None
x_test_flow = None
test_chunks_count += 1
# When you're done getting all the votes, write output csv
with open(result_csv, "a") as output_file:
for key in pose_votes:
idx = key.split("@")
actions = []
pv = pose_votes[key]
pose_vote = pv.argmax(axis=0) + 1
actions.append(pose_vote)
# Get 3 top voted object
ov = obj_votes[key]
top_three_obj_votes = ov.argsort(
)[-3:][::-1] + utils.POSE_CLASSES + 1
for t in top_three_obj_votes:
if t != 0: # Often there might only be two top voted or one
actions.append(t)
# Get 3 top voted human
hv = human_votes[key]
top_three_human_votes = hv.argsort(
)[-3:][::-1] + utils.POSE_CLASSES + utils.OBJ_HUMAN_CLASSES + 1
for t in top_three_human_votes:
if t != 0: # Often there might only be two top voted or one
actions.append(t)
video_name = idx[0]
timestamp = idx[1]
bb_topx = idx[2]
bb_topy = idx[3]
bb_botx = idx[4]
bb_boty = idx[5]
for a in actions:
line = video_name + "," + timestamp + "," + bb_topx + "," + bb_topy + "," + bb_botx + "," + bb_boty + "," + str(
a)
output_file.write("%s\n" % line)
if params['sendmail']:
utils.sendemail(from_addr='*****@*****.**',
to_addr_list=['*****@*****.**'],
subject='Finished ' + params['gen_type'] +
' prediction for two stream.',
message='Testing fusion with following params: ' +
str(params),
login='******',
password='******')
def main():
K.clear_session()
root_dir = '../../data/AVA/files/'
# Load list of action classes and separate them (from utils_stream)
classes = utils.get_AVA_classes(root_dir + 'ava_action_list_custom.csv')
# Parameters for training (batch size 32 is supposed to be the best?)
# params = {'dim': (224, 224), 'batch_size': 32,
# 'n_classes': len(classes['label_id']), 'n_channels': 20,
# 'shuffle': False, 'nb_epochs': 200, 'model': 'resnet50', 'email': False,
# 'freeze_all': True, 'conv_fusion': False}
params = {
'dim': (224, 224),
'batch_size': 64,
'n_classes': len(classes['label_id']),
'n_channels': 10,
'nb_epochs': 157,
'model': "inceptionv3",
'email': True,
'freeze_all': True,
'conv_fusion': False,
'train_chunk_size': 2**10,
'validation_chunk_size': 2**10
}
crop = False # TODO Use crop flow or not
# Get validation set from directory
partition = {}
partition['test'] = get_AVA_set(classes=classes,
filename=root_dir +
"AVA_Test_Custom_Corrected.csv",
soft_sigmoid=True)
time_str = time.strftime("%y%m%d%H%M", time.localtime())
result_csv = "test_outputs/kinetics_init/output_test_flow_kineticsinit_" + time_str + ".csv"
# Load trained model
# flow_weights = "../models/flowcrop_resnet50_1807180022.hdf5"
flow_weights = "../models/flow_kineticsinit_inceptionv3_1808290834.hdf5"
model, keras_layer_names = flow_create_model(
classes=classes['label_id'],
model_name=params['model'],
soft_sigmoid=True,
image_shape=(224, 224),
opt_flow_len=10,
freeze_all=params['freeze_all'],
conv_fusion=params['conv_fusion'])
model = compile_model(model, soft_sigmoid=True)
model.load_weights(flow_weights)
print("Test set size: " + str(len(partition['test'])))
# Load chunks
test_splits = utils.make_chunks(original_list=partition['test'],
size=len(partition['test']),
chunk_size=2**10)
# Test directories where pre-processed test files are
#flow_dir = "/media/pedro/actv-ssd/flowcrop_test/"
flow_dir = "/media/pedro/actv-ssd/flow_test/"
test_chunks_count = 0
pose_votes = {}
obj_votes = {}
human_votes = {}
for row in partition['test']:
row = row.split("@")
i = row[0] + "@" + row[1] + "@" + str(row[2]) + "@" + str(
row[3]) + "@" + str(row[4]) + "@" + str(row[5])
pose_votes[i] = np.zeros(utils.POSE_CLASSES)
obj_votes[i] = np.zeros(utils.OBJ_HUMAN_CLASSES)
human_votes[i] = np.zeros(utils.HUMAN_HUMAN_CLASSES)
with tf.device('/gpu:0'):
for testIDS in test_splits:
# TODO Technically it shouldnt return labels here (these are ground truth)
#x_test_flow, y_test_pose, y_test_object, y_test_human = load_split(testIDS, None, params['dim'], params['n_channels'], "test", 10, False, encoding="rgb", soft_sigmoid=True, crop=crop)
x_test_flow, y_test_pose, y_test_object, y_test_human = load_split(
testIDS,
None,
params['dim'],
params['n_channels'],
"test",
5,
False,
encoding="rgb",
soft_sigmoid=True,
crop=crop)
print("Predicting on chunk " + str(test_chunks_count) + "/" +
str(len(test_splits)))
predictions = model.predict(x_test_flow,
batch_size=params['batch_size'],
verbose=1)
# Convert predictions to readable output and perform majority voting
voting.pred2classes(testIDS,
predictions,
pose_votes,
obj_votes,
human_votes,
thresh=0.4)
x_test_flow = None
test_chunks_count += 1
# When you're done getting all the votes, write output csv
with open(result_csv, "a") as output_file:
for key in pose_votes:
idx = key.split("@")
actions = []
pv = pose_votes[key]
pose_vote = pv.argmax(axis=0) + 1
actions.append(pose_vote)
# Get 3 top voted object
ov = obj_votes[key]
top_three_obj_votes = ov.argsort(
)[-3:][::-1] + utils.POSE_CLASSES + 1
for t in top_three_obj_votes:
if t != 0: # Often there might only be two top voted or one
actions.append(t)
# Get 3 top voted human
hv = human_votes[key]
top_three_human_votes = hv.argsort(
)[-3:][::-1] + utils.POSE_CLASSES + utils.OBJ_HUMAN_CLASSES + 1
for t in top_three_human_votes:
if t != 0: # Often there might only be two top voted or one
actions.append(t)
video_name = idx[0]
timestamp = idx[1]
bb_topx = idx[2]
bb_topy = idx[3]
bb_botx = idx[4]
bb_boty = idx[5]
for a in actions:
line = video_name + "," + timestamp + "," + bb_topx + "," + bb_topy + "," + bb_botx + "," + bb_boty + "," + str(
a)
output_file.write("%s\n" % line)
if params['email']:
utils.sendemail(from_addr='*****@*****.**',
to_addr_list=['*****@*****.**'],
subject='Finished prediction for flow (crop)',
message='Testing flow with following params: ' +
str(params),
login='******',
password='******')
def main():
# root_dir = '../../../AVA2.1/' # root_dir for the files
root_dir = '../../data/AVA/files/'
K.clear_session()
# Load list of action classes and separate them (from _stream)
classes = utils.get_AVA_classes(root_dir + 'ava_action_list_custom.csv')
# Parameters for training (batch size 32 is supposed to be the best?)
params = {
'dim': (224, 224),
'batch_size': 64,
'n_classes': len(classes['label_id']),
'n_channels': 3,
'nb_epochs': 100,
'model': 'resnet50',
'email': True,
'train_chunk_size': 2**10,
'validation_chunk_size': 2**10
}
minValLoss = 9999990.0
# Get ID's and labels from the actual dataset
partition = {}
partition['train'] = get_AVA_set(classes=classes,
filename=root_dir +
"AVA_Train_Custom_Corrected.csv",
train=True) # IDs for training
partition['validation'] = get_AVA_set(classes=classes,
filename=root_dir +
"AVA_Val_Custom_Corrected.csv",
train=True) # IDs for validation
# Labels
labels_train = get_AVA_labels(classes,
partition,
"train",
filename=root_dir +
"AVA_Train_Custom_Corrected.csv")
labels_val = get_AVA_labels(classes,
partition,
"validation",
filename=root_dir +
"AVA_Val_Custom_Corrected.csv")
# Create + compile model, load saved weights if they exist
rgb_weights = "../models/rgb_gauss_resnet50_1806290918.hdf5"
flow_weights = "../models/flow_resnet50_1806281901.hdf5"
pose_weights = "../models/pose_alexnet_1808310209.hdf5"
rgb_dir = "/media/pedro/actv-ssd/gauss_"
flow_dir = "/media/pedro/actv-ssd/flow_"
pose_dir = "/media/pedro/actv-ssd/pose_rgb_crop"
time_str = time.strftime("%y%m%d%H%M", time.localtime())
bestModelPath = "../models/fusion_pose_gauss_" + params[
'model'] + "_" + time_str + ".hdf5"
traincsvPath = "../loss_acc_plots/fusion_pose_train_gauss_plot_" + params[
'model'] + "_" + time_str + ".csv"
valcsvPath = "../loss_acc_plots/fusion_pose_val_gauss_plot_" + params[
'model'] + "_" + time_str + ".csv"
nsmodel = FusionPoseModel(classes['label_id'], rgb_weights, flow_weights,
pose_weights)
nsmodel.compile_model(soft_sigmoid=True)
model = nsmodel.model
modelpath = None
if modelpath is not None:
print("Loading previous weights")
model.load_weights(modelpath)
print("Training set size: " + str(len(partition['train'])))
# Load splits
train_splits = utils.make_chunks(original_list=partition['train'],
size=len(partition['train']),
chunk_size=params['train_chunk_size'])
val_splits = utils.make_chunks(original_list=partition['validation'],
size=len(partition['validation']),
chunk_size=params['validation_chunk_size'])
with tf.device('/gpu:0'):
for epoch in range(params['nb_epochs']):
epoch_chunks_count = 0
for trainIDS in train_splits:
start_time = timeit.default_timer()
# -----------------------------------------------------------
x_val_rgb = x_val_flow = x_val_pose = y_val_pose = y_val_object = y_val_human = x_train_rgb = x_train_pose = x_train_flow = y_train_pose = y_train_object = y_train_human = None
x_train_rgb, x_train_flow, x_train_pose, y_train_pose, y_train_object, y_train_human = load_split(
trainIDS,
labels_train,
params['dim'],
params['n_channels'],
10,
pose_dir,
rgb_dir,
flow_dir,
"rgb",
"train",
train=True)
y_train_pose = to_categorical(y_train_pose,
num_classes=utils.POSE_CLASSES)
y_train_object = utils.to_binary_vector(
y_train_object,
size=utils.OBJ_HUMAN_CLASSES,
labeltype='object-human')
y_train_human = utils.to_binary_vector(
y_train_human,
size=utils.HUMAN_HUMAN_CLASSES,
labeltype='human-human')
print("Loaded, running through CNN")
history = model.fit(
[x_train_rgb, x_train_flow, x_train_pose],
[y_train_pose, y_train_object, y_train_human],
batch_size=params['batch_size'],
epochs=1,
verbose=0)
elapsed = timeit.default_timer() - start_time
# learning_rate_schedule(model, epoch, params['nb_epochs'])
# ------------------------------------------------------------
print("Epoch " + str(epoch) + " chunk " +
str(epoch_chunks_count) + " (" + str(elapsed) +
") acc[pose,obj,human] = [" +
str(history.history['pred_pose_categorical_accuracy']) +
"," +
str(history.
history['pred_obj_human_categorical_accuracy']) +
"," +
str(history.
history['pred_human_human_categorical_accuracy']) +
"] loss: " + str(history.history['loss']))
with open(traincsvPath, 'a') as f:
writer = csv.writer(f)
avg_acc = (
history.history['pred_pose_categorical_accuracy'][0] +
history.history['pred_obj_human_categorical_accuracy']
[0] + history.history[
'pred_human_human_categorical_accuracy'][0]) / 3
writer.writerow([
str(avg_acc),
history.history['pred_pose_categorical_accuracy'],
history.history['pred_obj_human_categorical_accuracy'],
history.
history['pred_human_human_categorical_accuracy'],
history.history['loss']
])
epoch_chunks_count += 1
print("Validating data: ")
loss_acc_list = [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]
for valIDS in val_splits:
x_val_rgb = x_val_flow = x_val_pose = y_val_pose = y_val_object = y_val_human = x_train_rgb = x_train_pose = x_train_flow = y_train_pose = y_train_object = y_train_human = None
x_val_rgb, x_val_flow, x_val_pose, y_val_pose, y_val_object, y_val_human = load_split(
valIDS,
labels_val,
params['dim'],
params['n_channels'],
"val",
10,
pose_dir,
flow_dir,
"rgb",
"val",
train=True)
y_val_pose = to_categorical(y_val_pose,
num_classes=utils.POSE_CLASSES)
y_val_object = utils.to_binary_vector(
y_val_object,
size=utils.OBJ_HUMAN_CLASSES,
labeltype='object-human')
y_val_human = utils.to_binary_vector(
y_val_human,
size=utils.HUMAN_HUMAN_CLASSES,
labeltype='human-human')
vglobal_loss, vpose_loss, vobject_loss, vhuman_loss, vpose_acc, vobject_acc, vhuman_acc = model.evaluate(
[x_val_rgb, x_val_flow, x_val_pose],
[y_val_pose, y_val_object, y_val_human],
batch_size=params['batch_size'])
loss_acc_list[0] += vglobal_loss
loss_acc_list[1] += vpose_loss
loss_acc_list[2] += vobject_loss
loss_acc_list[3] += vhuman_loss
loss_acc_list[4] += vpose_acc
loss_acc_list[5] += vobject_acc
loss_acc_list[6] += vhuman_acc
loss_acc_list = [x / len(val_splits) for x in loss_acc_list]
with open(valcsvPath, 'a') as f:
writer = csv.writer(f)
acc = (loss_acc_list[4] + loss_acc_list[5] +
loss_acc_list[6]) / 3
writer.writerow([
str(acc), loss_acc_list[4], loss_acc_list[5],
loss_acc_list[6], loss_acc_list[0], loss_acc_list[1],
loss_acc_list[2], loss_acc_list[3]
])
if loss_acc_list[0] < minValLoss:
print("New best loss " + str(loss_acc_list[0]))
model.save(bestModelPath)
minValLoss = loss_acc_list[0]
if params['email']:
utils.sendemail(from_addr='*****@*****.**',
to_addr_list=['*****@*****.**'],
cc_addr_list=[],
subject='Finished training pose fusion',
message='Training fusion with following params: ' +
str(params),
login='******',
password='******')
"""
名詞を含む文節が，動詞を含む文節に係るとき，これらをタブ区切り形式で抽出せよ．
ただし，句読点などの記号は出力しないようにせよ．
"""
from utils import make_chunks
from constants import FNAME_PARSED
# 12文目まで表示
if __name__ == '__main__':
for i, chunks in enumerate(make_chunks(FNAME_PARSED)):
for chunk in chunks:
if chunk.dst != -1 and chunk.chk_pos('名詞') and chunks[
chunk.dst].chk_pos('動詞'):
print('{}\t{}'.format(chunk.normalized_surface(),
chunks[chunk.dst].normalized_surface()))
if i == 12:
break
# phones
path_phones = '/home/deniz/Schreibtisch/ctcLstmNN/rikoSkripts/phnrec/phones.txt'
utterance_dict = utils.load_pickled_file(path_utterances)
phones = utils.load_phone_file(path_phones)
int2sym = utils.make_int2sym(phones)
sym2int = utils.make_sym2int(int2sym)
min_phone, max_phone = utils.min_max_int2sym(int2sym)
alignments = utils.filter_alignments(path_alignments, 50, min_phone, max_phone)
mfcc_chunks, alignment_chunks = utils.make_chunks(utterance_dict, alignments)
inputs = OrderedDict()
for utt_key, mfcc in mfcc_chunks:
inputs[utt_key] = mfcc
targets = OrderedDict()
for ali_key, ali in alignment_chunks:
targets[ali_key] = ali
t_inputs = list()
t_targets = list()
for utt_key, mfcc in inputs.items():
t_inputs.append(mfcc)
t_targets.append(np.asarray(targets[utt_key]))
def main():
GPU = False
CPU = True
num_cores = 8
if GPU:
num_GPU = 1
num_CPU = 1
if CPU:
num_CPU = 1
num_GPU = 0
config = tf.ConfigProto(intra_op_parallelism_threads=num_cores,
inter_op_parallelism_threads=num_cores,
allow_soft_placement=True,
device_count={
'CPU': num_CPU,
'GPU': num_GPU
})
session = tf.Session(config=config)
K.set_session(session)
root_dir = '../../data/AVA/files/'
K.clear_session()
# Load list of action classes and separate them (from utilsm)
classes = utils.get_AVA_classes(root_dir + 'ava_action_list_custom.csv')
# Parameters for training
params = {
'dim': (224, 224),
'batch_size': 64,
'n_classes': len(classes['label_id']),
'n_channels': 3,
'shuffle': False,
'nb_epochs': 150,
'model': 'resnet50',
'sendmail': False,
'gen_type': 'test'
}
# Get validation set from directory
partition = {}
partition['test'] = get_AVA_set(classes=classes,
filename=root_dir +
"AVA_Test_Custom_Corrected.csv",
train=False)
filter_type = "gauss"
time_str = time.strftime("%y%m%d%H%M", time.localtime())
result_csv = "output_" + params[
'gen_type'] + "_fusion_context_" + filter_type + "_" + time_str + ".csv"
# Load trained model
# rgb_weights = "../models/rgb_fovea_resnet50_1806301953.hdf5"
# rgb_weights = "../models/rgb_crop_resnet50_1806300210.hdf5"
rgb_weights = "../models/rgb_gauss_resnet50_1806290918.hdf5"
flow_weights = "../models/flow_resnet50_1806281901.hdf5"
context_weights = "../models/TW1/context_mlp128.hdf5"
# fusion_context_weights = "../models/fusion_context_fusion_fovea_resnet50_1807061835.hdf5"
# fusion_context_weights = "../models/fusion_context_fusion_crop_resnet50_1807181004.hdf5"
fusion_context_weights = "../models/fusion_context_fusion_gaussian_resnet50_1807210918.hdf5"
nsmodel = FusionContextModel(classes['label_id'], rgb_weights,
flow_weights, context_weights)
nsmodel.compile_model(soft_sigmoid=True)
model = nsmodel.model
model.load_weights(fusion_context_weights)
print("Test set size: " + str(len(partition['test'])))
print(
"Building context dictionary from context file (these should be generated)..."
)
Xfilename = root_dir + "context_files/XContext_test_pastfuture.csv"
test_context_rows = {}
with open(Xfilename) as csvDataFile:
csvReader = csv.reader(csvDataFile)
for row in csvReader:
rkey = row[0] + "_" + row[1].lstrip("0") + \
"@" + str(row[2]) + "@" + str(row[3]) + "@" + str(row[4]) + "@" + str(row[5])
test_context_rows[rkey] = row[6]
# Load chunks
test_splits = utils.make_chunks(original_list=partition['test'],
size=len(partition['test']),
chunk_size=2**10)
# Test directories where pre-processed test files are
rgb_dir = "/media/pedro/actv-ssd/" + filter_type + "_"
flow_dir = "/media/pedro/actv-ssd/flow_"
test_chunks_count = 0
pose_votes = {}
obj_votes = {}
human_votes = {}
for row in partition['test']:
row = row.split("@")
i = row[0] + "@" + row[1] + "@" + str(row[2]) + "@" + str(
row[3]) + "@" + str(row[4]) + "@" + str(row[5])
pose_votes[i] = np.zeros(utils.POSE_CLASSES)
obj_votes[i] = np.zeros(utils.OBJ_HUMAN_CLASSES)
human_votes[i] = np.zeros(utils.HUMAN_HUMAN_CLASSES)
print("Starting testing:")
with tf.device('/gpu:0'):
for testIDS in test_splits:
x_test_rgb, x_test_flow, x_test_context, y_test_pose, y_test_object, y_test_human = load_split(
testIDS,
None,
params['dim'],
params['n_channels'],
10,
test_context_rows,
rgb_dir,
flow_dir,
"rgb",
"test",
train=False)
print("Predicting on chunk " + str(test_chunks_count) + "/" +
str(len(test_splits)) + ":")
# Convert predictions to readable output and perform majority voting
predictions = model.predict(
[x_test_rgb, x_test_flow, x_test_context],
batch_size=params['batch_size'],
verbose=1)
voting.pred2classes(testIDS,
predictions,
pose_votes,
obj_votes,
human_votes,
thresh=0.4)
x_test_rgb = None
x_test_flow = None
x_test_context = None
test_chunks_count += 1
# When you're done getting all the votes, write output csv
with open(result_csv, "a") as output_file:
for key in pose_votes:
idx = key.split("@")
actions = []
pv = pose_votes[key]
pose_vote = pv.argmax(axis=0) + 1
actions.append(pose_vote)
# Get 3 top voted object
ov = obj_votes[key]
top_three_obj_votes = ov.argsort(
)[-3:][::-1] + utils.POSE_CLASSES + 1
for t in top_three_obj_votes:
if t != 0: # Often there might only be two top voted or one
actions.append(t)
# Get 3 top voted human
hv = human_votes[key]
top_three_human_votes = hv.argsort(
)[-3:][::-1] + utils.POSE_CLASSES + utils.OBJ_HUMAN_CLASSES + 1
for t in top_three_human_votes:
if t != 0: # Often there might only be two top voted or one
actions.append(t)
video_name = idx[0]
timestamp = idx[1]
bb_topx = idx[2]
bb_topy = idx[3]
bb_botx = idx[4]
bb_boty = idx[5]
for a in actions:
line = video_name + "," + timestamp + "," + bb_topx + "," + bb_topy + "," + bb_botx + "," + bb_boty + "," + str(
a)
output_file.write("%s\n" % line)
if params['sendmail']:
utils.sendemail(
from_addr='*****@*****.**',
to_addr_list=[
'*****@*****.**', '*****@*****.**'
],
subject='Finished ' + params['gen_type'] +
' prediction for three stream.',
message='Testing fusion with following params: ' + str(params),
login='******',
password='******')
def main():
root_dir = '../../data/AVA/files/'
# Load list of action classes and separate them (from utils_stream)
classes = utils.get_AVA_classes(root_dir + 'ava_action_list_custom.csv')
# Parameters for training (batch size 32 is supposed to be the best?)
# Parameters for training
params = {
'dim': (300, 300),
'batch_size': 32,
'n_classes': len(classes['label_id']),
'n_channels': 3,
'nb_epochs': 200,
'model': 'alexnet',
'email': True,
'train_chunk_size': 2**12,
'validation_chunk_size': 2**12
}
soft_sigmoid = True
store_predictions = True
minValLoss = 9999990.0
split = "test"
# Get validation set from directory
partition = {}
partition['test'] = get_AVA_set(classes=classes,
filename=root_dir + "AVA_" +
split.title() + "_Custom_Corrected.csv",
soft_sigmoid=True)
time_str = time.strftime("%y%m%d%H%M", time.localtime())
result_csv = "output_test_pose_" + time_str + ".csv"
# Load trained model
pose_weights = "../models/pose_alexnet_1808310209.hdf5"
model = pose_create_model(classes=classes['label_id'],
soft_sigmoid=soft_sigmoid,
image_shape=params['dim'],
model_name=params['model'])
model = compile_model(model, soft_sigmoid=soft_sigmoid)
model.load_weights(pose_weights)
print("Test set size: " + str(len(partition['test'])))
# Load chunks
test_splits = utils.make_chunks(original_list=partition['test'],
size=len(partition['test']),
chunk_size=2**11)
# Test directories where pre-processed test files are
pose_dir = "/media/pedro/actv-ssd/pose_" + split + "/"
test_chunks_count = 0
pose_votes = {}
obj_votes = {}
human_votes = {}
for row in partition['test']:
row = row.split("@")
i = row[0] + "@" + row[1] + "@" + str(row[2]) + "@" + str(
row[3]) + "@" + str(row[4]) + "@" + str(row[5])
pose_votes[i] = np.zeros(utils.POSE_CLASSES)
obj_votes[i] = np.zeros(utils.OBJ_HUMAN_CLASSES)
human_votes[i] = np.zeros(utils.HUMAN_HUMAN_CLASSES)
test_predictions = []
with tf.device('/gpu:0'):
for testIDS in test_splits:
# TODO Technically it shouldnt return labels here (these are ground truth)
x_test_pose, y_test_pose, y_test_object, y_test_human = load_split(
testIDS,
None,
params['dim'],
params['n_channels'],
split,
filter_type,
soft_sigmoid=True,
train=False)
print("Predicting on chunk " + str(test_chunks_count) + "/" +
str(len(test_splits)))
predictions = model.predict(x_test_pose,
batch_size=params['batch_size'],
verbose=1)
if store_predictions is True:
# print(predictions[0][0])
# print(predictions[1][0])
# print(predictions[2][0])
# tarr = np.hstack((np.vstack(predictions[0]), np.vstack(predictions[1]), np.vstack(predictions[2])))
test_predictions.append(predictions)
# Convert predictions to readable output and perform majority voting
voting.pred2classes(testIDS,
predictions,
pose_votes,
obj_votes,
human_votes,
thresh=0.4)
test_chunks_count += 1
if store_predictions is True:
#tp = np.vstack(test_predictions)
# print(tp.shape)
with open("thresholds/pose/predictions_pose_" + time_str + ".pickle",
'wb') as handle:
pickle.dump(test_predictions,
handle,
protocol=pickle.HIGHEST_PROTOCOL)
# When you're done getting all the votes, write output csv
with open(result_csv, "a") as output_file:
for key in pose_votes:
idx = key.split("@")
actions = []
pv = pose_votes[key]
pose_vote = pv.argmax(axis=0) + 1
actions.append(pose_vote)
# Get 3 top voted object
ov = obj_votes[key]
top_three_obj_votes = ov.argsort(
)[-3:][::-1] + utils.POSE_CLASSES + 1
for t in top_three_obj_votes:
if t != 0: # Often there might only be two top voted or one
actions.append(t)
# Get 3 top voted human
hv = human_votes[key]
top_three_human_votes = hv.argsort(
)[-3:][::-1] + utils.POSE_CLASSES + utils.OBJ_HUMAN_CLASSES + 1
for t in top_three_human_votes:
if t != 0: # Often there might only be two top voted or one
actions.append(t)
video_name = idx[0]
timestamp = idx[1]
bb_topx = idx[2]
bb_topy = idx[3]
bb_botx = idx[4]
bb_boty = idx[5]
for a in actions:
line = video_name + "," + timestamp + "," + bb_topx + "," + bb_topy + "," + bb_botx + "," + bb_boty + "," + str(
a)
output_file.write("%s\n" % line)
if params['email']:
utils.sendemail(from_addr='*****@*****.**',
to_addr_list=['*****@*****.**'],
subject='Finished prediction for ' + filter_type,
message='Testing pose with following params: ' +
str(params),
login='******',
password='******')
"""
set 3 challenge 18: implement AES-CTR mode.
"""
from base64 import b64decode
from Crypto.Cipher import AES
from utils import BLOCK_SIZE, make_chunks, to_ascii
ciphertxt = b64decode(
"L77na/nrFsKvynd6HzOoG7GHTLXsTVu9qvY/2syLXzhPweyyMTJULu/6/kXX0KSvoOLSFQ==")
key = b"YELLOW SUBMARINE"
nonce = b"\x00" * BLOCK_SIZE
def ctr_mode(key, nonce):
a = AES.new(key, AES.MODE_ECB)
while True:
yield a.encrypt(nonce)
nonce = nonce[:BLOCK_SIZE // 2] + bytes(
[nonce[BLOCK_SIZE // 2] + 1]) + nonce[(BLOCK_SIZE // 2) + 1:]
print("".join([
"".join([chr(c ^ i) for c, i in zip(chunk, ctr)])
for chunk, ctr in zip(make_chunks(ciphertxt), ctr_mode(key, nonce))
]))
def main():
# root_dir for the files
root_dir = '../../data/AVA/files/'
# Erase previous models from GPU memory
K.clear_session()
# Load list of action classes and separate them (from utils_stream)
classes = utils.get_AVA_classes(root_dir + 'ava_action_list_custom.csv')
# Parameters for training (batch size 32 is supposed to be the best?)
params = {
'dim': (224, 224),
'batch_size': 64,
'n_classes': len(classes['label_id']),
'n_channels': 10,
'nb_epochs': 157,
'model': "inceptionv3",
'email': True,
'freeze_all': True,
'conv_fusion': False,
'train_chunk_size': 2**10,
'validation_chunk_size': 2**10
}
minValLoss = 9999990.0
soft_sigmoid = True
warp = False # TODO Use warped (camera motion corrected) flow or not
crop = False # TODO Use crop flow or not
encoding = "rgb" # TODO Are flows stored as rgb or as 2 grayscales
# Get ID's and labels from the actual dataset
partition = {}
partition['train'] = get_AVA_set(
classes=classes,
filename=root_dir + "AVA_Train_Custom_Corrected.csv",
soft_sigmoid=soft_sigmoid) # IDs for training
partition['validation'] = get_AVA_set(
classes=classes,
filename=root_dir + "AVA_Val_Custom_Corrected.csv",
soft_sigmoid=soft_sigmoid) # IDs for validation
# Labels
labels_train = get_AVA_labels(classes,
partition,
"train",
filename=root_dir +
"AVA_Train_Custom_Corrected.csv",
soft_sigmoid=soft_sigmoid)
labels_val = get_AVA_labels(classes,
partition,
"validation",
filename=root_dir +
"AVA_Val_Custom_Corrected.csv",
soft_sigmoid=soft_sigmoid)
# Create + compile model, load saved weights if they exist
# saved_weights = "saved_models/RGB_Stream_Softmax_inceptionv3.hdf5"
saved_weights = "../models/flow_kineticsinit_inceptionv3_1808282350.hdf5"
ucf_weights = "../models/ucf_keras/keras-ucf101-TVL1flow-" + params[
'model'] + "-newsplit.hdf5" # Outdated model
# ucf_weights = None
# ucf_weights = None
model, keras_layer_names = flow_create_model(
classes=classes['label_id'],
model_name=params['model'],
soft_sigmoid=soft_sigmoid,
image_shape=(224, 224),
opt_flow_len=10,
freeze_all=params['freeze_all'],
conv_fusion=params['conv_fusion'])
model = compile_model(model, soft_sigmoid=soft_sigmoid)
# TODO Experiment: 1. no initialization, 2. ucf initialization 3. kinetics initialization
initialization = True # Set to True to use initialization
kinetics_weights = None
ucf_weights = ""
if saved_weights is not None:
model.load_weights(saved_weights)
else:
if initialization is True:
if kinetics_weights is None:
if params['model'] == "inceptionv3":
print("Loading kinetics weights: ")
keras_weights = [
"../models/kinetics_keras/tsn_flow_params_names.pkl",
"../models/kinetics_keras/tsn_flow_params.pkl"
]
utils.convert_inceptionv3(model, keras_weights,
keras_layer_names)
model.save(
"../models/kinetics_keras/keras-kinetics-flow-inceptionv3.hdf5"
)
if ucf_weights is None:
print("Loading UCF weights: ")
if params['model'] == "resnet50":
# TODO Better initialization, average UCF models overt he 3 splits provided
ucf_weights = utils.loadmat(
"../models/ucf_matconvnet/ucf101-TVL1flow-resnet-50-split1.mat"
)
utils.convert_resnet(model, ucf_weights)
model.save(
"../models/ucf_keras/keras-ucf101-TVL1flow-resnet50-newsplit.hdf5"
)
else:
if ucf_weights != "":
model.load_weights(ucf_weights)
# Try to train on more than 1 GPU if possible
# try:
# print("Trying MULTI-GPU")
# model = multi_gpu_model(model)
print("Training set size: " + str(len(partition['train'])))
# Load first train_size of partition{'train'}
train_splits = utils.make_chunks(original_list=partition['train'],
size=len(partition['train']),
chunk_size=params['train_chunk_size'])
val_splits = utils.make_chunks(original_list=partition['validation'],
size=len(partition['validation']),
chunk_size=params['validation_chunk_size'])
time_str = time.strftime("%y%m%d%H%M", time.localtime())
if crop is True:
bestModelPath = "../models/flowcrop_" + params[
'model'] + "_" + time_str + ".hdf5"
traincsvPath = "../loss_acc_plots/flowcrop_customcsv_train_plot_" + params[
'model'] + "_" + time_str + ".csv"
valcsvPath = "../loss_acc_plots/flowcrop_customcsv_val_plot_" + params[
'model'] + "_" + time_str + ".csv"
else:
if warp is True:
bestModelPath = "../models/flow_warp_" + params[
'model'] + "_" + time_str + ".hdf5"
traincsvPath = "../loss_acc_plots/flow_warp_customcsv_train_plot_" + params[
'model'] + "_" + time_str + ".csv"
valcsvPath = "../loss_acc_plots/flow_warp_customcsv_val_plot_" + params[
'model'] + "_" + time_str + ".csv"
else:
bestModelPath = "../models/flow_kineticsinit_" + params[
'model'] + "_" + time_str + ".hdf5"
traincsvPath = "../loss_acc_plots/flow_kineticsinit_customcsv_train_plot_" + params[
'model'] + "_" + time_str + ".csv"
valcsvPath = "../loss_acc_plots/flow_kineticsinit_customcsv_val_plot_" + params[
'model'] + "_" + time_str + ".csv"
first_epoch = True
with tf.device('/gpu:0'): # TODO Multi GPU
for epoch in range(params['nb_epochs']):
epoch_chunks_count = 0
if epoch > 0:
first_epoch = False
for trainIDS in train_splits:
start_time = timeit.default_timer()
# -----------------------------------------------------------
x_val = y_val_pose = y_val_object = y_val_human = x_train = y_train_pose = y_train_object = y_train_human = None
x_train, y_train_pose, y_train_object, y_train_human = load_split(
trainIDS,
labels_train,
params['dim'],
params['n_channels'],
"train",
5,
first_epoch,
encoding=encoding,
soft_sigmoid=soft_sigmoid,
crop=True)
y_t = []
y_t.append(
to_categorical(y_train_pose,
num_classes=utils.POSE_CLASSES))
y_t.append(
utils.to_binary_vector(y_train_object,
size=utils.OBJ_HUMAN_CLASSES,
labeltype='object-human'))
y_t.append(
utils.to_binary_vector(y_train_human,
size=utils.HUMAN_HUMAN_CLASSES,
labeltype='human-human'))
history = model.fit(x_train,
y_t,
batch_size=params['batch_size'],
epochs=1,
verbose=0)
utils.learning_rate_schedule(model, epoch, params['nb_epochs'])
# ------------------------------------------------------------
elapsed = timeit.default_timer() - start_time
print("Epoch " + str(epoch) + " chunk " +
str(epoch_chunks_count) + " (" + str(elapsed) +
") acc[pose,obj,human] = [" +
str(history.history['pred_pose_categorical_accuracy']) +
"," +
str(history.
history['pred_obj_human_categorical_accuracy']) +
"," +
str(history.
history['pred_human_human_categorical_accuracy']) +
"] loss: " + str(history.history['loss']))
with open(traincsvPath, 'a') as f:
writer = csv.writer(f)
avg_acc = (
history.history['pred_pose_categorical_accuracy'][0] +
history.history['pred_obj_human_categorical_accuracy']
[0] + history.history[
'pred_human_human_categorical_accuracy'][0]) / 3
writer.writerow([
str(avg_acc),
history.history['pred_pose_categorical_accuracy'],
history.history['pred_obj_human_categorical_accuracy'],
history.
history['pred_human_human_categorical_accuracy'],
history.history['loss']
])
epoch_chunks_count += 1
# Load val_data
print("Validating data: ")
loss_acc_list = [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]
for valIDS in val_splits:
x_val = y_val_pose = y_val_object = y_val_human = x_train = y_train_pose = y_train_object = y_train_human = None
x_val, y_val_pose, y_val_object, y_val_human = load_split(
valIDS,
labels_val,
params['dim'],
params['n_channels'],
"val",
5,
first_epoch,
encoding=encoding,
soft_sigmoid=soft_sigmoid,
crop=True)
y_v = []
y_v.append(
to_categorical(y_val_pose, num_classes=utils.POSE_CLASSES))
y_v.append(
utils.to_binary_vector(y_val_object,
size=utils.OBJ_HUMAN_CLASSES,
labeltype='object-human'))
y_v.append(
utils.to_binary_vector(y_val_human,
size=utils.HUMAN_HUMAN_CLASSES,
labeltype='human-human'))
vglobal_loss, vpose_loss, vobject_loss, vhuman_loss, vpose_acc, vobject_acc, vhuman_acc = model.evaluate(
x_val, y_v, batch_size=params['batch_size'])
loss_acc_list[0] += vglobal_loss
loss_acc_list[1] += vpose_loss
loss_acc_list[2] += vobject_loss
loss_acc_list[3] += vhuman_loss
loss_acc_list[4] += vpose_acc
loss_acc_list[5] += vobject_acc
loss_acc_list[6] += vhuman_acc
loss_acc_list = [x / len(val_splits) for x in loss_acc_list]
with open(valcsvPath, 'a') as f:
writer = csv.writer(f)
acc = (loss_acc_list[4] + loss_acc_list[5] +
loss_acc_list[6]) / 3
writer.writerow([
str(acc), loss_acc_list[4], loss_acc_list[5],
loss_acc_list[6], loss_acc_list[0], loss_acc_list[1],
loss_acc_list[2], loss_acc_list[3]
])
if loss_acc_list[0] < minValLoss:
print("New best loss " + str(loss_acc_list[0]))
model.save(bestModelPath)
minValLoss = loss_acc_list[0]
if params['email']:
utils.sendemail(from_addr='*****@*****.**',
to_addr_list=['*****@*****.**'],
subject='Finished training OF-stream ',
message='Training OF with following params: ' +
str(params),
login='******',
password='******')