def main():
if len(sys.argv) < 2:
print("Behavioral Cloning\n", "Usage: python3 main.py config.json")
else:
config_file = sys.argv[1]
with open(config_file) as yaml_file:
# The FullLoader parameter handles the conversion from YAML
# scalar values to Python the dictionary format
configs = yaml.load(yaml_file, Loader=yaml.FullLoader)
# Data configurations
data_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), configs["data_path"])
labels_file = configs["labels_file"]
skip_header = bool(configs["skipHeader"])
use_side_images = bool(configs["useSideImages"])
correction_factor = float(configs["correctionFactor"])
train_valid_split = float(configs["trainValidSplit"])
do_flip = bool(configs["doFlip"])
# Training configurations
top_crop = int(configs["topCrop"])
bottom_crop = int(configs["bottomCrop"])
batch_size = int(configs["batchSize"])
epochs = int(configs["epochs"])
loss = configs["loss"]
optimizer = configs["optimizer"]
verbose = int(configs["verbose"])
model_name = configs["modelName"]
output_dir = configs["outputDir"]
# Init Preprocessing
preprocess = PreprocessData(data_path, labels_file, correction_factor, skip_header, use_side_images,
train_valid_split, do_flip)
# Preprocess data and extract training and validation samples
train_samples, validation_samples = preprocess.splitData()
# Initialize train and validation generators
train_generator = preprocess.generator(train_samples, batch_size=batch_size)
validation_generator = preprocess.generator(validation_samples, batch_size=batch_size)
# Get image shape
img_shape = preprocess.get_image_shape()
# Initialize training network
network = Model(img_shape, top_crop, bottom_crop, batch_size, epochs, loss, optimizer, verbose,
train_generator,
validation_generator, train_samples, validation_samples, model_name)
model = network.create_model()
# Initialize visualization
visualize = Visualization(model, output_dir)
visualize.visualize_model()
network.get_summary(model)
results = network.train_model(model)
visualize.save_plots(results)
def prepare_data(data_size, max_len_1, max_len_2, preprocess, balance_data):
if preprocess:
pd = PreprocessData(data_size, max_len_1, max_len_2)
pd.storeData()
data_padded = pickle.load(
open("data/data_emb200_pad_%d.p" % data_size, "rb"))
print(data_padded.columns)
if balance_data:
data = balance_model(data_padded)
else:
data = data_padded
# Split to train, dev and test pandas dataframe
# print "Splitting embedded data..."
train, dev, test = df_splitter(data)
# Obtain train, dev, test data
train_X1, train_X2, train_label = get_x_y(train)
dev_X1, dev_X2, dev_label = get_x_y(dev)
test_X1, test_X2, test_label = get_x_y(test)
# Normalize x1, x2
train_X1, dev_X1, test_X1 = standardize(train_X1, dev_X1, test_X1)
train_X2, dev_X2, test_X2 = standardize(train_X2, dev_X2, test_X2)
print('train non-duplicate prop:', 1 - np.mean(train_label))
print('dev non-duplicate prop:', 1 - np.mean(dev_label))
print('test non-duplicate prop:', 1 - np.mean(test_label))
# Transform y to one hot vector
n_labels = 2
train_y = np.zeros((train_label.shape[0], n_labels))
dev_y = np.zeros((dev_label.shape[0], n_labels))
test_y = np.zeros((test_label.shape[0], n_labels))
train_y[np.arange(train_label.shape[0]), train_label] = 1
dev_y[np.arange(dev_label.shape[0]), dev_label] = 1
test_y[np.arange(test_label.shape[0]), test_label] = 1
return data_padded, train, train_X1, train_X2, train_y, dev_X1, dev_X2, dev_y, test_X1, test_X2, test_y
dataset_path = sys.argv[1]
train_dir = sys.argv[2]
split_type = sys.argv[3]
experiment_type = sys.argv[4]
orthographic_insertion_place = 'NONE' if len(
sys.argv) <= 5 else sys.argv[5]
orthographic_insertion_type = 'ONE_HOT' if len(
sys.argv) <= 6 else sys.argv[6]
orthographic_insertion_place = OrthographicConcatePlace[
orthographic_insertion_place.upper()]
orthographic_insertion_type = OrthographicFeatureForm[
orthographic_insertion_type.upper()]
p = PreprocessData(dataset_type='wsj')
files = p.preProcessDirectory(dataset_path)
if split_type == 'standard':
train_files, val_files, test_files = p.get_standard_split(files)
else:
shuffle(files)
train_files, test_val_files = p.split_data(files, 0.8)
test_files, val_files = p.split_data(test_val_files, 0.5)
train_mat = p.get_raw_data(train_files, 'train')
val_mat = p.get_raw_data(val_files, 'validation')
test_mat = p.get_raw_data(test_files, 'test')
X_train, y_train, P_train, S_train, XC_train, XN_train, XH_train, _ = p.get_processed_data(
# specify directory where model is saved or loaded
train_dir = sys.argv[3]
if not os.path.exists(train_dir):
os.makedirs(train_dir)
# specify train or test
experiment_type = sys.argv[4]
if len(sys.argv) >= 6:
MAX_LENGTH = int(sys.argv[6])
print "Using max length: {0}".format(MAX_LENGTH)
# initialize a new PreprocessData instance
p = PreprocessData(sys.argv[5])
# split them into training, validation, and test files
# these will be saved in train_dir/train.txt, train_dir/val.txt, train_dir/test.txt
train_file, val_file, test_file = p.get_standard_split(
clickbait_raw_path, normal_raw_path, train_dir)
train_mat = p.process_single_file(train_file, 'train')
val_mat = p.process_single_file(val_file, 'validation')
test_mat = p.process_single_file(test_file, 'test')
X_train, Y_train = p.get_processed_data(train_mat, MAX_LENGTH)
X_val, Y_val = p.get_processed_data(val_mat, MAX_LENGTH)
X_test, Y_test = p.get_processed_data(test_mat, MAX_LENGTH)
if experiment_type == 'train':
if os.path.exists(train_dir):
saver.restore(sess, ckpt.model_checkpoint_path)
global_step = ckpt.model_checkpoint_path.split('/')[-1].split('-')[-1]
test_loss, test_accuracy, test_oov_accuracy = compute_summary_metrics(sess, m, sentence_words_test,
sentence_tags_test)
print 'Test Accuracy: {:.3f}'.format(test_accuracy)
print 'Test Loss: {:.3f}'.format(test_loss)
print 'Test OOV Accuracy: {:.3f}'.format(test_oov_accuracy)
if __name__ == '__main__':
dataset_path = sys.argv[1]
train_dir = sys.argv[2]
split_type = sys.argv[3]
experiment_type = sys.argv[4]
p = PreprocessData(dataset_type='wsj', model_type_=int(sys.argv[5]))
files = p.preProcessDirectory(dataset_path)
if split_type == 'standard':
train_files, val_files, test_files = p.get_standard_split(files)
else:
shuffle(files)
train_files, test_val_files = p.split_data(files, 0.8)
test_files, val_files = p.split_data(test_val_files, 0.5)
train_mat = p.get_raw_data(train_files, 'train')
val_mat = p.get_raw_data(val_files, 'validation')
test_mat = p.get_raw_data(test_files, 'test')
X_train, y_train, _ = p.get_processed_data(train_mat, MAX_LENGTH)
args = parser.parse_args()
if args.load_data:
_ = LoadData(data_path=args.data,
checkpoint_path=args.data_checkpoint,
train_file=args.dataset_name + '_' + 'train.csv',
dev_file=args.dataset_name + '_' + 'dev.csv')
# Create Checkpoint folders for in between sessions storage
create_checkpoint_paths(args.data_checkpoint)
create_checkpoint_paths(args.model_checkpoint)
data_preprocessor = PreprocessData(data_path=args.data_checkpoint,
glove_size=args.glove_size,
batch_size=args.batch_size,
train_file='train1.csv',
dev_file='dev1.csv')
# train_file=args.dataset_name + '_' + 'train.csv',
# dev_file=args.dataset_name + '_' + 'dev.csv')
save_vocab(data_preprocessor.WORDS.vocab, args.model_checkpoint + '/' + args.dataset_name + '_' + 'WORDS.vocab')
save_vocab(data_preprocessor.CHAR.vocab, args.model_checkpoint + '/' + args.dataset_name + '_' + 'CHAR.vocab')
# Initializing Bidaf Model
model = BidaF(data_preprocessor.WORDS,
data_preprocessor.CHAR,
char_embedding_size=args.char_embedding_size,
char_conv_kernel_size=args.kernel_size,
char_conv_channels_count=args.channels_count).to(device)
from preprocess import PreprocessData
if __name__ == '__main__':
PreprocessData.runTxt()
PreprocessData.runImgs()