def main():
# Load the parameters from json file
args = parser.parse_args()
json_path = os.path.join(args.model_dir, 'params.json')
params = Params(json_path)
# Set the logger
set_logger(os.path.join(args.model_dir, 'train.log'))
# Create the input data pipeline
logging.info('Creating the dataset...')
data_dir = args.data_dir
valid_data_dir = os.path.join(data_dir, 'valid')
# Get the filenames and labels from the test set
valid_filenames, valid_labels = get_filenames_and_labels(
valid_data_dir, params)
params.valid_size = len(valid_filenames)
params.num_labels = len(set(valid_labels))
# Create the two iterators over the two datasets
valid_inputs = input_fn(False, valid_filenames,
valid_labels, params)
# Define the model
logging.info("Creating the model...")
model_spec = model_fn('eval', valid_inputs, params,
reuse=False)
logging.info("Starting evaluation")
evaluate(model_spec, args.model_dir, params,
args.restore_from)
def __init__(self):
# Load the parameters
args = EvaluatePointConfig()
json_path = os.path.join(args.model_dir, 'params.json')
assert os.path.isfile(
json_path), "No json configuration file found at {}".format(
json_path)
params = Params(json_path)
if params.mlp_sizes is None or len(params.mlp_sizes) == 0:
logging.error(
'mlp_sizes are not set correctly, at least one MLP layer is required'
)
params.dict['loss_fn'] = args.loss_fn
# Load the parameters from the dataset, that gives the size etc. into params
json_path = os.path.join(args.data_dir, 'dataset_params.json')
assert os.path.isfile(
json_path), "No json file found at {}, run build.py".format(
json_path)
params.update(json_path)
# Set the logger
set_logger(os.path.join(args.model_dir, 'evaluate.log'))
# # Get paths for tfrecords
path_eval_tfrecords = os.path.join(args.data_dir,
'test_' + args.tfrecords_filename)
# Create the input data pipeline
logging.info("Creating the dataset...")
eval_dataset = load_dataset_from_tfrecords(path_eval_tfrecords)
# Create iterator over the test set
# eval_inputs = input_fn('test', eval_dataset, params)
eval_inputs = online_input_fn()
logging.info("- done.")
# print(type(eval_inputs))
# Define the model
logging.info("Creating the model...")
weak_learner_id = load_best_ndcgs(
os.path.join(args.model_dir, args.restore_from, 'learner.json'))[0]
self.model_spec = model_fn('test',
eval_inputs,
params,
reuse=False,
weak_learner_id=int(weak_learner_id))
# node_names = [n.name for n in tf.get_default_graph().as_graph_def().node]
# print(node_names)
logging.info("- done.")
logging.info("Starting evaluation")
logging.info("Optimized using {} learners".format(weak_learner_id))
self.saver = tf.train.Saver()
self.sess = tf.Session()
self.params = params
self.sess.run(self.model_spec['variable_init_op'])
save_path = os.path.join(args.model_dir, args.restore_from)
if os.path.isdir(save_path):
save_path = tf.train.latest_checkpoint(save_path)
self.saver.restore(self.sess, save_path)
def main():
# Load the parameters from json file
args = parser.parse_args()
json_path = os.path.join(args.model_dir, 'params.json')
assert os.path.isfile(json_path), 'No json configuration file found at {}'.format(json_path)
params = Params(json_path)
# Set the logger
set_logger(os.path.join(args.model_dir, 'train.log'))
if not os.path.exists(args.restore_from):
os.makedirs(args.restore_from)
# Create the input data pipeline
logging.info('Creating the datasets...')
data_dir = args.data_dir
train_data_dir = os.path.join(data_dir, 'train')
valid_data_dir = os.path.join(data_dir, 'valid')
# Get the filenames and labels from the train and valid sets
train_filenames, train_labels = get_filenames_and_labels(
train_data_dir, params)
valid_filenames, valid_labels = get_filenames_and_labels(
valid_data_dir, params)
params.train_size = len(train_filenames)
params.valid_size = len(valid_filenames)
params.num_labels = len(set(train_labels))
# Create the two iterators over the two datasets
train_inputs = input_fn(True, train_filenames,
train_labels, params)
valid_inputs = input_fn(False, valid_filenames,
valid_labels, params)
# Define the model
logging.info('Creating the model...')
train_model_spec = model_fn('train', train_inputs,
params)
valid_model_spec = model_fn('eval', valid_inputs,
params, reuse=True)
# Train the model
logging.info('Starting training for {} epoch(s)'.format(
params.num_epochs))
train_and_evaluate(train_model_spec, valid_model_spec,
args.model_dir, params, args.restore_from)
def main():
# Load parameters from json file
args = parser.parse_args()
json_path = os.path.join(args.model_dir, 'params.json')
assert os.path.isfile(
json_path), "No json configuration file found at {}".format(json_path)
params = Params(json_path)
# use GPU if found
params.cuda = torch.cuda.is_available()
if params.cuda:
params.device = torch.device('cuda:0')
else:
params.device = torch.device('cpu')
# Set a seed for reproducible experiments
torch.manual_seed(141)
if params.cuda:
torch.cuda.manual_seed(141)
# Set the training logger for updates
set_logger(os.path.join(args.model_dir, 'train.log'))
logging.info("Creating input pipelines...")
data_pipelines = fetch_pipeline(['train', 'validation'], args.data_dir,
params)
train_pipeline = data_pipelines['train']
logging.info("Completed (Training Dataset)!")
valid_pipeline = data_pipelines['validation']
logging.info("Completed (Validation Dataset)!")
logging.info("Building network model...")
model_spec = model_fn(params)
logging.info("Building completed!")
logging.info("Initiate training procedure!")
train_and_validate(model_spec, train_pipeline, valid_pipeline,
args.model_dir, params, args.restore_from)
logging.info("Training completed!")
def funct(x):
# Set the random seed for the whole graph
tf.set_random_seed(230)
# Load the parameters
args = parser.parse_args()
json_path = os.path.join(args.model_dir, 'params.json')
assert os.path.isfile(json_path), "No json configuration file found at {}".format(json_path)
params = Params(json_path)
# Set the logger
set_logger(os.path.join(args.data_dir, 'predict.log'))
# Create the input data pipeline
data_dir = args.data_dir
test_data_dir = os.path.join(data_dir)
# Get the filenames from the test set
test_filenames = [os.path.join(test_data_dir, 'predict.jpg') ]
test_labels = [x]
# print(test_labels)
# specify the size of the evaluation set
params.eval_size = len(test_filenames)
# create the iterator over the dataset
test_inputs = input_fn(False, test_filenames, test_labels, params)
# Define the model
model_spec = model_fn('eval', test_inputs, params, reuse=tf.AUTO_REUSE)
evaluate(model_spec, args.model_dir, params, args.restore_from)
help="Directory containing params.json")
parser.add_argument('--data_dir',
default='data',
help="Directory containing the dataset")
parser.add_argument(
'--restore_from',
default='best_weights',
help="Subdirectory of model dir or file containing the weights")
if __name__ == '__main__':
# Load the parameters
args = parser.parse_args()
json_path = os.path.join(args.model_dir, 'params.json')
assert os.path.isfile(
json_path), "No json configuration file found at {}".format(json_path)
params = Params(json_path)
# Set the random seed for the whole graph
tf.set_random_seed(params.seed)
# Load the parameters from the dataset, that gives the size etc. into params
json_path = os.path.join(args.data_dir, 'dataset_params.json')
assert os.path.isfile(
json_path), "No json file found at {}, run build.py".format(json_path)
params.update(json_path)
num_oov_buckets = params.num_oov_buckets # number of buckets for unknown words
# Set the logger
set_logger(os.path.join(args.model_dir, 'evaluate.log'))
# Get paths for vocabularies and dataset
parser.add_argument('--restore_from', default='best_weights',
help="Subdirectory of model dir or file containing the weights")
parser.add_argument('--is_toy', default='False')
args = parser.parse_args()
if args.is_toy == 'False':
toy = ''
elif args.is_toy == 'True':
toy = '_small'
else:
raise ValueError("Please specify is_toy as either 'True' or 'False'")
# Load parameters of model
sentiment_model_path = os.path.join(args.model_dir, 'params.json')
params_sentiment = Params(sentiment_model_path)
# Load parameters from the dataset (sizes, etc) into params
# data_params_path = os.path.join(args.data_dir, 'dataset_params_small.json')
data_params_path = os.path.join(args.data_dir, 'dataset_params{}.json'.format(toy))
params_sentiment.update(data_params_path)
num_oov_buckets = params_sentiment.num_oov_buckets
# Update model params to include number of tags attribute
params_sentiment.number_of_tags = params_sentiment.number_of_sentiments
# Get paths for vocabularies and dataset
# path_words = os.path.join(args.data_dir, 'words_small.txt')
path_words = os.path.join(args.data_dir, 'words{}.txt'.format(toy))
path_sentiment_tags = os.path.join(args.data_dir, 'sentiment_tags.txt')
# path_reviews = os.path.join(args.data_dir, 'reviews_small.txt')
default='best_weights',
help="Subdirectory of model dir or file containing the weights")
parser.add_argument('--is_toy', default='False')
args = parser.parse_args()
if args.is_toy == 'False':
toy = ''
elif args.is_toy == 'True':
toy = '_small'
else:
raise ValueError("Please specify is_toy as either 'True' or 'False'")
# Load parameters of model
era_model_path = os.path.join(args.model_dir, 'params.json')
params_era = Params(era_model_path)
# Load parameters from the dataset (sizes, etc) into params
# data_params_path = os.path.join(args.data_dir, 'dataset_params_small.json')
data_params_path = os.path.join(args.data_dir,
'dataset_params{}.json'.format(toy))
params_era.update(data_params_path)
num_oov_buckets = params_era.num_oov_buckets
# Update model params to include number of tags attribute
params_era.number_of_tags = params_era.number_of_eras
# Get paths for vocabularies and dataset
# path_words = os.path.join(args.data_dir, 'words_small.txt')
path_words = os.path.join(args.data_dir, 'words{}.txt'.format(toy))
path_era_tags = os.path.join(args.data_dir, 'era_tags.txt')
parser = argparse.ArgumentParser()
parser.add_argument('--model_dir', default='experiments/test', help="Experiment directory containing params.json")
parser.add_argument('--data_dir', default='data/64x64_SIGNS', help="Directory containing the dataset")
parser.add_argument('--restore_from', default='best_weights', help="Subdirectory of model dir or file containing the weights")
if __name__ == '__main__':
# Get arguments
args = parser.parse_args()
# Set the random seed for the whole graph
tf.set_random_seed(230)
# Load the parameters
params = Params(os.path.join(args.model_dir, 'params.json'))
# Set the logger
set_logger(os.path.join(args.model_dir, 'evaluate.log'))
# Create the input data pipeline
logging.info("Creating the dataset...")
data_dir = args.data_dir
test_data_dir = os.path.join(data_dir, "test_signs")
# Get the filenames from the test set
test_filenames = os.listdir(test_data_dir)
test_filenames = [os.path.join(test_data_dir, f) for f in test_filenames if f.endswith('.jpg')]
test_labels = [int(f.split('/')[-1][0]) for f in test_filenames]
parser.add_argument('--restore_dir', default=None,
help="Optional, directory containing weights to reload before training")
if __name__ == '__main__':
args = parser.parse_args()
args.model_dir = os.path.join("experiments", args.data_dir, args.model_dir)
if args.restore_dir is not None:
args.restore_dir = os.path.join("experiments", args.data_dir, args.restore_dir)
args.data_dir = os.path.join("data", args.data_dir)
print(args)
if not os.path.isdir(args.model_dir):
os.makedirs(args.model_dir)
# Load the parameters from the experiment params.json file in model_dir
params = Params('params.json')
# Load the parameters from the dataset, that gives the size etc. into params
json_path = os.path.join(args.data_dir, 'dataset_params.json')
assert os.path.isfile(json_path), "No json file found at {}, run build_vocab.py".format(json_path)
params.update(json_path)
# Set the random seed for the whole graph for reproductible experiments
tf.set_random_seed(params.seed)
# Save the params in model_dir
json_path = os.path.join(args.model_dir, 'params.json')
params.save(json_path)
num_oov_buckets = params.num_oov_buckets # number of buckets for unknown words
# Check that we are not overwriting some previous experiment