def __init__(self, model):
self.model = model
self.reader = data_input.DataInput()
self.split = split.Split(self.reader)
self.evaluator = evaluator.Evaluator()
self.writer = output.Output()
self.model.setup(self.reader)
def main(unused_argv):
if FLAGS.task_name == 'svhn':
FLAGS.input_image_size = 32
FLAGS.small_image_model = True
FLAGS.num_label_classes = 10
if FLAGS.num_train_images is None:
FLAGS.num_train_images = task_info.get_num_train_images(
FLAGS.task_name)
if FLAGS.num_eval_images is None:
FLAGS.num_eval_images = task_info.get_num_eval_images(FLAGS.task_name)
if FLAGS.num_test_images is None and FLAGS.task_name != 'imagenet':
FLAGS.num_test_images = task_info.get_num_test_images(FLAGS.task_name)
steps_per_epoch = (FLAGS.num_train_images /
(FLAGS.train_batch_size * FLAGS.label_data_sample_prob))
if FLAGS.mode == 'train' or FLAGS.mode == 'train_and_eval':
tf.gfile.MakeDirs(FLAGS.model_dir)
flags_dict = tf.app.flags.FLAGS.flag_values_dict()
with tf.gfile.Open(os.path.join(FLAGS.model_dir, 'FLAGS.json'),
'w') as ouf:
json.dump(flags_dict, ouf)
input_image_size = FLAGS.input_image_size
if not input_image_size:
_, _, input_image_size, _ = efficientnet_builder.efficientnet_params(
FLAGS.model_name)
FLAGS.input_image_size = input_image_size
if FLAGS.train_last_step_num == -1:
FLAGS.train_last_step_num = FLAGS.train_steps
if FLAGS.train_ratio != 1:
FLAGS.train_last_step_num *= FLAGS.train_ratio
FLAGS.train_steps *= FLAGS.train_ratio
FLAGS.train_last_step_num = int(FLAGS.train_last_step_num)
FLAGS.train_steps = int(FLAGS.train_steps)
if (FLAGS.tpu or FLAGS.use_tpu) and not FLAGS.master:
tpu_cluster_resolver = tf.distribute.cluster_resolver.TPUClusterResolver(
FLAGS.tpu, zone=FLAGS.tpu_zone, project=FLAGS.gcp_project)
else:
tpu_cluster_resolver = None
if FLAGS.use_tpu:
tpu_config = tf.estimator.tpu.TPUConfig(
iterations_per_loop=FLAGS.iterations_per_loop,
num_shards=FLAGS.num_tpu_cores,
per_host_input_for_training=tf.estimator.tpu.InputPipelineConfig.
PER_HOST_V2)
else:
tpu_config = tf.estimator.tpu.TPUConfig(
iterations_per_loop=FLAGS.iterations_per_loop,
num_shards=FLAGS.num_tpu_cores,
per_host_input_for_training=tf.estimator.tpu.InputPipelineConfig.
PER_HOST_V2)
config = tf.estimator.tpu.RunConfig(
cluster=tpu_cluster_resolver,
master=FLAGS.master,
model_dir=FLAGS.model_dir,
save_checkpoints_steps=max(FLAGS.save_checkpoints_steps, FLAGS.iterations_per_loop),
log_step_count_steps=FLAGS.log_step_count_steps,
keep_checkpoint_max=FLAGS.keep_checkpoint_max,
session_config=tf.ConfigProto(
graph_options=tf.GraphOptions(
rewrite_options=rewriter_config_pb2.RewriterConfig(
disable_meta_optimizer=True))),
tpu_config=tpu_config) # pylint: disable=line-too-long
# Initializes model parameters.
params = dict(steps_per_epoch=steps_per_epoch,
use_bfloat16=FLAGS.use_bfloat16)
est = tf.estimator.tpu.TPUEstimator(
use_tpu=FLAGS.use_tpu,
model_fn=model_fn,
config=config,
train_batch_size=FLAGS.train_batch_size,
eval_batch_size=FLAGS.eval_batch_size,
predict_batch_size=8,
params=params)
# Input pipelines are slightly different (with regards to shuffling and
# preprocessing) between training and evaluation.
if FLAGS.label_data_dir == FAKE_DATA_DIR:
tf.logging.info('Using fake dataset.')
else:
tf.logging.info('Using dataset: %s', FLAGS.label_data_dir)
train_data = data_input.DataInput(is_training=True,
data_dir=FLAGS.label_data_dir,
transpose_input=FLAGS.transpose_input,
cache=FLAGS.use_cache,
image_size=input_image_size,
use_bfloat16=FLAGS.use_bfloat16)
if FLAGS.mode == 'train' or FLAGS.mode == 'train_and_eval':
current_step = estimator._load_global_step_from_checkpoint_dir(
FLAGS.model_dir) # pylint: disable=protected-access,line-too-long
tf.logging.info(
'Training for %d steps (%.2f epochs in total). Current'
' step %d.', FLAGS.train_last_step_num,
FLAGS.train_last_step_num / params['steps_per_epoch'],
current_step)
start_timestamp = time.time(
) # This time will include compilation time
if FLAGS.mode == 'train':
est.train(input_fn=train_data.input_fn,
max_steps=FLAGS.train_last_step_num,
hooks=[])
elif FLAGS.mode == 'eval':
input_fn_mapping = {}
for subset in ['dev', 'test']:
input_fn_mapping[subset] = data_input.DataInput(
is_training=False,
data_dir=FLAGS.label_data_dir,
transpose_input=FLAGS.transpose_input,
cache=False,
image_size=input_image_size,
use_bfloat16=FLAGS.use_bfloat16,
subset=subset).input_fn
if subset == 'dev':
num_images = FLAGS.num_eval_images
else:
num_images = FLAGS.num_test_images
eval_results = est.evaluate(input_fn=input_fn_mapping[subset],
steps=num_images //
FLAGS.eval_batch_size)
tf.logging.info('%s, results: %s', subset, eval_results)
elif FLAGS.mode == 'predict':
predict_label.run_prediction(est)
else:
assert False
#test_location = "Boulder" #Folder name
#test_location = "Desert_Rock" #Folder name
#test_location = "Fort_Peck" #Folder name
#test_location = "Goodwin_Creek" #Folder name
#test_location = "Penn_State" #Folder name
#test_location = "Sioux_Falls" #Folder name
# bird_sky_model.py's output here
clear_sky_ghi = bird_sky_model.ClearSky(test_location, test_year, run_train)
cs_test, cs_2010and2011 = clear_sky_ghi.cs_ghi()
print("bird_sky_model.py module executed successfully")
# data_input.py's output here
input_data = data_input.DataInput(test_location, test_year, run_train, cs_test, cs_2010and2011)
df_train, df_test = input_data.load_n_merge()
print("data_input.py module executed successfully")
# exploratory data analysis
eda_plots = exploratory_data_analysis.EDA(df_test)
plots = eda_plots.ghi_plot()
print("exploratory_data_analysis.py module executed successfully")
# cleaning the data - removing the outliers
df = data_cleaning.CleanData(df_train, df_test, run_train)
df_train, df_test = df.clean()
print("data_cleaning.py module executed successfully")
### start of LSTM
def main():