def prepare_dataloader(
dataframe: pd.DataFrame,
target_datetimes: typing.List[datetime.datetime],
stations: typing.Dict[typing.AnyStr, typing.Tuple[float, float, float]],
target_time_offsets: typing.List[datetime.timedelta],
config: typing.Dict[typing.AnyStr, typing.Any],
) -> tf.data.Dataset:
"""This function should be modified in order to prepare & return your own data loader.
Note that you can use either the netCDF or HDF5 data. Each iteration over your data loader should return a
2-element tuple containing the tensor that should be provided to the model as input, and the target values. In
this specific case, you will not be able to provide the latter since the dataframe contains no GHI, and we are
only interested in predictions, not training. Therefore, you must return a placeholder (or ``None``) as the second
tuple element.
Reminder: the dataframe contains imagery paths for every possible timestamp requested in ``target_datetimes``.
However, we expect that you will use some of the "past" imagery (i.e. imagery at T<=0) for any T in
``target_datetimes``, but you should NEVER rely on "future" imagery to generate predictions (for T>0). We
will be inspecting data loader implementations to ensure this is the case, and those who "cheat" will be
dramatically penalized.
See https://github.com/mila-iqia/ift6759/tree/master/projects/project1/evaluation.md for more information.
Args:
dataframe: a pandas dataframe that provides the netCDF file path (or HDF5 file path and offset) for all
relevant timestamp values over the test period.
target_datetimes: a list of timestamps that your data loader should use to provide imagery for your model.
The ordering of this list is important, as each element corresponds to a sequence of GHI values
to predict. By definition, the GHI values must be provided for the offsets given by ``target_time_offsets``
which are added to each timestamp (T=0) in this datetimes list.
stations: a map of station names of interest paired with their coordinates (latitude, longitude, elevation).
target_time_offsets: the list of timedeltas to predict GHIs for (by definition: [T=0, T+1h, T+3h, T+6h]).
config: configuration dictionary holding any extra parameters that might be required by the user. These
parameters are loaded automatically if the user provided a JSON file in their submission. Submitting
such a JSON file is completely optional, and this argument can be ignored if not needed.
Returns:
A ``tf.data.Dataset`` object that can be used to produce input tensors for your model. One tensor
must correspond to one sequence of past imagery data. The tensors must be generated in the order given
by ``target_sequences``.
"""
################################## MODIFY BELOW ##################################
# WE ARE PROVIDING YOU WITH A DUMMY DATA GENERATOR FOR DEMONSTRATION PURPOSES.
# MODIFY EVERYTHINGIN IN THIS BLOCK AS YOU SEE FIT
from libs import helpers
helpers.validate_user_config(config)
data_loader = helpers.get_online_data_loader(
user_config_dict=config,
dataframe=dataframe,
target_datetimes=target_datetimes,
stations=stations,
target_time_offsets=target_time_offsets,
preprocessed_data_path=config['data_loader']['hyper_params']
['preprocessed_data_source']['test'])
################################### MODIFY ABOVE ##################################
return data_loader
def main(config_path: typing.AnyStr,
tensorboard_tracking_folder: typing.AnyStr):
"""
Train a model
:param config_path: path to the JSON config file that follows configs/user/schema.json
:param tensorboard_tracking_folder: path where to store TensorBoard data and save trained model
"""
user_config_dict = helpers.load_dict(config_path)
helpers.validate_user_config(user_config_dict)
if tensorboard_tracking_folder is not None:
tensorboard_tracking_folder = Path(tensorboard_tracking_folder)
tensorboard_tracking_folder.mkdir(parents=True, exist_ok=True)
train_models(config=user_config_dict,
tensorboard_tracking_folder=tensorboard_tracking_folder)
def main(training_config_path: typing.AnyStr,
validation_config_path: typing.AnyStr,
user_config_path: typing.AnyStr,
tensorboard_tracking_folder: typing.AnyStr):
"""
Train a model
:param training_config_path: path to the JSON config file used to store training set parameters
:param validation_config_path: path to the JSON config file used to store validation set parameters
:param user_config_path: path to the JSON config file used to store user model, dataloader and trainer parameters
:param tensorboard_tracking_folder: path where to store TensorBoard data and save trained model
"""
training_config_dict = helpers.load_dict(training_config_path)
validation_config_dict = helpers.load_dict(validation_config_path)
user_config_dict = helpers.load_dict(user_config_path)
helpers.validate_admin_config(training_config_dict)
helpers.validate_admin_config(validation_config_dict)
helpers.validate_user_config(user_config_dict)
training_source = user_config_dict['data_loader']['hyper_params'][
'preprocessed_data_source']['training']
validation_source = user_config_dict['data_loader']['hyper_params'][
'preprocessed_data_source']['validation']
training_data_loader = helpers.get_online_data_loader(
user_config_dict,
training_config_dict,
preprocessed_data_path=training_source)
validation_data_loader = helpers.get_online_data_loader(
user_config_dict,
validation_config_dict,
preprocessed_data_path=validation_source)
print("Eager mode", tf.executing_eagerly())
mirrored_strategy = helpers.get_mirrored_strategy()
train_models(user_config_dict=user_config_dict,
training_config_dict=training_config_dict,
training_data_loader=training_data_loader,
validation_data_loader=validation_data_loader,
tensorboard_tracking_folder=tensorboard_tracking_folder,
mirrored_strategy=mirrored_strategy)
def generate_all_predictions(
target_stations: typing.Dict[typing.AnyStr, typing.Tuple[float, float,
float]],
target_datetimes: typing.List[datetime.datetime],
target_time_offsets: typing.List[datetime.timedelta],
dataframe: pd.DataFrame,
user_config: typing.Dict[typing.AnyStr, typing.Any],
) -> np.ndarray:
"""Generates and returns model predictions g<iven the data prepared by a data loader."""
# we will create one data loader per station to make sure we avoid mixups in predictions
predictions = []
################################### TEAM 3's REQUIRED EDIT ##################################
# Justification:
# We need to start by pre processing all stations to be able to satisfy the 30 minute evaluation restriction
from tools.netcdf_crop import netcdf_preloader
from libs import helpers
helpers.validate_user_config(user_config)
if user_config['data_loader']['hyper_params']['should_preprocess_data']:
netcdf_preloader(dataframe=dataframe,
target_datetimes=target_datetimes,
stations=target_stations,
path_output=user_config['data_loader']['hyper_params']
['preprocessed_data_source']['test'])
################################### TEAM 3's REQUIRED EDIT ##################################
for station_idx, station_name in enumerate(target_stations):
# usually, we would create a single data loader for all stations, but we just want to avoid trouble...
stations = {station_name: target_stations[station_name]}
print(
f"preparing data loader & model for station '{station_name}' ({station_idx + 1}/{len(target_stations)})"
)
data_loader = prepare_dataloader(dataframe, target_datetimes, stations,
target_time_offsets, user_config)
model = prepare_model(stations, target_time_offsets, user_config)
station_preds = generate_predictions(data_loader,
model,
pred_count=len(target_datetimes))
assert len(station_preds) == len(
target_datetimes
), "number of predictions mismatch with requested datetimes"
predictions.append(station_preds)
return np.concatenate(predictions, axis=0)
def prepare_model(
stations: typing.Dict[typing.AnyStr, typing.Tuple[float, float, float]],
target_time_offsets: typing.List[datetime.timedelta],
config: typing.Dict[typing.AnyStr, typing.Any],
) -> tf.keras.Model:
"""This function should be modified in order to prepare & return your own prediction model.
See https://github.com/mila-iqia/ift6759/tree/master/projects/project1/evaluation.md for more information.
Args:
stations: a map of station names of interest paired with their coordinates (latitude, longitude, elevation).
target_time_offsets: the list of timedeltas to predict GHIs for (by definition: [T=0, T+1h, T+3h, T+6h]).
config: configuration dictionary holding any extra parameters that might be required by the user. These
parameters are loaded automatically if the user provided a JSON file in their submission. Submitting
such a JSON file is completely optional, and this argument can be ignored if not needed.
Returns:
A ``tf.keras.Model`` object that can be used to generate new GHI predictions given imagery tensors.
"""
################################### MODIFY BELOW ##################################
from libs import helpers
helpers.validate_user_config(config)
mirrored_strategy = helpers.get_mirrored_strategy()
if mirrored_strategy is not None and mirrored_strategy.num_replicas_in_sync > 1:
with mirrored_strategy.scope():
model = helpers.prepare_model(
user_config_dict=config,
stations=stations,
target_time_offsets=target_time_offsets)
else:
model = helpers.prepare_model(user_config_dict=config,
stations=stations,
target_time_offsets=target_time_offsets)
################################### MODIFY ABOVE ##################################
return model
def generate_predictions(input_file_path: str, pred_file_path: str):
"""Generates predictions for the machine translation task (EN->FR).
You are allowed to modify this function as needed, but one again, you cannot
modify any other part of this file. We will be importing only this function
in our final evaluation script. Since you will most definitely need to import
modules for your code, you must import these inside the function itself.
Args:
input_file_path: the file path that contains the input data.
pred_file_path: the file path where to store the predictions.
Returns: None
"""
##### MODIFY BELOW #####
import tensorflow as tf
from libs import helpers
from libs.data_loaders.abstract_dataloader import AbstractDataloader
from libs.models import transformer
import tqdm
import logging
from libs.data_loaders.abstract_dataloader import create_masks_fm
from libs.data_loaders.dataloader_bilingual_huggingface import BilingualTranslationHFSubword
from libs.data_loaders.dataloader_bilingual_tensorflow import BilingualTranslationTFSubword
from libs.data_loaders.mass_subword import MassSubwordDataLoader
from libs.models.transformer import Encoder, Decoder
logger = tf.get_logger()
logger.setLevel(logging.DEBUG)
import numpy as np
import random
from libs.seeds import TENSOR_FLOW_SEED, NUMPY_SEED, RANDOM_SEED
tf.random.set_seed(TENSOR_FLOW_SEED)
np.random.seed(NUMPY_SEED)
random.seed(RANDOM_SEED)
best_config_file = '/project/cq-training-1/project2/teams/team03/models/transformer_mass_v1_translation_with_pretraining_eval.json'
# best_config_file = 'configs/user/transformers-fm/TFM_TINY_BBPE_eval.json'
logger.info(f"Using best config file: {best_config_file}")
best_config = helpers.load_dict(best_config_file)
helpers.validate_user_config(best_config)
# TODO: Edit our AbstractDataloader to support a raw_english_test_set_file_path. Currently it only supports
# preprocessed data defined directly in best_config.
data_loader: AbstractDataloader = helpers.get_online_data_loader(
config=best_config, raw_english_test_set_file_path=input_file_path)
if best_config["model"]["definition"][
"module"] == 'libs.models.transformerv2':
model = transformer.load_transformer(best_config)
else:
mirrored_strategy = helpers.get_mirrored_strategy()
if mirrored_strategy is not None and mirrored_strategy.num_replicas_in_sync > 1:
with mirrored_strategy.scope():
model: tf.keras.Model = helpers.prepare_model(
config=best_config)
else:
model: tf.keras.Model = helpers.prepare_model(config=best_config)
# batch_size = 32 # 32 is max for 6GB GPU memory
batch_size = 128
data_loader.build(batch_size=batch_size)
test_dataset = data_loader.test_dataset
all_predictions = []
if isinstance(data_loader, MassSubwordDataLoader):
all_predictions = transformer.inference(data_loader.tokenizer, model,
test_dataset)
else:
if isinstance(data_loader, BilingualTranslationTFSubword) or \
isinstance(data_loader, BilingualTranslationHFSubword):
sample_to_display = 10
encoder: Encoder = model.get_layer("encoder")
decoder: Decoder = model.get_layer("decoder")
final_layer: tf.keras.layers.Dense = model.layers[-1]
for inputs, mask in tqdm.tqdm(test_dataset,
total=data_loader.test_steps):
mini_batch_size = inputs.shape[0]
dec_inp = tf.Variable(
tf.zeros(
(mini_batch_size, data_loader.get_seq_length() + 1),
dtype=tf.int32))
bos_tensor = tf.convert_to_tensor(data_loader.bos)
bos_tensor = tf.reshape(bos_tensor, [1, 1])
bos_tensor = tf.tile(bos_tensor,
multiples=[mini_batch_size, 1])
dec_inp[:, 0].assign(bos_tensor[:, 0]) # BOS token
# WARNING: IF THE MODEL USED WAS FROM A TF FILE, A LOT OF WARNINGS WILL APPEAR
# Workaround: Use the hdf5 format to load the final model
# https://github.com/tensorflow/tensorflow/issues/35146
def get_preds(encoder, decoder, final_layer, dec_inp, inputs,
mask, max_seq):
enc_output: tf.Tensor = encoder.__call__(inputs=inputs,
mask=mask,
training=False)
for timestep in range(max_seq):
_, combined_mask, dec_padding_mask = create_masks_fm(
inp=inputs, tar=dec_inp[:, :-1])
dec_output, attention_weights = decoder(
inputs=dec_inp[:, :-1],
enc_output=enc_output,
look_ahead_mask=combined_mask,
padding_mask=dec_padding_mask)
outputs = final_layer(
inputs=dec_output
) # (batch_size, seq_length, vocab_size)
pred = tf.argmax(outputs[:, timestep, :], axis=-1)
pred = tf.cast(pred, dtype=tf.int32)
dec_inp[:, timestep + 1].assign(pred)
return dec_inp
predictions = get_preds(
encoder=encoder,
decoder=decoder,
final_layer=final_layer,
dec_inp=dec_inp,
inputs=inputs,
mask=mask,
# TODO Decision to be made, 100 seq length doesn't seem to hurt perfs
max_seq=100) # data_loader.get_seq_length())
for prediction in predictions.numpy():
if sample_to_display > 0:
logger.info(
f"Example of generated translation: {data_loader.decode(prediction)}"
)
sample_to_display -= 1
all_predictions += [data_loader.decode(prediction)]
else:
raise NotImplementedError(
f"No method to generate for class {data_loader.__class__.__name__}"
)
with open(pred_file_path, 'w+') as file_handler:
for prediction in all_predictions:
file_handler.write(f'{prediction}\n')