def main(params):
# setup random seeds
set_seed(params.seed)
params.ar = True
if not os.path.isdir(params.output_dir): os.makedirs(params.output_dir)
print("Loading the model from {0}".format(params.model_path))
# load everything from checkpoint
model_params, dico, model = reload_ar_checkpoint(params.model_path)
if params.local_cpu is False:
model = model.cuda()
# evaluate distributional results
generator = SmilesTransformerGenerator(params, dico, model,
params.sample_temperature)
json_file_path = os.path.join(params.output_dir,
'distribution_learning_results.json')
smiles_output_path = os.path.join(params.output_dir,
'generated_smiles.txt')
print("Starting distributional evaluation")
t1 = time.time()
if params.evaluate is True:
assess_distribution_learning(generator,
chembl_training_file=params.dist_file,
json_output_file=json_file_path,
benchmark_version=params.suite)
else:
smiles_list = generator.generate(params.num_samples)
with open(smiles_output_path, 'w') as f:
for smiles in smiles_list:
f.write(smiles + '\n')
t2 = time.time()
print("Total time taken {}".format(t2 - t1))
def cross_val(self, seed: int) -> None:
r"""
When we have a very small test set, we use will
train the model multiple times using
different train / test distributions. This will
allow us to understand the "true" performance of the
model, reducing the noisiness of having a tiny test set
"""
set_seed(seed)
pos_train, pos_test = self.split_train_test(self.positive_paths,
self.num_test)
neg_train, neg_test = self.split_train_test(self.negative_paths,
self.num_test)
self.subset_to_tasks = {
"testing":
KenyaCropTypeFromPaths(
negative_paths=neg_test,
positive_paths=pos_test,
cache=True,
),
"training":
KenyaCropTypeFromPaths(
negative_paths=neg_train,
positive_paths=pos_train,
cache=self.cache,
),
}
def main():
config = utils.load_config(args.config)
config["weighted"] = "weighted" in config.keys()
# copy args to config
config["mode"] = args.mode
config["fold"] = args.fold
config["apex"] = args.apex
config["output"] = args.output
config["snapshot"] = args.snapshot
# config["resume_from"] = args.resume_from if args.resume_from
utils.set_seed(SEED)
device = torch.device(DEVICE)
log(f"Fold {args.fold}")
model = factory.get_model(config).to(device)
log(f"Model type: {model.__class__.__name__}")
if config["mode"] == 'train':
train(config, model)
valid(config, model)
elif config["mode"] == 'valid':
valid(config, model)
elif config["mode"] == 'test':
valid(config, model, all_exam=True)
def main():
parser = ArgumentParser()
parser.add_argument('--config_file', type=str, required=True)
args = parser.parse_args()
# settings
config_file = Path(args.config_file)
config = Config.load(config_file)
output_dir = Path(f'./output/{config_file.stem}/')
output_dir.mkdir(parents=True, exist_ok=True)
set_seed(config.seed)
logger = getLogger(__name__)
setup_logger(logger, output_dir / 'log.txt')
# load data
data_bundle = build_data_module(config)
# train
runner = build_runner(config)
runner.run(data_bundle)
# save file
runner.save(output_dir)
def __init__(self, hparams: Namespace) -> None:
super().__init__()
set_seed()
self.hparams = hparams
self.data_folder = Path(hparams.data_folder)
dataset = self.get_dataset(subset="training", cache=False)
self.num_outputs = dataset.num_output_classes
self.num_timesteps = dataset.num_timesteps
self.input_size = dataset.num_input_features
# we save the normalizing dict because we calculate weighted
# normalization values based on the datasets we combine.
# The number of instances per dataset (and therefore the weights) can
# vary between the train / test / val sets - this ensures the normalizing
# dict stays constant between them
self.normalizing_dict = dataset.normalizing_dict
if self.hparams.forecast:
num_output_timesteps = self.num_timesteps - self.hparams.input_months
print(f"Predicting {num_output_timesteps} timesteps in the forecaster")
self.forecaster = Forecaster(
num_bands=self.input_size, output_timesteps=num_output_timesteps, hparams=hparams,
)
self.forecaster_loss = F.smooth_l1_loss
self.classifier = Classifier(input_size=self.input_size, hparams=hparams)
self.global_loss_function: Callable = F.binary_cross_entropy
self.local_loss_function: Callable = F.binary_cross_entropy
def __init__(self, hparams: Namespace) -> None:
super().__init__()
set_seed()
self.hparams = hparams
self.data_folder = Path(hparams.data_folder)
self.version_folder = (self.data_folder / "maml_models" /
f"version_{self.hparams.model_version}")
# load the model info
with (self.version_folder / "model_info.json").open("r") as f:
self.model_info = json.load(f)
self.classifier = Classifier(
input_size=self.model_info["input_size"],
classifier_vector_size=self.model_info["classifier_vector_size"],
classifier_dropout=self.model_info["classifier_dropout"],
classifier_base_layers=self.model_info["classifier_base_layers"],
num_classification_layers=self.
model_info["num_classification_layers"],
)
self.best_val_loss: float = np.inf
self.loss = nn.BCELoss()
self.normalizing_dict = self.get_dataset(
subset="training").normalizing_dict
def cross_val(self, seed: int) -> None:
set_seed(seed)
# we shuffle the training indices, so that when sample_train is called
# (which is deterministic) it will return different samples
random.shuffle(self.subset_to_tasks["training"].positive_indices)
random.shuffle(self.subset_to_tasks["training"].negative_indices)
def create_training_env(self):
if(self.parallel_size == 1):
self.env = SingleEnv(self.get_new_env())
else:
self.env = SubprocVecEnv([self.get_new_env for _ in range(self.parallel_size)])
if(self.seed):
set_seed(self.seed)
def __init__(self, data_folder: Path) -> None:
set_seed()
self.data_folder = data_folder
self.raw_folder = self.data_folder / "raw" / self.dataset
assert self.raw_folder.exists(), f"{self.raw_folder} does not exist!"
self.output_folder = self.data_folder / "processed" / self.dataset
self.output_folder.mkdir(exist_ok=True, parents=True)
def main():
"""主函数"""
paddle.set_device("gpu:1")
set_seed(2021) # 设置随机数种子
logger.info("构建数据集")
data_file = os.path.join(work_root, "data/NewsTrain.txt")
datasets, labels = load_data(data_file)
save_dict_obj(labels, os.path.join(work_root,
"data/news_labels_info.json"))
for i in range(3):
random.shuffle(datasets)
train_data_num = int(len(datasets) * 0.8)
train_dataset, valid_dataset = datasets[:train_data_num], datasets[
train_data_num:]
train_dataset, valid_dataset = MapDataset(train_dataset), MapDataset(
valid_dataset)
logger.info("数据转换word2id")
tokenizer = paddlenlp.transformers.ErnieTinyTokenizer.from_pretrained(
'ernie-tiny')
trans_func = partial(convert_example,
tokenizer=tokenizer,
max_seq_length=64)
batchify_fn = lambda samples, fn=Tuple(
Pad(axis=0, pad_val=tokenizer.pad_token_id), # input
Pad(axis=0, pad_val=tokenizer.pad_token_type_id), # segment
Stack(dtype="int64") # label
): fn(samples) # 有点难以理解, 没有Torch便于理解, pad_val非常好, 可以动态设置batch最大序列长度
train_loader = create_dataloader(train_dataset,
mode="train",
batch_size=512,
batchify_fn=batchify_fn,
trans_fn=trans_func)
valid_loader = create_dataloader(valid_dataset,
mode="valid",
batch_size=256,
batchify_fn=batchify_fn,
trans_fn=trans_func)
epochs = 5 # 训练epoch number
num_training_steps = len(train_loader) * epochs
num_classes = len(labels)
model, optimizer, criterion, lr_scheduler = create_classification_model(
num_classes, num_training_steps)
logger.info("训练模型")
metric = paddle.metric.Accuracy()
train(model,
optimizer,
criterion,
lr_scheduler,
metric,
tokenizer,
train_loader,
valid_loader,
epochs=epochs)
def __init__(self, hparams: Namespace) -> None:
super().__init__()
set_seed()
self.hparams = hparams
self.data_folder = Path(hparams.data_folder)
dataset = self.get_dataset(subset="training")
self.input_size = dataset.num_input_features
self.num_outputs = dataset.num_output_classes
# we save the normalizing dict because we calculate weighted
# normalization values based on the datasets we combine.
# The number of instances per dataset (and therefore the weights) can
# vary between the train / test / val sets - this ensures the normalizing
# dict stays constant between them
self.normalizing_dict = dataset.normalizing_dict
self.model_base_name = hparams.model_base
self.base = STR2BASE[hparams.model_base](input_size=self.input_size,
hparams=self.hparams)
global_classification_layers: List[nn.Module] = []
for i in range(hparams.num_classification_layers):
global_classification_layers.append(
nn.Linear(
in_features=hparams.hidden_vector_size,
out_features=self.num_outputs
if i == (hparams.num_classification_layers -
1) else hparams.hidden_vector_size,
))
if i < (hparams.num_classification_layers - 1):
global_classification_layers.append(nn.ReLU())
self.global_classifier = nn.Sequential(*global_classification_layers)
if self.hparams.multi_headed:
local_classification_layers: List[nn.Module] = []
for i in range(hparams.num_classification_layers):
local_classification_layers.append(
nn.Linear(
in_features=hparams.hidden_vector_size,
out_features=self.num_outputs
if i == (hparams.num_classification_layers -
1) else hparams.hidden_vector_size,
))
if i < (hparams.num_classification_layers - 1):
local_classification_layers.append(nn.ReLU())
self.local_classifier = nn.Sequential(*local_classification_layers)
self.loss_function: Callable = F.binary_cross_entropy
def __init__(self, data_folder: Path) -> None:
set_seed()
self.data_folder = data_folder
self.geospatial_files = self.get_geospatial_files(data_folder)
self.labels = self.read_labels(data_folder)
self.savedir = self.data_folder / "features" / self.dataset
self.savedir.mkdir(exist_ok=True, parents=True)
self.normalizing_dict_interim: Dict[str, Union[np.ndarray, int]] = {
"n": 0
}
def load_cfg() -> Tuple[Config, str]:
from src.dict2obj import Config
from src.base import Adversary
from src.utils import gpu, load, set_seed
cfg = Config()
set_seed(opts.seed)
# load the model
model = load_model(opts.model)(num_classes=get_num_classes(opts.dataset))
device = gpu(model)
load(model=model, path=opts.info_path, device=device)
# load the testset
testset = load_dataset(dataset_type=opts.dataset,
transform=opts.transform,
train=False)
cfg['testloader'] = load_dataloader(dataset=testset,
batch_size=opts.batch_size,
train=False,
show_progress=opts.progress)
normalizer = load_normalizer(dataset_type=opts.dataset)
# generate the log path
_, log_path = generate_path(method=METHOD,
dataset_type=opts.dataset,
model=opts.model,
description=opts.description)
# set the attacker
attack, bounds, preprocessing = load_attacks(attack_type=opts.attack,
dataset_type=opts.dataset,
stepsize=opts.stepsize,
steps=opts.steps)
epsilons = torch.linspace(opts.epsilon_min, opts.epsilon_max,
opts.epsilon_times).tolist()
cfg['attacker'] = Adversary(model=model,
attacker=attack,
device=device,
bounds=bounds,
preprocessing=preprocessing,
epsilon=epsilons)
return cfg, log_path
def run(cfg: DictConfig) -> Dict:
"""
Model's inference on hold-out dataset
----------
Example:
python predict.py
:param cfg: general hydra config
:return: dict: test metrics
"""
hparams = flatten_omegaconf(cfg)
exp = Experiment(
api_key=cfg.logger.comet_api,
project_name=cfg.general.project_name,
workspace=cfg.general.workspace,
)
exp.log_parameters(hparams)
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
bert_model = load_obj(cfg.model.class_name).from_pretrained(cfg)
bert_model.model.load_state_dict(
torch.load(f"../../{cfg.inference.save_dir}/pytorch_model.bin"))
# print(bert_model)
bert_model.to(device)
bert_model.eval()
collator = load_obj(cfg.dataset.collator)(
percentile=cfg.data_params.percentile,
pad_value=cfg.data_params.pad_value,
)
criterion = nn.CrossEntropyLoss()
test = get_test_dataset(cfg)
evaluator = Evaluator(val_dataset=test,
collator=collator,
criterion=criterion,
cfg=cfg,
device=device)
set_seed(cfg.trainer.seed)
test_metrics = evaluator(model=bert_model, experiment=exp, epoch=0, step=0)
return test_metrics
def load_cfg() -> Tuple[Config, str]:
from src.dict2obj import Config
from src.utils import gpu, load, set_seed
cfg = Config()
set_seed(opts.seed)
# load the model
model = load_model(opts.model)(num_classes=get_num_classes(opts.dataset))
device = gpu(model)
load(model=model, path=opts.info_path, device=device)
model.eval()
# load the testset
testset = load_dataset(dataset_type=opts.dataset,
transform='None',
train=False)
data = []
targets = []
for i in range(len(testset)):
img, label = testset[i]
data.append(img)
targets.append(label)
cfg['data'] = torch.stack(data)
cfg['targets'] = torch.tensor(targets, dtype=torch.long)
normalizer = load_normalizer(opts.dataset)
# generate the log path
_, log_path = generate_path(METHOD, opts.dataset, opts.model,
opts.description)
cfg['attacker'] = AutoAttack(Defense(model, normalizer),
norm=opts.norm,
eps=opts.epsilon,
version=opts.version,
device=device)
return cfg, log_path
def main(params):
# setup random seeds
set_seed(params.seed)
params.ar = True
print("Loading the model from {0}".format(params.model_path))
# load everything from checkpoint
model_params, dico, model = reload_ar_checkpoint(params.model_path)
if params.local_cpu is False:
model = model.cuda()
# evaluate distributional results
generator = SmilesTransformerGenerator(params, dico, model)
json_file_path = os.path.join('/'.join(params.model_path.split('/')[:-1]),
'distribution_learning_results.json')
print("Starting distributional evaluation")
t1 = time.time()
assess_distribution_learning(generator,
chembl_training_file=params.dist_file,
json_output_file=json_file_path,
benchmark_version=params.suite)
t2 = time.time()
print("Total time taken {}".format(t2 - t1))
def run(cfg: DictConfig) -> None:
"""
Initialize model training
Use to start training cycle
:param cfg: general hydra config
:return: None
----------
Example:
python train.py
"""
hparams = flatten_omegaconf(cfg)
exp = Experiment(
api_key=cfg.logger.comet_api,
project_name=cfg.general.project_name,
workspace=cfg.general.workspace,
)
exp.log_parameters(hparams)
trainer = Trainer(cfg)
set_seed(cfg.trainer.seed)
trainer.train(exp)
return None
def main():
"""主函数"""
paddle.set_device("gpu:1")
set_seed(2021)
logger.info("构建数据集")
dataset = MovieDataset(use_poster=False)
vocab_sizes = {
"max_usr_id": int(dataset.max_usr_id),
"max_usr_age": int(dataset.max_usr_age),
"max_usr_job": int(dataset.max_usr_job),
"max_mov_id": int(dataset.max_mov_id),
"movie_category_size": len(dataset.movie_cat),
"movie_title_size": len(dataset.movie_title)
}
save_dict_obj(vocab_sizes,
os.path.join(work_root, "data/movie_vocab_size.json"))
train_data_loader = dataset.load_data(dataset.train_dataset,
mode="train",
use_poster=False)
eval_data_loader = dataset.load_data(dataset.valid_dataset,
mode="eval",
use_poster=False)
logger.info("构建模型")
model, opt = create_model_opt(vocab_sizes)
logger.info("训练模型")
EPOCHS = 10
train(model,
opt,
data_loader=train_data_loader,
eval_data_loader=eval_data_loader,
epochs=EPOCHS)
logger.info("构建特征信息")
get_usr_mov_features(model, dataset.dataset)
def main():
parser = ArgumentParser()
parser.add_argument('--config_file', type=str, required=True)
args = parser.parse_args()
# settings
config_path = Path(args.config_file)
config = Config.load(config_path)
warnings.filterwarnings('ignore')
set_seed(config.seed)
start_time = time.time()
with timer('load data'):
DATA_DIR = './input/riiid-test-answer-prediction/'
usecols = [
'row_id',
'timestamp',
'user_id',
'content_id',
'content_type_id',
'answered_correctly',
'prior_question_elapsed_time',
]
dtype = {
'row_id': 'int64',
'timestamp': 'int64',
'user_id': 'int32',
'content_id': 'int16',
'content_type_id': 'int8',
'answered_correctly': 'int8',
'prior_question_elapsed_time': 'float32'
}
train_df = pd.read_csv(DATA_DIR + 'train.csv',
usecols=usecols,
dtype=dtype)
question_df = pd.read_csv(DATA_DIR + 'questions.csv',
usecols=['question_id', 'part'])
train_df = train_df[train_df['content_type_id'] == 0].reset_index(
drop=True)
question_df['part'] += 1 # 0: padding id, 1: start id
train_df['content_id'] += 2 # 0: padding id, 1: start id
question_df['question_id'] += 2
train_df = train_df.merge(question_df,
how='left',
left_on='content_id',
right_on='question_id')
with timer('validation split'):
train_idx, valid_idx, epoch_valid_idx = virtual_time_split(
train_df,
valid_size=config.valid_size,
epoch_valid_size=config.epoch_valid_size)
valid_y = train_df.iloc[valid_idx]['answered_correctly'].values
epoch_valid_y = train_df.iloc[epoch_valid_idx][
'answered_correctly'].values
print('-' * 20)
print(f'train size: {len(train_idx)}')
print(f'valid size: {len(valid_idx)}')
with timer('prepare data loader'):
train_user_seqs = get_user_sequences(train_df.iloc[train_idx])
valid_user_seqs = get_user_sequences(train_df.iloc[valid_idx])
train_dataset = TrainDataset(train_user_seqs,
window_size=config.window_size,
stride_size=config.stride_size)
valid_dataset = ValidDataset(train_df,
train_user_seqs,
valid_user_seqs,
valid_idx,
window_size=config.window_size)
train_loader = DataLoader(train_dataset, **config.train_loader_params)
valid_loader = DataLoader(valid_dataset, **config.valid_loader_params)
# valid loader for epoch validation
epoch_valid_user_seqs = get_user_sequences(
train_df.iloc[epoch_valid_idx])
epoch_valid_dataset = ValidDataset(train_df,
train_user_seqs,
epoch_valid_user_seqs,
epoch_valid_idx,
window_size=config.window_size)
epoch_valid_loader = DataLoader(epoch_valid_dataset,
**config.valid_loader_params)
with timer('train'):
if config.model == 'akt':
content_encoder_config = BertConfig(
**config.content_encoder_config)
knowledge_encoder_config = BertConfig(
**config.knowledge_encoder_config)
decoder_config = BertConfig(**config.decoder_config)
content_encoder_config.max_position_embeddings = config.window_size + 1
knowledge_encoder_config.max_position_embeddings = config.window_size
decoder_config.max_position_embeddings = config.window_size + 1
model = AktEncoderDecoderModel(content_encoder_config,
knowledge_encoder_config,
decoder_config)
elif config.model == 'saint':
encoder_config = BertConfig(**config.encoder_config)
decoder_config = BertConfig(**config.decoder_config)
encoder_config.max_position_embeddings = config.window_size
decoder_config.max_position_embeddings = config.window_size
model = SaintEncoderDecoderModel(encoder_config, decoder_config)
else:
raise ValueError(f'Unknown model: {config.model}')
model.to(config.device)
model.zero_grad()
optimizer = optim.Adam(model.parameters(), **config.optimizer_params)
scheduler = NoamLR(optimizer, warmup_steps=config.warmup_steps)
loss_ema = None
for epoch in range(config.n_epochs):
epoch_start_time = time.time()
model.train()
progress = tqdm(train_loader,
desc=f'epoch {epoch + 1}',
leave=False)
for i, (x_batch, w_batch, y_batch) in enumerate(progress):
y_pred = model(**x_batch.to(config.device).to_dict())
loss = nn.BCEWithLogitsLoss(weight=w_batch.to(config.device))(
y_pred, y_batch.to(config.device))
loss.backward()
if (config.gradient_accumulation_steps is None
or (i + 1) % config.gradient_accumulation_steps == 0):
optimizer.step()
optimizer.zero_grad()
scheduler.step()
loss_ema = loss_ema * 0.9 + loss.item(
) * 0.1 if loss_ema is not None else loss.item()
progress.set_postfix(loss=loss_ema)
valid_preds = predict(model,
epoch_valid_loader,
device=config.device)
valid_score = roc_auc_score(epoch_valid_y, valid_preds)
elapsed_time = time.time() - epoch_start_time
print(
f'Epoch {epoch + 1}/{config.n_epochs} \t valid score: {valid_score:.5f} \t time: {elapsed_time / 60:.1f} min'
)
with timer('predict'):
valid_preds = predict(model, valid_loader, device=config.device)
valid_score = roc_auc_score(valid_y, valid_preds)
print(f'valid score: {valid_score:.5f}')
output_dir = Path(f'./output/{config_path.stem}/')
output_dir.mkdir(parents=True, exist_ok=True)
torch.save(model.state_dict(), output_dir / 'model.pt')
torch.save(optimizer.state_dict(), output_dir / 'optimizer.pt')
elapsed_time = time.time() - start_time
print(f'all processes done in {elapsed_time / 60:.1f} min.')
def main(args: DictConfig):
# Distributed training
torch.multiprocessing.set_sharing_strategy('file_system')
if str(args.exp.gpus) == '-1':
args.exp.gpus = torch.cuda.device_count()
# Secondary data args
args.data.setting = 'in-topic' if args.data.test_id is None else 'cross-topic'
dataset_name = args.data.path.split('/')[1]
args.data.path = f'{ROOT_PATH}/{args.data.path}'
# MlFlow Logging
if args.exp.logging:
experiment_name = f'{dataset_name}/{args.setting}-{args.data.setting}/{args.exp.task_name}'
mlf_logger = MLFlowLogger(experiment_name=experiment_name,
tracking_uri=MLFLOW_URI)
experiment = mlf_logger._mlflow_client.get_experiment_by_name(
experiment_name)
if experiment is not None:
experiment_id = experiment.experiment_id
if args.exp.check_exisisting_hash:
args.hash = calculate_hash(args)
existing_runs = mlf_logger._mlflow_client.search_runs(
filter_string=f"params.hash = '{args.hash}'",
run_view_type=mlflow.tracking.client.ViewType.ACTIVE_ONLY,
experiment_ids=[experiment_id])
if len(existing_runs) > 0:
logger.info('Skipping existing run.')
return
else:
logger.info('No runs found - perfoming one.')
# cpnt_path = f'{ROOT_PATH}/mlruns/{experiment_id}/{run_id}/artifacts'
# else:
# cpnt_path = None
# Load pretrained model and tokenizer
set_seed(args)
model = instantiate(args.lightning_module, args=args)
logger.info(f'Run arguments: \n{args.pretty()}')
# Early stopping & Checkpointing
early_stop_callback = EarlyStopping(
min_delta=0.00,
patience=args.exp.early_stopping_patience,
verbose=False,
mode='min')
checkpoint_callback = CustomModelCheckpoint(
model=model,
verbose=True,
mode='min',
save_top_k=1,
period=0 if args.exp.val_check_interval < 1.0 else 1)
lr_logging_callback = LearningRateLogger(logging_interval='epoch')
# Training
trainer = Trainer(
gpus=eval(str(args.exp.gpus)),
logger=mlf_logger if args.exp.logging else None,
max_epochs=args.exp.max_epochs,
gradient_clip_val=args.optimizer.max_grad_norm,
early_stop_callback=early_stop_callback,
val_check_interval=args.exp.val_check_interval,
checkpoint_callback=checkpoint_callback
if args.exp.checkpoint else None,
accumulate_grad_batches=args.exp.gradient_accumulation_steps,
auto_lr_find=args.optimizer.auto_lr_find,
precision=args.exp.precision,
distributed_backend='dp',
callbacks=[lr_logging_callback])
trainer.fit(model)
trainer.test(model)
# Cleaning cache
torch.cuda.empty_cache()
# Ending the run
if args.exp.logging:
mlf_logger.finalize()
def __init__(self, yaml_path):
config_file = yaml_path
config = yaml.load(open(config_file), Loader=yaml.FullLoader)
args = config["training"]
SEED = args["seed"]
DATASET = args["dataset"] # Multi30k or ISWLT
MODEL = args["model"] # gru**2, gru_attn**2, transformer, gcn_gru, gcngru_gru, gcngruattn_gru, gcnattn_gru
REVERSE = args["reverse"]
BATCH_SIZE = args["batch_size"]
ENC_EMB_DIM = args["encoder_embed_dim"]
DEC_EMB_DIM = args["decoder_embed_dim"]
ENC_HID_DIM = args["encoder_hidden_dim"]
DEC_HID_DIM = args["decoder_hidden_dim"]
ENC_DROPOUT = args["encoder_dropout"]
DEC_DROPOUT = args["decoder_dropout"]
NLAYERS = args["num_layers"]
N_EPOCHS = args["num_epochs"]
CLIP = args["grad_clip"]
LR = args["lr"]
LR_DECAY_RATIO = args["lr_decay_ratio"]
ID = args["id"]
PATIENCE = args["patience"]
DIR = 'checkpoints/{}-{}-{}/'.format(DATASET, MODEL, ID)
MODEL_PATH = DIR
LOG_PATH = '{}test-log.log'.format(DIR)
set_seed(SEED)
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
self.config = args
self.device = device
if 'transformer' in MODEL:
ENC_HEADS = args["encoder_heads"]
DEC_HEADS = args["decoder_heads"]
ENC_PF_DIM = args["encoder_pf_dim"]
DEC_PF_DIM = args["decoder_pf_dim"]
MAX_LEN = args["max_len"]
SRC = Field(tokenize = lambda text: tokenize_de(text, REVERSE),
init_token = '<sos>',
eos_token = '<eos>',
lower = True)
TGT = Field(tokenize = tokenize_en,
init_token = '<sos>',
eos_token = '<eos>',
lower = True)
GRH = RawField(postprocessing=batch_graph)
data_fields = [('src', SRC), ('trg', TGT), ('grh', GRH)]
train_data = Dataset(torch.load("data/Multi30k/train_data.pt"), data_fields)
valid_data = Dataset(torch.load("data/Multi30k/valid_data.pt"), data_fields)
test_data = Dataset(torch.load("data/Multi30k/test_data.pt"), data_fields)
self.train_data, self.valid_data, self.test_data = train_data, valid_data, test_data
train_iterator, valid_iterator, test_iterator = BucketIterator.splits(
(train_data, valid_data, test_data),
batch_size = BATCH_SIZE,
sort_key = lambda x: len(x.src),
sort_within_batch=False,
device = device)
self.train_iterator, self.valid_iterator, self.test_iterator = train_iterator, valid_iterator, test_iterator
SRC.build_vocab(train_data, min_freq = 2)
TGT.build_vocab(train_data, min_freq = 2)
self.SRC, self.TGT, self.GRH = SRC, TGT, GRH
print(f"Number of training examples: {len(train_data.examples)}")
print(f"Number of validation examples: {len(valid_data.examples)}")
print(f"Number of testing examples: {len(test_data.examples)}")
print(f"Unique tokens in source (de) vocabulary: {len(SRC.vocab)}")
print(f"Unique tokens in target (en) vocabulary: {len(TGT.vocab)}")
src_c, tgt_c = get_sentence_lengths(train_data)
src_lengths = counter2array(src_c)
tgt_lengths = counter2array(tgt_c)
print("maximum src, tgt sent lengths: ")
np.quantile(src_lengths, 1), np.quantile(tgt_lengths, 1)
# Get models and corresponding training scripts
INPUT_DIM = len(SRC.vocab)
OUTPUT_DIM = len(TGT.vocab)
SRC_PAD_IDX = SRC.vocab.stoi[SRC.pad_token]
TGT_PAD_IDX = TGT.vocab.stoi[TGT.pad_token]
self.SRC_PAD_IDX = SRC_PAD_IDX
self.TGT_PAD_IDX = TGT_PAD_IDX
if MODEL == "gru**2": # gru**2, gru_attn**2, transformer, gcn_gru
from models.gru_seq2seq import GRUEncoder, GRUDecoder, Seq2Seq
enc = GRUEncoder(INPUT_DIM, ENC_EMB_DIM, ENC_HID_DIM, DEC_HID_DIM, NLAYERS, ENC_DROPOUT)
dec = GRUDecoder(OUTPUT_DIM, DEC_EMB_DIM, ENC_HID_DIM, DEC_HID_DIM, NLAYERS, DEC_DROPOUT)
model = Seq2Seq(enc, dec, device).to(device)
from src.train import train_epoch_gru, evaluate_gru, epoch_time
train_epoch = train_epoch_gru
evaluate = evaluate_gru
self.enc, self.dec, self.model, self.train_epoch, self.evaluate = enc, dec, model, train_epoch, evaluate
elif MODEL == "gru_attn**2":
from models.gru_attn import GRUEncoder, GRUDecoder, Seq2Seq, Attention
attn = Attention(ENC_HID_DIM, DEC_HID_DIM)
enc = GRUEncoder(INPUT_DIM, ENC_EMB_DIM, ENC_HID_DIM, DEC_HID_DIM, NLAYERS, ENC_DROPOUT)
dec = GRUDecoder(OUTPUT_DIM, DEC_EMB_DIM, ENC_HID_DIM, DEC_HID_DIM, NLAYERS, DEC_DROPOUT, attn)
model = Seq2Seq(enc, dec, device).to(device)
from src.train import train_epoch_gru_attn, evaluate_gru_attn, epoch_time
train_epoch = train_epoch_gru_attn
evaluate = evaluate_gru_attn
self.enc, self.dec, self.model, self.train_epoch, self.evaluate, self.attn = enc, dec, model, train_epoch, evaluate, attn
elif MODEL == "transformer":
from models.transformer import Encoder, Decoder, Seq2Seq
enc = Encoder(INPUT_DIM, ENC_HID_DIM, NLAYERS, ENC_HEADS,
ENC_PF_DIM, ENC_DROPOUT, device, MAX_LEN)
dec = Decoder(OUTPUT_DIM, DEC_HID_DIM, NLAYERS, DEC_HEADS,
DEC_PF_DIM, DEC_DROPOUT, device, MAX_LEN)
model = Seq2Seq(enc, dec, SRC_PAD_IDX, TGT_PAD_IDX, device).to(device)
from src.train import train_epoch_tfmr, evaluate_tfmr, epoch_time
train_epoch = train_epoch_tfmr
evaluate = evaluate_tfmr
self.enc, self.dec, self.model, self.train_epoch, self.evaluate = enc, dec, model, train_epoch, evaluate
elif MODEL == "gcn_gru":
from models.gru_seq2seq import GCNEncoder, GRUDecoder, GCN2Seq
enc = GCNEncoder(INPUT_DIM, ENC_EMB_DIM, ENC_HID_DIM, NLAYERS, ENC_DROPOUT)
dec = GRUDecoder(OUTPUT_DIM, DEC_EMB_DIM, ENC_HID_DIM, DEC_HID_DIM, NLAYERS, DEC_DROPOUT)
model = GCN2Seq(enc, dec, device).to(device)
from src.train import train_epoch_gcn_gru, evaluate_gcn_gru, epoch_time
train_epoch = train_epoch_gcn_gru
evaluate = evaluate_gcn_gru
self.enc, self.dec, self.model, self.train_epoch, self.evaluate = enc, dec, model, train_epoch, evaluate
elif MODEL == "gcngru_gru":
from models.gru_seq2seq import GCNGRUEncoder, GRUDecoder, GCN2Seq
enc = GCNGRUEncoder(INPUT_DIM, ENC_EMB_DIM, ENC_HID_DIM, DEC_HID_DIM, NLAYERS, ENC_DROPOUT, device)
dec = GRUDecoder(OUTPUT_DIM, DEC_EMB_DIM, ENC_HID_DIM, DEC_HID_DIM, NLAYERS, DEC_DROPOUT)
model = GCN2Seq(enc, dec, device).to(device)
from src.train import train_epoch_gcn_gru, evaluate_gcn_gru, epoch_time
train_epoch = train_epoch_gcn_gru
evaluate = evaluate_gcn_gru
self.enc, self.dec, self.model, self.train_epoch, self.evaluate = enc, dec, model, train_epoch, evaluate
elif MODEL == "gcnattn_gru":
from models.gru_attn import GCNEncoder, GRUDecoder, GCN2Seq, Attention
attn = Attention(ENC_HID_DIM, DEC_HID_DIM)
enc = GCNEncoder(INPUT_DIM, ENC_EMB_DIM, ENC_HID_DIM, DEC_HID_DIM, NLAYERS, ENC_DROPOUT)
dec = GRUDecoder(OUTPUT_DIM, DEC_EMB_DIM, ENC_HID_DIM, DEC_HID_DIM, NLAYERS, DEC_DROPOUT, attn)
model = GCN2Seq(enc, dec, device).to(device)
from src.train import train_epoch_gcnattn_gru, evaluate_gcnattn_gru, epoch_time
train_epoch = train_epoch_gcnattn_gru
evaluate = evaluate_gcnattn_gru
self.enc, self.dec, self.model, self.train_epoch, self.evaluate, self.attn = enc, dec, model, train_epoch, evaluate, attn
elif MODEL == "gcngruattn_gru":
from models.gru_attn import GCNGRUEncoder, GRUDecoder, GCN2Seq, Attention
attn = Attention(ENC_HID_DIM, DEC_HID_DIM)
enc = GCNGRUEncoder(INPUT_DIM, ENC_EMB_DIM, ENC_HID_DIM, DEC_HID_DIM, NLAYERS, ENC_DROPOUT, device)
dec = GRUDecoder(OUTPUT_DIM, DEC_EMB_DIM, ENC_HID_DIM, DEC_HID_DIM, NLAYERS, DEC_DROPOUT, attn)
model = GCN2Seq(enc, dec, device).to(device)
from src.train import train_epoch_gcnattn_gru, evaluate_gcnattn_gru, epoch_time
train_epoch = train_epoch_gcnattn_gru
evaluate = evaluate_gcnattn_gru
self.enc, self.dec, self.model, self.train_epoch, self.evaluate, self.attn = enc, dec, model, train_epoch, evaluate, attn
else:
raise ValueError("Wrong model choice")
if 'gcn' in MODEL:
from src.utils import init_weights_uniform as init_weights
else:
from src.utils import init_weights_xavier as init_weights
model.apply(init_weights)
n_params = count_parameters(model)
print("Model initialized...{} params".format(n_params))
self.criterion = nn.CrossEntropyLoss(ignore_index=TGT_PAD_IDX)
print(os.path.join(MODEL_PATH, "checkpoint.pt"))
# try:
# state_dict = torch.load(os.path.join(MODEL_PATH, "checkpoint.pt"), map_location=device)['model_state_dict']
# except:
# state_dict = torch.load(os.path.join(MODEL_PATH, "checkpoint.pt"), map_location=device)
state_dict = torch.load(os.path.join(MODEL_PATH, "checkpoint.pt"), map_location=device)
if 'model_state_dict' in state_dict:
state_dict = state_dict['model_state_dict']
model.load_state_dict(state_dict)
self.model = model
f"Elapsed time: {elapsed_min}min {elapsed_sec:.4f}seconds.")
if __name__ == "__main__":
warnings.filterwarnings("ignore")
args = utils.get_parser().parse_args()
config = utils.load_config(args.config)
global_params = config["globals"]
output_dir = Path(global_params["output_dir"])
output_dir.mkdir(exist_ok=True, parents=True)
logger = utils.get_logger(output_dir / "output.log")
utils.set_seed(global_params["seed"])
device = C.get_device(global_params["device"])
df, datadir = C.get_metadata(config)
splitter = C.get_split(config)
calltype_labels = C.get_calltype_labels(df)
if config["data"].get("event_level_labels") is not None:
event_level_labels = C.get_event_level_labels(config)
else:
event_level_labels = None
if "Multilabel" in config["split"]["name"]:
y = calltype_labels
else:
def train_KFolds(train,
test,
target,
molecules,
coupling_types,
batch_size=1024,
n_folds=5,
seed=42,
debug=False):
oof = np.zeros(len(train))
preds = np.zeros(len(test))
gkf = GroupKFold(n_splits=n_folds)
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
criterion = nn.L1Loss()
utils.set_seed(seed)
for fold, (train_idx, val_idx) in enumerate(
gkf.split(train, target, groups=molecules), 1):
if debug:
if fold != 1:
continue
xtrain, ytrain = torch.from_numpy(train[train_idx]), torch.from_numpy(
target[train_idx])
xval, yval = torch.from_numpy(train[val_idx]), torch.from_numpy(
target[val_idx])
eval_types = coupling_types[val_idx]
train_set = TensorDataset(xtrain, ytrain)
val_set = TensorDataset(xval, yval)
train_loader = DataLoader(train_set,
batch_size=batch_size,
shuffle=True)
val_loader = DataLoader(val_set, batch_size=batch_size)
model = PointCNN().to(device).float()
optimizer = optim.Adam(model.parameters(), lr=.0003)
summary_path = 'runs/' + time.strftime("%Y%m%d-%H%M%S")
model_path = configs.models_folder / time.strftime("%Y%m%d-%H%M%S")
writer = SummaryWriter(summary_path)
best_val_score = 2.0
for epoch in range(1, 70):
trn_loss = 0.0
val_loss = 0.0
for i, (features, targets) in enumerate(train_loader):
writer.add_graph(model, features.float())
features, targets = features.float().to(device), targets.to(
device)
model.train()
outputs = model(features)
loss = criterion(outputs, targets)
trn_loss += loss.item()
loss.backward()
optimizer.step()
optimizer.zero_grad()
with torch.no_grad():
print('[%d] loss: %.5f' %
(epoch, trn_loss / len(train_loader)))
writer.add_scalar('training loss',
trn_loss / len(train_loader), epoch)
for i, (features, targets) in enumerate(val_loader):
features, targets = features.float().to(
device), targets.to(device)
model.eval()
outputs = model(features)
loss = criterion(outputs, targets)
val_loss += loss.item()
if i == 0:
eval_set = outputs.cpu().numpy()
else:
eval_set = np.append(eval_set, outputs.cpu().numpy())
print('[%d] validation loss: %.5f' %
(epoch, val_loss / len(val_loader)))
writer.add_scalar('validation loss',
val_loss / len(val_loader), epoch)
val_score = utils.mean_log_mae(yval, eval_set, eval_types)
print('[%d] validation score: %.5f' % (epoch, val_score))
writer.add_scalar('validation score', val_score, epoch)
if val_score < best_val_score:
torch.save(model.state_dict(), model_path)
best_val_score = val_score
writer.close()
ENC_HID_DIM = args["encoder_hidden_dim"]
DEC_HID_DIM = args["decoder_hidden_dim"]
ENC_DROPOUT = args["encoder_dropout"]
DEC_DROPOUT = args["decoder_dropout"]
NLAYERS = args["num_layers"]
N_EPOCHS = args["num_epochs"]
CLIP = args["grad_clip"]
LR = args["lr"]
LR_DECAY_RATIO = args["lr_decay_ratio"]
ID = args["id"]
PATIENCE = args["patience"]
DIR = 'checkpoints/{}-{}-{}/'.format(DATASET, MODEL, ID)
MODEL_PATH = DIR
LOG_PATH = '{}log.log'.format(DIR)
CONFIG_PATH = '{}config.yaml'.format(DIR)
set_seed(SEED)
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
ensure_path_exist(DIR)
yaml.dump(config, open(CONFIG_PATH, 'w'))
if 'transformer' in MODEL:
ENC_HEADS = args["encoder_heads"]
DEC_HEADS = args["decoder_heads"]
ENC_PF_DIM = args["encoder_pf_dim"]
DEC_PF_DIM = args["decoder_pf_dim"]
MAX_LEN = args["max_len"]
# dataset
SRC = Field(tokenize=lambda text: tokenize_de(text, REVERSE),
init_token='<sos>',
def train_KFolds(meta_df,
feature_df,
batch_size=1024,
seed=42,
n_folds=5,
debug=False):
oof = np.zeros(len(meta_df))
criterion = nn.BCELoss()
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
set_seed(seed)
# split 2017 and 2018 data temporarily for CV setup
idx_2017 = meta_df.index[meta_df.Season == 2017]
idx_2018 = meta_df.index[meta_df.Season == 2018]
meta_df_2017, feature_df_2017 = meta_df.loc[idx_2017], feature_df[idx_2017]
target_2017 = meta_df_2017.pop('Yards').values
meta_df_2018, feature_df_2018 = meta_df.loc[idx_2018], feature_df[idx_2018]
target_2018 = meta_df_2018.pop('Yards').values
gkf = GroupKFold(n_splits=n_folds)
games = meta_df_2018.GameId
for fold, (train_idx, val_idx) in enumerate(
gkf.split(feature_df_2018, target_2018, groups=games)):
if debug:
if fold != 1:
continue
# split 2018 data into training and evaluation set
xtrain_2018, ytrain_2018 = feature_df_2018[train_idx], target_2018[
train_idx]
xval, yval = feature_df_2018[val_idx], target_2018[val_idx]
yval = prepare_targets(yval)
# append 2017 data and 2018 training data --> full train set
xtrain = np.vstack((feature_df_2017, xtrain_2018))
ytrain = np.hstack((target_2017, ytrain_2018))
ytrain = prepare_targets(ytrain)
# create torch tensors
xtrain, ytrain = torch.from_numpy(xtrain), torch.from_numpy(ytrain)
xval, yval = torch.from_numpy(xval), torch.from_numpy(yval)
train_set = TensorDataset(xtrain, ytrain)
train_loader = DataLoader(train_set,
batch_size=batch_size,
shuffle=True)
val_set = TensorDataset(xval, yval)
val_loader = DataLoader(val_set, batch_size=batch_size, shuffle=False)
model = CNNTransformer()
optimizer = optim.Adam(model.parameters(), lr=.001)
summary_path = 'runs/' + time.strftime("%Y%m%d-%H%M%S")
writer = SummaryWriter(summary_path)
for epoch in range(1, 50):
trn_loss = 0.0
val_loss = 0.0
for i, (features, targets) in enumerate(train_loader):
features, targets = features.float().to(
device), targets.float().to(device)
writer.add_graph(model, features)
model.train()
outputs = model(features)
loss = criterion(outputs, targets)
trn_loss += loss.item()
loss.backward()
optimizer.step()
optimizer.zero_grad()
with torch.no_grad():
print('[%d] loss: %.5f' %
(epoch, trn_loss / len(train_loader)))
writer.add_scalar('training loss',
trn_loss / len(train_loader), epoch)
for i, (features, targets) in enumerate(val_loader):
features, targets = features.float().to(
device), targets.float().to(device)
model.eval()
outputs = model(features)
loss = criterion(outputs, targets)
val_loss += loss.item()
if i == 0:
eval_set = outputs.cpu().numpy()
else:
eval_set = np.vstack((eval_set, outputs.cpu().numpy()))
print('[%d] validation loss: %.5f' %
(epoch, val_loss / len(val_loader)))
writer.add_scalar('validation loss',
val_loss / len(val_loader), epoch)
yval_cum = np.clip(np.cumsum(yval.numpy(), axis=1), 0, 1)
eval_set = np.clip(np.cumsum(eval_set, axis=1), 0, 1)
validation_score = ((eval_set - yval_cum)**2).sum(axis=1).sum(
axis=0) / (199 * yval_cum.shape[0])
print('[%d] validation score: %.5f' %
(epoch, validation_score))
writer.add_scalar('validation score', validation_score, epoch)
writer.close()
def run(config: DictConfig, logger=None):
# ----------- setup experiment ------------------- #
# modify the datamodule
def train_dataloader(self):
ds = CutMixDatasetWrapper(self.train, **config.cutmix)
return DataLoader(ds, shuffle=True, **self.config)
LitDataModule.train_dataloader = train_dataloader
# setup logging
if logger is None:
logger = logging.getLogger(__name__)
set_seed(config.training.seed)
logger.info(f"using seed {config.training.seed}")
wandb.login(key=config.logger.api)
# init wandb logger
wb = load_obj(config.logger.class_name)(**config.logger.params)
# log the training config to wandb
# create a new hparam dictionary with the relevant hparams and
# log the hparams to wandb
wb_hparam = OrderedDict({
"training_fold": config.fold_num,
"input_dims": config.training.image_dim,
"batch_size": config.training.dataloaders.batch_size,
"optimizer": config.optimizer.class_name,
"scheduler": config.scheduler.class_name,
"learning_rate": config.optimizer.params.lr,
"weight_decay": config.optimizer.params.weight_decay,
"num_epochs": config.training.num_epochs,
})
wb.log_hyperparams(wb_hparam)
# ----------- prepare datasets ------------------- #
logger.info("Prepare Training/Validation Datasets.")
processor = Preprocessor(config.csv_dir, config.json_dir, config.image_dir,
5)
df = pd.read_csv(config.fold_csv_dir)
imsdir = config.image_dir
df.filePath = [
os.path.join(imsdir, df.image_id[i]) for i in range(len(df))
]
processor.dataframe = df
fold_num = config.fold_num
trainFold, valFold = processor.get_fold(fold_num)
# testFold, valFold = train_test_split(valFold, stratify=valFold.label, test_size=0.5)
trainFold.reset_index(drop=True, inplace=True)
# testFold.reset_index(drop=True, inplace=True)
valFold.reset_index(drop=True, inplace=True)
# init weights for loss function
weights = None # no weights for cutmix
tfms_config = config.augmentation
trn_augs = A.Compose(
[
load_obj(augs.class_name)(**augs.params)
for augs in tfms_config.train_augs
],
p=1.0,
)
valid_augs = A.Compose(
[
load_obj(augs.class_name)(**augs.params)
for augs in tfms_config.valid_augs
],
p=1.0,
)
test_augs = A.Compose(
[
load_obj(augs.class_name)(**augs.params)
for augs in tfms_config.test_augs
],
p=1.0,
)
tfms = {
"train": trn_augs,
"valid": valid_augs,
"test": test_augs,
}
# init datamodule
dl_config = config.training.dataloaders
dm = LitDataModule(trainFold, valFold, valFold, tfms, dl_config)
dm.setup()
# set training total steps
config.training.total_steps = (len(dm.train_dataloader()) *
config.training.num_epochs)
logger.info(f"Train dataset size: {len(dm.train_dataloader())}")
logger.info(f"Validation dataset size: {len(dm.val_dataloader())}")
# ----------- load lightning trainer ------------------- #
trainer_cfg = config.lightning
# init lightning callbacks
chkpt = pl.callbacks.ModelCheckpoint(**trainer_cfg.model_checkpoint)
cb_config = config.lightning.callbacks
cbs = [
load_obj(module.class_name)(**module.params) for module in cb_config
]
if config.log_to_stdout:
cbs.append(PrintCallback(log=logger))
# init trainer
args = trainer_cfg.init_args
trainer = pl.Trainer(callbacks=cbs,
checkpoint_callback=chkpt,
logger=wb,
**args)
# ----------- init lightning module ------------------- #
logger.info("Build network.")
model = LitModel(config, weights=weights)
# update model loss function to soft cross entropy loss
model.loss_fn = SoftTargetCrossEntropy(weight=weights)
model.unfreeze_classifier()
wb.watch(model.net)
model_name = config.model.params.model_name or config.model.class_name
logger.info(f"Init from base net: {model_name}")
logger.info(
f"Uses {str(config.optimizer.class_name).split('.')[-1]} optimizer.")
logger.info(
f"Learning Rate: {config.optimizer.params.lr}, Weight Decay: {config.optimizer.params.weight_decay}"
)
logger.info(
f"Uses {str(config.scheduler.class_name).split('.')[-1]} scheduler.")
tr_config = config.training
logger.info(
f"Training over {tr_config.num_epochs} epochs ~ {tr_config.total_steps} steps."
)
# ----------- start train/validaiton/test ------------------- #
# Pass the datamodule as arg to trainer.fit to override model hooks :)
trainer.fit(model, datamodule=dm)
# Compute metrics on test dataset
_ = trainer.test(model, datamodule=dm, ckpt_path=chkpt.best_model_path)
# ----------- finish experiment/cleanup/save weights ------------------- #
PATH = chkpt.best_model_path # path to the best performing model
WEIGHTS_PATH = config.training.model_save_dir
# init best model
logger.info(f"Restored best model weights from {PATH}.")
params = {"config": config, "weights": weights}
loaded_model = model.load_from_checkpoint(PATH, **params)
torchmodel = loaded_model.net
torch.save(torchmodel.state_dict(), WEIGHTS_PATH)
# upload the weights file to wandb
wandb.save(WEIGHTS_PATH)
# upload the full config file to wandb
conf_pth = f"{config.run_name}.yaml"
OmegaConf.save(config, f=conf_pth)
logger.info(f"Saved config file {conf_pth}.")
wandb.save(conf_pth)
logger.info(f"Saved model {WEIGHTS_PATH}.")
wandb.finish()
def load_cfg() -> Tuple[Config, str]:
from src.dict2obj import Config
from src.base import Coach
from src.utils import gpu, set_seed, load_checkpoint
cfg = Config()
set_seed(opts.seed)
# the model and other settings for training
model = load_model(opts.model)(num_classes=get_num_classes(opts.dataset),
scale=opts.scale)
device = gpu(model)
# load the dataset
trainset = load_dataset(dataset_type=opts.dataset,
transform=opts.transform,
train=True)
cfg['trainloader'] = load_dataloader(dataset=trainset,
batch_size=opts.batch_size,
train=True,
show_progress=opts.progress)
testset = load_dataset(dataset_type=opts.dataset,
transform=opts.transform,
train=False)
cfg['testloader'] = load_dataloader(dataset=testset,
batch_size=opts.batch_size,
train=False,
show_progress=opts.progress)
normalizer = load_normalizer(dataset_type=opts.dataset)
# load the optimizer and learning_policy
optimizer = load_optimizer(model=model,
optim_type=opts.optimizer,
lr=opts.lr,
momentum=opts.momentum,
betas=(opts.beta1, opts.beta2),
weight_decay=opts.weight_decay)
learning_policy = load_learning_policy(
optimizer=optimizer,
learning_policy_type=opts.learning_policy,
T_max=opts.epochs)
# generate the path for logging information and saving parameters
cfg['info_path'], cfg['log_path'] = generate_path(
method=METHOD,
dataset_type=opts.dataset,
model=opts.model,
description=opts.description)
if opts.resume:
cfg['start_epoch'] = load_checkpoint(path=cfg.info_path,
model=model,
optimizer=optimizer,
lr_scheduler=learning_policy)
else:
cfg['start_epoch'] = 0
cfg['coach'] = Coach(model=model,
device=device,
loss_func=load_loss_func(opts.loss)(model=model),
normalizer=normalizer,
optimizer=optimizer,
learning_policy=learning_policy)
# for validation
cfg['valider'] = load_valider(model=model,
device=device,
dataset_type=opts.dataset)
return cfg
def main(params):
# setup random seeds
set_seed(params.seed)
params.ar = True
exp_path = os.path.join(params.dump_path, params.exp_name)
# create exp path if it doesn't exist
if not os.path.exists(exp_path):
os.makedirs(exp_path)
# create logger
logger = create_logger(os.path.join(exp_path, 'train.log'), 0)
logger.info("============ Initialized logger ============")
logger.info("Random seed is {}".format(params.seed))
logger.info("\n".join("%s: %s" % (k, str(v))
for k, v in sorted(dict(vars(params)).items())))
logger.info("The experiment will be stored in %s\n" % exp_path)
logger.info("Running command: %s" % 'python ' + ' '.join(sys.argv))
logger.info("")
# load data
data, loader = load_smiles_data(params)
if params.data_type == 'ChEMBL':
all_smiles_mols = open(os.path.join(params.data_path, 'guacamol_v1_all.smiles'), 'r').readlines()
else:
all_smiles_mols = open(os.path.join(params.data_path, 'QM9_all.smiles'), 'r').readlines()
train_data, val_data = data['train'], data['valid']
dico = data['dico']
logger.info ('train_data len is {}'.format(len(train_data)))
logger.info ('val_data len is {}'.format(len(val_data)))
# keep cycling through train_loader forever
# stop when max iters is reached
def rcycle(iterable):
saved = [] # In-memory cache
for element in iterable:
yield element
saved.append(element)
while saved:
random.shuffle(saved) # Shuffle every batch
for element in saved:
yield element
train_loader = rcycle(train_data.get_iterator(shuffle=True, group_by_size=True, n_sentences=-1))
# extra param names for transformermodel
params.n_langs = 1
# build Transformer model
model = TransformerModel(params, is_encoder=False, with_output=True)
if params.local_cpu is False:
model = model.cuda()
opt = get_optimizer(model.parameters(), params.optimizer)
scores = {'ppl': np.float('inf'), 'acc': 0}
if params.load_path:
reloaded_iter, scores = load_model(params, model, opt, logger)
for total_iter, train_batch in enumerate(train_loader):
if params.load_path is not None:
total_iter += reloaded_iter + 1
epoch = total_iter // params.epoch_size
if total_iter == params.max_steps:
logger.info("============ Done training ... ============")
break
elif total_iter % params.epoch_size == 0:
logger.info("============ Starting epoch %i ... ============" % epoch)
model.train()
opt.zero_grad()
train_loss = calculate_loss(model, train_batch, params)
train_loss.backward()
if params.clip_grad_norm > 0:
clip_grad_norm_(model.parameters(), params.clip_grad_norm)
opt.step()
if total_iter % params.print_after == 0:
logger.info("Step {} ; Loss = {}".format(total_iter, train_loss))
if total_iter > 0 and total_iter % params.epoch_size == (params.epoch_size - 1):
# run eval step (calculate validation loss)
model.eval()
n_chars = 0
xe_loss = 0
n_valid = 0
logger.info("============ Evaluating ... ============")
val_loader = val_data.get_iterator(shuffle=True)
for val_iter, val_batch in enumerate(val_loader):
with torch.no_grad():
val_scores, val_loss, val_y = calculate_loss(model, val_batch, params, get_scores=True)
# update stats
n_chars += val_y.size(0)
xe_loss += val_loss.item() * len(val_y)
n_valid += (val_scores.max(1)[1] == val_y).sum().item()
ppl = np.exp(xe_loss / n_chars)
acc = 100. * n_valid / n_chars
logger.info("Acc={}, PPL={}".format(acc, ppl))
if acc > scores['acc']:
scores['acc'] = acc
scores['ppl'] = ppl
save_model(params, data, model, opt, dico, logger, 'best_model', epoch, total_iter, scores)
logger.info('Saving new best_model {}'.format(epoch))
logger.info("Best Acc={}, PPL={}".format(scores['acc'], scores['ppl']))
logger.info("============ Generating ... ============")
number_samples = 100
gen_smiles = generate_smiles(params, model, dico, number_samples)
generator = ARMockGenerator(gen_smiles)
try:
benchmark = ValidityBenchmark(number_samples=number_samples)
validity_score = benchmark.assess_model(generator).score
except:
validity_score = -1
try:
benchmark = UniquenessBenchmark(number_samples=number_samples)
uniqueness_score = benchmark.assess_model(generator).score
except:
uniqueness_score = -1
try:
benchmark = KLDivBenchmark(number_samples=number_samples, training_set=all_smiles_mols)
kldiv_score = benchmark.assess_model(generator).score
except:
kldiv_score = -1
logger.info('Validity Score={}, Uniqueness Score={}, KlDiv Score={}'.format(validity_score, uniqueness_score, kldiv_score))
save_model(params, data, model, opt, dico, logger, 'model', epoch, total_iter, {'ppl': ppl, 'acc': acc})
def main():
# Set seed for reproducibility
set_seed(seed_value=12345, use_cuda=False)
print("Hello world!")
help='Choose the device for testing')
parser.add_argument('--dim',
default=100,
type=int,
help='dim for feature size (default: 100)')
parser.add_argument('--distance_type',
default='euclidean',
type=str,
help='Distance type for testing')
return parser.parse_args()
if __name__ == '__main__':
args = parse_args()
set_seed(123)
test_dataset = MiniDataset(args.test_csv, args.test_data_dir)
test_loader = DataLoader(test_dataset,
batch_size=args.n_way *
(args.n_query + args.n_shot),
num_workers=3,
pin_memory=False,
worker_init_fn=worker_init_fn,
sampler=TestSampler(args.testcase_csv))
# TODO: load your model
model = Convnet4(out_channels=args.dim).to(args.device)
model.load_state_dict(torch.load(args.model))
distance = Distance(args)
if args.distance_type == 'param':
