def export_onnx_models(models, output_dir):
print('Exporting ONNX models...')
ensure_directory_exists(output_dir)
for model in tqdm(models):
model.export(output_dir)
print(f'Exported ONNX models to {output_dir}.')
def download(self):
"""
Downloads and extracts the dataset's resources from self.resources.
"""
if self.data_exist():
return
download_dir = path.join(datasets_dir(self.root_dir), 'download')
ensure_directory_exists(download_dir)
for filename, (url, md5) in self.resources.items():
download_and_extract_archive(url,
download_root=download_dir,
filename=filename,
md5=md5,
remove_finished=False)
print('Downloading done!')
def evaluate(models, datasets, output_dir, model_dir, split='test'):
"""
Runs the evaluation of every model in `models` on every dataset from `datasets`.
:param models: list of models deriving from `BaseModel`
:param datasets: list of datasets deriving from `BaseDataset`
:param output_dir: the output directory for the evaluation .csv file
:param split: the split (train, val, test) of the dataset to use
"""
results = {'model': [], 'dataset': [], 'score': []}
for dataset in datasets:
eval_set = dataset.get_split(split)
for model in models:
model = load_model(model, dataset, model_dir)
print(
f'Evaluating model {model.name()} on dataset {dataset.name()}, \'{split}\' partition...'
)
score = model.score(eval_set, cv=False)
print(score)
results['model'].append(model.name())
results['dataset'].append(dataset.name())
results['score'].append(score)
results = pd.DataFrame(results)
print(results)
ensure_directory_exists(output_dir)
output_filename = path.join(output_dir, f'results_{timestamp()}.csv')
results.to_csv(output_filename)
print(f'Evaluation results saved to {output_filename}.')
def save_model(model, dataset, models_dir):
ensure_directory_exists(models_dir)
model.save(get_model_fullname(model, dataset, models_dir))
def save_image(self, fig, name, width=900, height=300, **kwargs):
ensure_directory_exists(self.image_dir)
fig.write_image(path.join(self.image_dir, name),
width=width,
height=height,
**kwargs)