def build_dataset(model, **kwargs):
kwargs = {key: value for key, value in kwargs.items() if value is not None}
datasets, metrics = multitask.build_dataset(model, **kwargs)
distill_datasets = {}
for task, task_data_dir in kwargs.items():
task_distill_path = os.path.join(task_data_dir, 'output.npz')
task_distill_data = numpy.load(task_distill_path, allow_pickle=True)
task_distill_data = task_distill_data['data'].tolist()
distill_datasets[task] = task_distill_data
return (datasets, distill_datasets), metrics
def build_dataset(model, **kwargs):
kwargs = {key: value for key, value in kwargs.items() if value is not None}
distill_datasets = {}
distill_datasets_extra = {}
for task, task_data_dir in kwargs.items():
task_distill_path = os.path.join(task_data_dir, task, 'output.npz')
task_distill_data = np.load(task_distill_path, allow_pickle=True)
distill_datasets[task] = task_distill_data['data'].tolist()
distill_datasets_extra[task] = task_distill_data.get('extra', None)
if distill_datasets_extra[task] is not None:
distill_datasets_extra[task] = distill_datasets_extra[task].tolist(
)
datasets, metrics = multitask.build_dataset(model, **kwargs)
return (datasets, distill_datasets, distill_datasets_extra), metrics