def build_dataset(model, **kwargs): kwargs = {key: value for key, value in kwargs.items() if value is not None} datasets, metrics = multitask.build_dataset(model, **kwargs) distill_datasets = {} for task, task_data_dir in kwargs.items(): task_distill_path = os.path.join(task_data_dir, 'output.npz') task_distill_data = numpy.load(task_distill_path, allow_pickle=True) task_distill_data = task_distill_data['data'].tolist() distill_datasets[task] = task_distill_data return (datasets, distill_datasets), metrics
def build_dataset(model, **kwargs): kwargs = {key: value for key, value in kwargs.items() if value is not None} distill_datasets = {} distill_datasets_extra = {} for task, task_data_dir in kwargs.items(): task_distill_path = os.path.join(task_data_dir, task, 'output.npz') task_distill_data = np.load(task_distill_path, allow_pickle=True) distill_datasets[task] = task_distill_data['data'].tolist() distill_datasets_extra[task] = task_distill_data.get('extra', None) if distill_datasets_extra[task] is not None: distill_datasets_extra[task] = distill_datasets_extra[task].tolist( ) datasets, metrics = multitask.build_dataset(model, **kwargs) return (datasets, distill_datasets, distill_datasets_extra), metrics