def download_dataset(azure_dataset_id: str, target_folder: Path, dataset_csv: str, azure_config: AzureConfig) -> Path: """ Downloads or checks for an existing dataset on the executing machine. If a local_dataset is supplied and the directory is present, return that. Otherwise, download the dataset specified by the azure_dataset_id from the AzureML dataset attached to the given AzureML workspace. The dataset is downloaded into the `target_folder`, in a subfolder that has the same name as the dataset. If there already appears to be such a folder, and the folder contains a dataset csv file, no download is started. :param azure_dataset_id: The name of a dataset that is registered in the AzureML workspace. :param target_folder: The folder in which to download the dataset from Azure. :param dataset_csv: Name of the csv file describing the dataset. :param azure_config: All Azure-related configuration options. :return: A path on the local machine that contains the dataset. """ logging.info("Trying to download dataset via AzureML datastore now.") azure_dataset = get_or_create_dataset(azure_config, azure_dataset_id) if not isinstance(azure_dataset, FileDataset): raise ValueError(f"Expected to get a FileDataset, but got {type(azure_dataset)}") # The downloaded dataset may already exist from a previous run. expected_dataset_path = target_folder / azure_dataset_id expected_dataset_file = expected_dataset_path / dataset_csv logging.info(f"Model training will use dataset '{azure_dataset_id}' in Azure.") if expected_dataset_path.is_dir() and expected_dataset_file.is_file(): logging.info(f"The dataset appears to be downloaded already in {expected_dataset_path}. Skipping.") return expected_dataset_path logging.info("Starting to download the dataset - WARNING, this could take very long!") with logging_section("Downloading dataset"): t0 = time.perf_counter() azure_dataset.download(target_path=str(expected_dataset_path), overwrite=False) t1 = time.perf_counter() - t0 logging.info(f"Azure dataset '{azure_dataset_id}' downloaded in {t1} seconds") logging.info(f"Azure dataset '{azure_dataset_id}' is now available in {expected_dataset_path}") return expected_dataset_path
def download_dataset(azure_dataset_id: str, target_folder: Path, azure_config: AzureConfig) -> Path: """ Downloads or checks for an existing dataset on the executing machine. If a local_dataset is supplied and the directory is present, return that. Otherwise, download the dataset specified by the azure_dataset_id from the AzureML dataset attached to the given AzureML workspace. The dataset is downloaded into the `target_folder`, in a subfolder that has the same name as the dataset. If there already appears to be such a folder, and the folder contains a dataset.csv file, no download is started. :param local_dataset: The path to an existing local dataset. :param azure_dataset_id: The name of a dataset that is registered in the AzureML workspace. :param target_folder: The folder in which to download the dataset from Azure. :param azure_config: All Azure-related configuration options. :return: A path on the local machine that contains the dataset. """ workspace = azure_config.get_workspace() try: downloaded_via_blobxfer = download_dataset_via_blobxfer( dataset_id=azure_dataset_id, azure_config=azure_config, target_folder=target_folder) if downloaded_via_blobxfer: return downloaded_via_blobxfer except Exception as ex: print_exception(ex, message="Unable to download dataset via blobxfer.") logging.info("Trying to download dataset via AzureML datastore now.") azure_dataset = get_or_create_dataset(workspace, azure_dataset_id) if not isinstance(azure_dataset, FileDataset): raise ValueError( f"Expected to get a FileDataset, but got {type(azure_dataset)}") # The downloaded dataset may already exist from a previous run. expected_dataset_path = target_folder / azure_dataset_id expected_dataset_file = expected_dataset_path / DATASET_CSV_FILE_NAME logging.info( f"Model training will use dataset '{azure_dataset_id}' in Azure.") if expected_dataset_path.is_dir() and expected_dataset_file.is_file(): logging.info( f"The dataset appears to be downloaded already in {expected_dataset_path}. Skipping." ) return expected_dataset_path logging.info( "Starting to download the dataset - WARNING, this could take very long!" ) with logging_section("Downloading dataset"): azure_dataset.download(target_path=str(expected_dataset_path), overwrite=False) logging.info( f"Azure dataset '{azure_dataset_id}' is now available in {expected_dataset_path}" ) return expected_dataset_path