] valid_set = [ data_path + df_valid + x for x in os.listdir(data_path + df_valid) if x.startswith("day") and x.split(".")[0].split("_")[-1] in valid_days ] print(train_set, valid_set) cont_names = ["I" + str(x) for x in range(1, 14)] cat_names = ["C" + str(x) for x in range(1, 27)] cols = ["label"] + cont_names + cat_names print("Creating Workflow Object") proc = Workflow(cat_names=cat_names, cont_names=cont_names, label_name=["label"]) proc.add_feature([ZeroFill(replace=True), LogOp(replace=True)]) proc.add_preprocess(Normalize(replace=True)) if int(args.freq_thresh) == 0: proc.add_preprocess(Categorify(replace=True, out_path=args.out_dir)) else: proc.add_preprocess( Categorify( replace=True, use_frequency=True, freq_threshold=int(args.freq_thresh), out_path=args.out_dir, )) print("Creating Dataset Iterator") dataset_args = {"sep": "\t"} if args.in_file_type == "csv" else {} trains_ds = Dataset(train_set, engine=args.in_file_type,
def test_dask_workflow_api_dlrm(client, tmpdir, datasets, freq_threshold, part_mem_fraction, engine, cat_cache, on_host, shuffle): paths = glob.glob(str(datasets[engine]) + "/*." + engine.split("-")[0]) if engine == "parquet": df1 = cudf.read_parquet(paths[0])[mycols_pq] df2 = cudf.read_parquet(paths[1])[mycols_pq] elif engine == "csv": df1 = cudf.read_csv(paths[0], header=0)[mycols_csv] df2 = cudf.read_csv(paths[1], header=0)[mycols_csv] else: df1 = cudf.read_csv(paths[0], names=allcols_csv)[mycols_csv] df2 = cudf.read_csv(paths[1], names=allcols_csv)[mycols_csv] df0 = cudf.concat([df1, df2], axis=0) if engine == "parquet": cat_names = ["name-cat", "name-string"] else: cat_names = ["name-string"] cont_names = ["x", "y", "id"] label_name = ["label"] processor = Workflow(client=client, cat_names=cat_names, cont_names=cont_names, label_name=label_name) processor.add_feature( [ops.FillMissing(), ops.Clip(min_value=0), ops.LogOp()]) processor.add_preprocess( ops.Categorify( freq_threshold=freq_threshold, out_path=str(tmpdir), cat_cache=cat_cache, on_host=on_host, )) processor.finalize() if engine in ("parquet", "csv"): dataset = Dataset(paths, part_mem_fraction=part_mem_fraction) else: dataset = Dataset(paths, names=allcols_csv, part_mem_fraction=part_mem_fraction) output_path = os.path.join(tmpdir, "processed") processor.apply(dataset, output_path=output_path, shuffle=shuffle) # Can still access the final ddf if we didn't shuffle if not shuffle: result = processor.get_ddf().compute() assert len(df0) == len(result) assert result["x"].min() == 0.0 assert result["x"].isna().sum() == 0 assert result["y"].min() == 0.0 assert result["y"].isna().sum() == 0 # Check category counts cat_expect = df0.groupby("name-string").agg({ "name-string": "count" }).reset_index(drop=True) cat_result = (result.groupby("name-string").agg({ "name-string": "count" }).reset_index(drop=True)) if freq_threshold: cat_expect = cat_expect[ cat_expect["name-string"] >= freq_threshold] # Note that we may need to skip the 0th element in result (null mapping) assert_eq( cat_expect, cat_result.iloc[1:] if len(cat_result) > len(cat_expect) else cat_result, check_index=False, ) else: assert_eq(cat_expect, cat_result) # Read back from disk df_disk = dask_cudf.read_parquet(output_path, index=False).compute() for col in df_disk: assert_eq(result[col], df_disk[col]) else: # Read back from disk df_disk = dask_cudf.read_parquet(output_path, index=False).compute() assert len(df0) == len(df_disk)
def main(args): """Multi-GPU Criteo/DLRM Preprocessing Benchmark This benchmark is designed to measure the time required to preprocess the Criteo (1TB) dataset for Facebookâs DLRM model. The user must specify the path of the raw dataset (using the `--data-path` flag), as well as the output directory for all temporary/final data (using the `--out-path` flag) Example Usage ------------- python dask-nvtabular-criteo-benchmark.py --data-path /path/to/criteo_parquet --out-path /out/dir/` Dataset Requirements (Parquet) ------------------------------ This benchmark is designed with a parquet-formatted dataset in mind. While a CSV-formatted dataset can be processed by NVTabular, converting to parquet will yield significantly better performance. To convert your dataset, try using the `optimize_criteo.ipynb` notebook (also located in `NVTabular/examples/`) For a detailed parameter overview see `NVTabular/examples/MultiGPUBench.md` """ # Input data_path = args.data_path freq_limit = args.freq_limit out_files_per_proc = args.out_files_per_proc high_card_columns = args.high_cards.split(",") dashboard_port = args.dashboard_port if args.protocol == "ucx": UCX_TLS = os.environ.get("UCX_TLS", "tcp,cuda_copy,cuda_ipc,sockcm") os.environ["UCX_TLS"] = UCX_TLS # Cleanup output directory BASE_DIR = args.out_path dask_workdir = os.path.join(BASE_DIR, "workdir") output_path = os.path.join(BASE_DIR, "output") stats_path = os.path.join(BASE_DIR, "stats") if not os.path.isdir(BASE_DIR): os.mkdir(BASE_DIR) for dir_path in (dask_workdir, output_path, stats_path): if os.path.isdir(dir_path): shutil.rmtree(dir_path) os.mkdir(dir_path) # Use Criteo dataset by default (for now) cont_names = (args.cont_names.split(",") if args.cont_names else ["I" + str(x) for x in range(1, 14)]) cat_names = (args.cat_names.split(",") if args.cat_names else ["C" + str(x) for x in range(1, 27)]) label_name = ["label"] # Specify Categorify/GroupbyStatistics options tree_width = {} cat_cache = {} for col in cat_names: if col in high_card_columns: tree_width[col] = args.tree_width cat_cache[col] = args.cat_cache_high else: tree_width[col] = 1 cat_cache[col] = args.cat_cache_low # Use total device size to calculate args.device_limit_frac device_size = device_mem_size(kind="total") device_limit = int(args.device_limit_frac * device_size) device_pool_size = int(args.device_pool_frac * device_size) part_size = int(args.part_mem_frac * device_size) # Parse shuffle option shuffle = None if args.shuffle == "PER_WORKER": shuffle = nvt_io.Shuffle.PER_WORKER elif args.shuffle == "PER_PARTITION": shuffle = nvt_io.Shuffle.PER_PARTITION # Check if any device memory is already occupied for dev in args.devices.split(","): fmem = _pynvml_mem_size(kind="free", index=int(dev)) used = (device_size - fmem) / 1e9 if used > 1.0: warnings.warn( f"BEWARE - {used} GB is already occupied on device {int(dev)}!" ) # Setup LocalCUDACluster if args.protocol == "tcp": cluster = LocalCUDACluster( protocol=args.protocol, n_workers=args.n_workers, CUDA_VISIBLE_DEVICES=args.devices, device_memory_limit=device_limit, local_directory=dask_workdir, dashboard_address=":" + dashboard_port, ) else: cluster = LocalCUDACluster( protocol=args.protocol, n_workers=args.n_workers, CUDA_VISIBLE_DEVICES=args.devices, enable_nvlink=True, device_memory_limit=device_limit, local_directory=dask_workdir, dashboard_address=":" + dashboard_port, ) client = Client(cluster) # Setup RMM pool if args.device_pool_frac > 0.01: setup_rmm_pool(client, device_pool_size) # Define Dask NVTabular "Workflow" processor = Workflow(cat_names=cat_names, cont_names=cont_names, label_name=label_name, client=client) if args.normalize: processor.add_feature([ops.FillMissing(), ops.Normalize()]) else: processor.add_feature( [ops.FillMissing(), ops.Clip(min_value=0), ops.LogOp()]) processor.add_preprocess( ops.Categorify( out_path=stats_path, tree_width=tree_width, cat_cache=cat_cache, freq_threshold=freq_limit, search_sorted=not freq_limit, on_host=not args.cats_on_device, )) processor.finalize() dataset = Dataset(data_path, "parquet", part_size=part_size) # Execute the dask graph runtime = time.time() if args.profile is not None: with performance_report(filename=args.profile): processor.apply( dataset, shuffle=shuffle, out_files_per_proc=out_files_per_proc, output_path=output_path, num_io_threads=args.num_io_threads, ) else: processor.apply( dataset, num_io_threads=args.num_io_threads, shuffle=shuffle, out_files_per_proc=out_files_per_proc, output_path=output_path, ) runtime = time.time() - runtime print("\nDask-NVTabular DLRM/Criteo benchmark") print("--------------------------------------") print(f"partition size | {part_size}") print(f"protocol | {args.protocol}") print(f"device(s) | {args.devices}") print(f"rmm-pool-frac | {(args.device_pool_frac)}") print(f"out-files-per-proc | {args.out_files_per_proc}") print(f"num_io_threads | {args.num_io_threads}") print(f"shuffle | {args.shuffle}") print(f"cats-on-device | {args.cats_on_device}") print("======================================") print(f"Runtime[s] | {runtime}") print("======================================\n") client.close()
def main(args): # Input data_path = args.data_path out_path = args.out_path freq_limit = args.freq_limit out_files_per_proc = args.splits if args.protocol == "ucx": os.environ["UCX_TLS"] = "tcp,cuda_copy,cuda_ipc,sockcm" # Use Criteo dataset by default (for now) cont_names = (args.cont_names.split(",") if args.cont_names else ["I" + str(x) for x in range(1, 14)]) cat_names = (args.cat_names.split(",") if args.cat_names else ["C" + str(x) for x in range(1, 27)]) label_name = ["label"] if args.cat_splits: tree_width = { name: int(s) for name, s in zip(cat_names, args.cat_splits.split(",")) } else: tree_width = {col: 1 for col in cat_names} if args.cat_names is None: # Using Criteo... Use more hash partitions for # known high-cardinality columns tree_width["C20"] = 8 tree_width["C1"] = 8 tree_width["C22"] = 4 tree_width["C10"] = 4 tree_width["C21"] = 2 tree_width["C11"] = 2 tree_width["C23"] = 2 tree_width["C12"] = 2 # Specify categorical caching location cat_cache = None if args.cat_cache: cat_cache = args.cat_cache.split(",") if len(cat_cache) == 1: cat_cache = cat_cache[0] else: # If user is specifying a list of options, # they must specify an option for every cat column assert len(cat_names) == len(cat_cache) if isinstance(cat_cache, str): cat_cache = {col: cat_cache for col in cat_names} elif isinstance(cat_cache, list): cat_cache = {name: c for name, c in zip(cat_names, cat_cache)} else: # Criteo/DLRM Defaults cat_cache = {col: "device" for col in cat_names} if args.cat_names is None: cat_cache["C20"] = "host" cat_cache["C1"] = "host" # Only need to cache the largest two on a dgx-2 if args.n_workers < 16: cat_cache["C22"] = "host" cat_cache["C10"] = "host" # Use total device size to calculate args.device_limit_frac device_size = device_mem_size(kind="total") device_limit = int(args.device_limit_frac * device_size) device_pool_size = int(args.device_pool_frac * device_size) part_size = int(args.part_mem_frac * device_size) # Setup LocalCUDACluster if args.protocol == "tcp": cluster = LocalCUDACluster( protocol=args.protocol, n_workers=args.n_workers, CUDA_VISIBLE_DEVICES=args.devs, device_memory_limit=device_limit, local_directory=args.dask_workspace, dashboard_address=":3787", ) else: cluster = LocalCUDACluster( protocol=args.protocol, n_workers=args.n_workers, CUDA_VISIBLE_DEVICES=args.devs, enable_nvlink=True, device_memory_limit=device_limit, local_directory=args.dask_workspace, dashboard_address=":3787", ) client = Client(cluster) # Setup RMM pool if not args.no_rmm_pool: setup_rmm_pool(client, device_pool_size) # Define Dask NVTabular "Workflow" processor = Workflow(cat_names=cat_names, cont_names=cont_names, label_name=label_name, client=client) processor.add_feature([ops.ZeroFill(), ops.LogOp()]) processor.add_preprocess( ops.Categorify( out_path=out_path, tree_width=tree_width, cat_cache=cat_cache, freq_threshold=freq_limit, on_host=args.cat_on_host, )) processor.finalize() dataset = Dataset(data_path, "parquet", part_size=part_size) # Execute the dask graph runtime = time.time() if args.profile is not None: with performance_report(filename=args.profile): processor.apply( dataset, shuffle="full" if args.worker_shuffle else "partial", out_files_per_proc=out_files_per_proc, output_path=out_path, ) else: processor.apply( dataset, shuffle="full" if args.worker_shuffle else "partial", out_files_per_proc=out_files_per_proc, output_path=out_path, ) runtime = time.time() - runtime print("\nDask-NVTabular DLRM/Criteo benchmark") print("--------------------------------------") print(f"partition size | {part_size}") print(f"protocol | {args.protocol}") print(f"device(s) | {args.devs}") print(f"rmm-pool | {(not args.no_rmm_pool)}") print(f"out_files_per_proc | {args.splits}") print(f"worker-shuffle | {args.worker_shuffle}") print("======================================") print(f"Runtime[s] | {runtime}") print("======================================\n") client.close()