def _get_data_reference_list(self, data_asset_name: Optional[str] = None ) -> List[str]: """List objects in the underlying data store to create a list of data_references. This method is used to refresh the cache. """ path_list: List[ str] = get_filesystem_one_level_directory_glob_path_list( base_directory_path=self.base_directory, glob_directive=self._glob_directive) return sorted(path_list)
def _get_data_reference_list_for_asset(self, asset: Optional[Asset]) -> List[str]: base_directory: str = self.base_directory glob_directive: str = self._glob_directive if asset is not None: if asset.base_directory: base_directory = normalize_directory_path( dir_path=asset.base_directory, root_directory_path=base_directory ) if asset.glob_directive: glob_directive = asset.glob_directive path_list: List[str] = get_filesystem_one_level_directory_glob_path_list( base_directory_path=base_directory, glob_directive=glob_directive ) return sorted(path_list)
def bobster_columnar_table_multi_batch_normal_mean_5000_stdev_1000_data_context( tmp_path_factory, monkeypatch, ) -> DataContext: """ This fixture generates three years' worth (36 months; i.e., 36 batches) of taxi trip data with the number of rows of a batch sampled from a normal distribution with the mean of 5,000 rows and the standard deviation of 1,000 rows. """ # Re-enable GE_USAGE_STATS monkeypatch.delenv("GE_USAGE_STATS") project_path: str = str(tmp_path_factory.mktemp("taxi_data_context")) context_path: str = os.path.join(project_path, "great_expectations") os.makedirs(os.path.join(context_path, "expectations"), exist_ok=True) data_path: str = os.path.join(context_path, "..", "data") os.makedirs(os.path.join(data_path), exist_ok=True) shutil.copy( file_relative_path( __file__, os.path.join( "..", "integration", "fixtures", "yellow_trip_data_pandas_fixture", "great_expectations", "great_expectations.yml", ), ), str(os.path.join(context_path, "great_expectations.yml")), ) base_directory: str = file_relative_path( __file__, os.path.join( "..", "test_sets", "taxi_yellow_trip_data_samples", ), ) file_name_list: List[ str] = get_filesystem_one_level_directory_glob_path_list( base_directory_path=base_directory, glob_directive="*.csv") file_name_list = sorted(file_name_list) num_files: int = len(file_name_list) rnd_num_sample: np.float64 output_file_lenths: List[int] = [ round(rnd_num_sample) for rnd_num_sample in np.random.normal( loc=5.0e3, scale=1.0e3, size=num_files) ] idx: int file_name: str output_file_name_length_map: Dict[str, int] = { file_name_list[idx]: output_file_lenths[idx] for idx, file_name in enumerate(file_name_list) } csv_source_path: str df: pd.DataFrame for file_name in file_name_list: csv_source_path = os.path.join(base_directory, file_name) df = pd.read_csv(filepath_or_buffer=csv_source_path) df = df.sample(n=output_file_name_length_map[file_name], replace=False, random_state=1) df.to_csv(path_or_buf=os.path.join(context_path, "..", "data", file_name), index=False) context: DataContext = DataContext(context_root_dir=context_path) assert context.root_directory == context_path return context