def read_parquet(self, path, columns=None, metadata=None, schema=None, use_threads=True, use_pandas_metadata=False): """ Read Parquet data from path in file system. Can read from a single file or a directory of files Parameters ---------- path : str Single file path or directory columns : List[str], optional Subset of columns to read metadata : pyarrow.parquet.FileMetaData Known metadata to validate files against schema : pyarrow.parquet.Schema Known schema to validate files against. Alternative to metadata argument use_threads : boolean, default True Perform multi-threaded column reads use_pandas_metadata : boolean, default False If True and file has custom pandas schema metadata, ensure that index columns are also loaded Returns ------- table : pyarrow.Table """ from pyarrow.parquet import ParquetDataset dataset = ParquetDataset(path, schema=schema, metadata=metadata, filesystem=self) return dataset.read(columns=columns, use_threads=use_threads, use_pandas_metadata=use_pandas_metadata)
def read_parquet(self, path, columns=None, metadata=None, schema=None, nthreads=1): """ Read Parquet data from path in file system. Can read from a single file or a directory of files Parameters ---------- path : str Single file path or directory columns : List[str], optional Subset of columns to read metadata : pyarrow.parquet.FileMetaData Known metadata to validate files against schema : pyarrow.parquet.Schema Known schema to validate files against. Alternative to metadata argument nthreads : int, default 1 Number of columns to read in parallel. If > 1, requires that the underlying file source is threadsafe Returns ------- table : pyarrow.Table """ from pyarrow.parquet import ParquetDataset dataset = ParquetDataset(path, schema=schema, metadata=metadata, filesystem=self) return dataset.read(columns=columns, nthreads=nthreads)
def read_parquet(self, path, columns=None, metadata=None, schema=None, nthreads=1, use_pandas_metadata=False): """ Read Parquet data from path in file system. Can read from a single file or a directory of files Parameters ---------- path : str Single file path or directory columns : List[str], optional Subset of columns to read metadata : pyarrow.parquet.FileMetaData Known metadata to validate files against schema : pyarrow.parquet.Schema Known schema to validate files against. Alternative to metadata argument nthreads : int, default 1 Number of columns to read in parallel. If > 1, requires that the underlying file source is threadsafe use_pandas_metadata : boolean, default False If True and file has custom pandas schema metadata, ensure that index columns are also loaded Returns ------- table : pyarrow.Table """ from pyarrow.parquet import ParquetDataset dataset = ParquetDataset(path, schema=schema, metadata=metadata, filesystem=self) return dataset.read(columns=columns, nthreads=nthreads, use_pandas_metadata=use_pandas_metadata)
def read_parquet(self, path, columns=None, metadata=None, schema=None, use_threads=True, use_pandas_metadata=False): """ Read Parquet data from path in file system. Can read from a single file or a directory of files Parameters ---------- path : str Single file path or directory columns : List[str], optional Subset of columns to read metadata : pyarrow.parquet.FileMetaData Known metadata to validate files against schema : pyarrow.parquet.Schema Known schema to validate files against. Alternative to metadata argument use_threads : boolean, default True Perform multi-threaded column reads use_pandas_metadata : boolean, default False If True and file has custom pandas schema metadata, ensure that index columns are also loaded Returns ------- table : pyarrow.Table """ from pyarrow.parquet import ParquetDataset dataset = ParquetDataset(path, schema=schema, metadata=metadata, filesystem=self) return dataset.read(columns=columns, use_threads=use_threads, use_pandas_metadata=use_pandas_metadata)
def _read_parquet_arrow(self, hash_list): from pyarrow.parquet import ParquetDataset objfiles = [self._store.object_path(h) for h in hash_list] dataset = ParquetDataset(objfiles) table = dataset.read(nthreads=4) df = table.to_pandas() return df
def _read_parquet_arrow(self, hash_list): from pyarrow.parquet import ParquetDataset objfiles = [self.object_path(h) for h in hash_list] dataset = ParquetDataset(objfiles) try: table = dataset.read(use_threads=True) # pyarrow == 0.11 except TypeError: table = dataset.read(nthreads=4) # pyarrow < 0.11 try: dataframe = table.to_pandas() except Exception: # Try again to convert the table after removing # the possibly buggy Pandas-specific metadata. meta = table.schema.metadata.copy() meta.pop(b'pandas') newtable = table.replace_schema_metadata(meta) dataframe = newtable.to_pandas() return dataframe
async def test_local_arrow_storage_provider( tmp_path: Path, test_values: dt_test_values) -> None: test_table, visit_ids = test_values structured_provider = LocalArrowProvider(tmp_path) await structured_provider.init() for table_name, test_data in test_table.items(): await structured_provider.store_record(TableName(table_name), test_data["visit_id"], test_data) token_list = [] for i in visit_ids: token_list.append(await structured_provider.finalize_visit_id(i)) await structured_provider.flush_cache() await asyncio.gather(*token_list) for table_name, test_data in test_table.items(): dataset = ParquetDataset(tmp_path / table_name) df: DataFrame = dataset.read().to_pandas() assert df.shape[0] == 1 for row in df.itertuples(index=False): if test_data["visit_id"] == INVALID_VISIT_ID: del test_data["visit_id"] assert row._asdict() == test_data
def _build_node(build_dir, package, node_path, node, checks_contents=None, dry_run=False, env='default', ancestor_args={}): """ Parameters ---------- ancestor_args : dict any transform inherited from an ancestor plus any inherited handler kwargs Users can thus define kwargs that affect entire subtrees (e.g. transform: csv for 500 .txt files) and overriding of ancestor or peer values. Child transform or kwargs override ancestor k:v pairs. """ if _is_internal_node(node): if not dry_run: package.save_group(node_path, None) # Make a consumable copy. This is to cover a quirk introduced by accepting nodes named # like RESERVED keys -- if a RESERVED key is actually matched, it should be removed from # the node, or it gets treated like a subnode (or like a node with invalid content) node = node.copy() # NOTE: YAML parsing does not guarantee key order # fetch local transform and kwargs values; we do it using ifs # to prevent `key: None` from polluting the update local_args = _get_local_args( node, [RESERVED['transform'], RESERVED['kwargs']]) group_args = ancestor_args.copy() group_args.update(local_args) _consume(node, local_args) # if it's not a reserved word it's a group that we can descend groups = {k: v for k, v in iteritems(node) if _is_valid_group(v)} _consume(node, groups) if node: # Unused keys -- either keyword typos or node names with invalid values. # For now, until build.yml schemas, pointing out one should do. key, value = node.popitem() raise BuildException( "Invalid syntax: expected node data for {!r}, got {!r}".format( key, value)) for child_name, child_table in groups.items(): if glob.has_magic(child_name): # child_name is a glob string, use it to generate multiple child nodes for gchild_name, gchild_table in _gen_glob_data( build_dir, child_name, child_table): _build_node(build_dir, package, node_path + [gchild_name], gchild_table, checks_contents=checks_contents, dry_run=dry_run, env=env, ancestor_args=group_args) else: if not isinstance(child_name, str) or not is_nodename(child_name): raise StoreException("Invalid node name: %r" % child_name) _build_node(build_dir, package, node_path + [child_name], child_table, checks_contents=checks_contents, dry_run=dry_run, env=env, ancestor_args=group_args) else: # leaf node # prevent overwriting existing node names if '/'.join(node_path) in package: raise BuildException( "Naming conflict: {!r} added to package more than once".format( '/'.join(node_path))) # handle group leaf nodes (empty groups) if not node: if not dry_run: package.save_group(node_path, None) return include_package = node.get(RESERVED['package']) rel_path = node.get(RESERVED['file']) if rel_path and include_package: raise BuildException( "A node must define only one of {0} or {1}".format( RESERVED['file'], RESERVED['package'])) elif include_package: # package composition team, user, pkgname, subpath = parse_package(include_package, allow_subpath=True) existing_pkg = PackageStore.find_package(team, user, pkgname) if existing_pkg is None: raise BuildException("Package not found: %s" % include_package) if subpath: try: node = existing_pkg["/".join(subpath)] except KeyError: msg = "Package {team}:{owner}/{pkg} has no subpackage: {subpath}" raise BuildException( msg.format(team=team, owner=user, pkg=pkgname, subpath=subpath)) else: node = GroupNode(existing_pkg.get_contents().children) package.save_package_tree(node_path, node) elif rel_path: # handle nodes built from input files path = os.path.join(build_dir, rel_path) rel_meta_path = node.get(RESERVED['meta']) if rel_meta_path: with open(os.path.join(build_dir, rel_meta_path)) as fd: try: metadata = json.load(fd) except ValueError as ex: raise BuildException("Failed to parse %r as JSON: %s" % (rel_meta_path, ex)) if SYSTEM_METADATA in metadata: raise BuildException( "Invalid metadata in %r: not allowed to use key %r" % (rel_meta_path, SYSTEM_METADATA)) else: metadata = None # get either the locally defined transform and target or inherit from an ancestor transform = node.get(RESERVED['transform']) or ancestor_args.get( RESERVED['transform']) ID = 'id' # pylint:disable=C0103 PARQUET = 'parquet' # pylint:disable=C0103 if transform: transform = transform.lower() if transform in PANDAS_PARSERS: target = TargetType.PANDAS elif transform == PARQUET: target = TargetType.PANDAS elif transform == ID: target = TargetType.FILE else: raise BuildException("Unknown transform '%s' for %s" % (transform, rel_path)) else: # Guess transform and target based on file extension if not provided _, ext = splitext_no_dot(rel_path) if ext in PANDAS_PARSERS: transform = ext target = TargetType.PANDAS elif ext == PARQUET: transform = ext target = TargetType.PANDAS else: transform = ID target = TargetType.FILE print("Inferring 'transform: %s' for %s" % (transform, rel_path)) # TODO: parse/check environments: # environments = node.get(RESERVED['environments']) checks = node.get(RESERVED['checks']) if transform == ID: #TODO move this to a separate function if checks: with open(path, 'r') as fd: data = fd.read() _run_checks(data, checks, checks_contents, node_path, rel_path, target, env=env) if not dry_run: print("Registering %s..." % path) package.save_file(path, node_path, target, rel_path, transform, metadata) elif transform == PARQUET: if checks: from pyarrow.parquet import ParquetDataset dataset = ParquetDataset(path) table = dataset.read(nthreads=4) dataframe = table.to_pandas() _run_checks(dataframe, checks, checks_contents, node_path, rel_path, target, env=env) if not dry_run: print("Registering %s..." % path) package.save_file(path, node_path, target, rel_path, transform, metadata) else: # copy so we don't modify shared ancestor_args handler_args = dict(ancestor_args.get(RESERVED['kwargs'], {})) # local kwargs win the update handler_args.update(node.get(RESERVED['kwargs'], {})) # Check Cache store = PackageStore() path_hash = _path_hash(path, transform, handler_args) source_hash = digest_file(path) cachedobjs = [] if os.path.exists(store.cache_path(path_hash)): with open(store.cache_path(path_hash), 'r') as entry: cache_entry = json.load(entry) if cache_entry['source_hash'] == source_hash: cachedobjs = cache_entry['obj_hashes'] assert isinstance(cachedobjs, list) # TODO: check for changes in checks else use cache # below is a heavy-handed fix but it's OK for check builds to be slow if not checks and cachedobjs and all( os.path.exists(store.object_path(obj)) for obj in cachedobjs): # Use existing objects instead of rebuilding package.save_cached_df(cachedobjs, node_path, target, rel_path, transform, metadata) else: # read source file into DataFrame print("Serializing %s..." % path) if _have_pyspark(): dataframe = _file_to_spark_data_frame( transform, path, handler_args) else: dataframe = _file_to_data_frame( transform, path, handler_args) if checks: # TODO: test that design works for internal nodes... e.g. iterating # over the children and getting/checking the data, err msgs, etc. _run_checks(dataframe, checks, checks_contents, node_path, rel_path, target, env=env) # serialize DataFrame to file(s) if not dry_run: print("Saving as binary dataframe...") obj_hashes = package.save_df(dataframe, node_path, target, rel_path, transform, metadata) # Add to cache cache_entry = dict(source_hash=source_hash, obj_hashes=obj_hashes) with open(store.cache_path(path_hash), 'w') as entry: json.dump(cache_entry, entry) else: # rel_path and package are both None raise BuildException( "Leaf nodes must define either a %s or %s key" % (RESERVED['file'], RESERVED['package']))