def metadata_from_many(file_list, verify_schema=False, open_with=default_open, root=False): """ Given list of parquet files, make a FileMetaData that points to them Parameters ---------- file_list: list of paths of parquet files verify_schema: bool (False) Whether to assert that the schemas in each file are identical open_with: function Use this to open each path. root: str Top of the dataset's directory tree, for cases where it can't be automatically inferred. Returns ------- basepath: the root path that other paths are relative to fmd: metadata thrift structure """ from fastparquet import api if all(isinstance(pf, api.ParquetFile) for pf in file_list): pfs = file_list file_list = [pf.fn for pf in pfs] elif all(not isinstance(pf, api.ParquetFile) for pf in file_list): pfs = [api.ParquetFile(fn, open_with=open_with) for fn in file_list] else: raise ValueError("Merge requires all PaquetFile instances or none") basepath, file_list = analyse_paths(file_list, root=root) if verify_schema: for pf in pfs[1:]: if pf._schema != pfs[0]._schema: raise ValueError('Incompatible schemas') fmd = copy.copy(pfs[0].fmd) # we inherit "created by" field fmd.row_groups = [] for pf, fn in zip(pfs, file_list): if pf.file_scheme not in ['simple', 'empty']: for rg in pf.row_groups: rg = copy.copy(rg) rg.columns = [copy.copy(c) for c in rg.columns] for chunk in rg.columns: chunk.file_path = '/'.join([fn, chunk.file_path]) fmd.row_groups.append(rg) else: for rg in pf.row_groups: rg = copy.copy(rg) rg.columns = [copy.copy(c) for c in rg.columns] for chunk in rg.columns: chunk.file_path = fn fmd.row_groups.append(rg) fmd.num_rows = sum(rg.num_rows for rg in fmd.row_groups) return basepath, fmd
def metadata_from_many(file_list, verify_schema=False, open_with=default_open): """ Given list of parquet files, make a FileMetaData that points to them Parameters ---------- file_list: list of paths of parquet files verify_schema: bool (False) Whether to assert that the schemas in each file are identical open_with: function Use this to open each path. Returns ------- basepath: the root path that other paths are relative to fmd: metadata thrift structure """ from fastparquet import api sep = sep_from_open(open_with) if all(isinstance(pf, api.ParquetFile) for pf in file_list): pfs = file_list file_list = [pf.fn for pf in pfs] elif all(not isinstance(pf, api.ParquetFile) for pf in file_list): pfs = [api.ParquetFile(fn, open_with=open_with) for fn in file_list] else: raise ValueError("Merge requires all PaquetFile instances or none") basepath, file_list = analyse_paths(file_list, sep) if verify_schema: for pf in pfs[1:]: if pf._schema != pfs[0]._schema: raise ValueError('Incompatible schemas') fmd = thrift_copy(pfs[0].fmd) # we inherit "created by" field fmd.row_groups = [] for pf, fn in zip(pfs, file_list): if pf.file_scheme not in ['simple', 'empty']: # should remove 'empty' datasets up front? Get ignored on load # anyway. raise ValueError('Cannot merge multi-file input', fn) for rg in pf.row_groups: rg = thrift_copy(rg) for chunk in rg.columns: chunk.file_path = fn fmd.row_groups.append(rg) fmd.num_rows = sum(rg.num_rows for rg in fmd.row_groups) return basepath, fmd
def metadata_from_many(file_list, verify_schema=False, open_with=default_open, root=False, fs=None): """ Given list of parquet files, make a FileMetaData that points to them Parameters ---------- file_list: list of paths of parquet files verify_schema: bool (False) Whether to assert that the schemas in each file are identical open_with: function Use this to open each path. root: str Top of the dataset's directory tree, for cases where it can't be automatically inferred. fs: fsspsec.AbstractFileSystem Used in preference to open_with, if given Returns ------- basepath: the root path that other paths are relative to fmd: metadata thrift structure """ from fastparquet import api legacy = True if all(isinstance(pf, api.ParquetFile) for pf in file_list): pfs = file_list file_list = [pf.fn for pf in pfs] elif all(not isinstance(pf, api.ParquetFile) for pf in file_list): if verify_schema or fs is None or len(file_list) < 3: pfs = [ api.ParquetFile(fn, open_with=open_with) for fn in file_list ] else: # activate new code path here f0 = file_list[0] pf0 = api.ParquetFile(f0, open_with=open_with) if pf0.file_scheme not in ['empty', 'simple']: # set of directories, revert pfs = [pf0] + [ api.ParquetFile(fn, open_with=open_with) for fn in file_list[1:] ] else: # permits concurrent fetch of footers; needs fsspec >= 2021.6 size = int(1.4 * pf0._head_size) pieces = fs.cat(file_list[1:], start=-size) sizes = { path: int.from_bytes(piece[-8:-4], "little") + 8 for path, piece in pieces.items() } not_bigenough = [path for path, s in sizes.items() if s > size] if not_bigenough: new_pieces = fs.cat(not_bigenough, start=-max(sizes.values())) pieces.update(new_pieces) pieces = {k: _get_fmd(v) for k, v in pieces.items()} pieces = [(fn, pieces[fn]) for fn in file_list[1:]] # recover ordering legacy = False else: raise ValueError("Merge requires all PaquetFile instances or none") basepath, file_list = analyse_paths(file_list, root=root) if legacy: # legacy code path if verify_schema: for pf in pfs[1:]: if pf._schema != pfs[0]._schema: raise ValueError('Incompatible schemas') fmd = copy.copy(pfs[0].fmd) # we inherit "created by" field fmd.row_groups = [] for pf, fn in zip(pfs, file_list): if pf.file_scheme not in ['simple', 'empty']: for rg in pf.row_groups: rg = copy.copy(rg) rg.columns = [copy.copy(c) for c in rg.columns] for chunk in rg.columns: chunk.file_path = '/'.join([fn, chunk.file_path]) fmd.row_groups.append(rg) else: for rg in pf.row_groups: rg = copy.copy(rg) rg.columns = [copy.copy(c) for c in rg.columns] for chunk in rg.columns: chunk.file_path = fn fmd.row_groups.append(rg) fmd.num_rows = sum(rg.num_rows for rg in fmd.row_groups) return basepath, fmd for rg in pf0.fmd.row_groups: # chunks of first file, which would have file_path=None rg.columns[0].file_path = f0[len(basepath):].lstrip("/") for k, v in pieces: # Set file paths on other files rgs = v.row_groups or [] for rg in rgs: rg.columns[0].file_path = k[len(basepath):].lstrip("/") pf0.fmd.row_groups.extend(rgs) pf0.fmd.num_rows = sum(rg.num_rows for rg in pf0.fmd.row_groups) return basepath, pf0.fmd