def try_read(path, encoding=None):
# try with cPickle
# try with current pickle, if we have a Type Error then
# try with the compat pickle to handle subclass changes
# pass encoding only if its not None as py2 doesn't handle
# the param
# cpickle
# GH 6899
try:
with warnings.catch_warnings(record=True):
# We want to silence any warnings about, e.g. moved modules.
warnings.simplefilter("ignore", Warning)
return read_wrapper(lambda f: pkl.load(f))
except Exception: # noqa: E722
# reg/patched pickle
# compat not used in pandas/compat/pickle_compat.py::load
# TODO: remove except block OR modify pc.load to use compat
try:
return read_wrapper(
lambda f: pc.load(f, encoding=encoding, compat=False))
# compat pickle
except Exception: # noqa: E722
return read_wrapper(
lambda f: pc.load(f, encoding=encoding, compat=True))
def try_read(path, encoding=None):
# try with cPickle
# try with current pickle, if we have a Type Error then
# try with the compat pickle to handle subclass changes
# pass encoding only if its not None as py2 doesn't handle
# the param
# cpickle
# GH 6899
try:
with warnings.catch_warnings(record=True):
# We want to silence any warnings about, e.g. moved modules.
warnings.simplefilter("ignore", Warning)
return read_wrapper(lambda f: pkl.load(f))
except Exception: # noqa: E722
# reg/patched pickle
# compat not used in pandas/compat/pickle_compat.py::load
# TODO: remove except block OR modify pc.load to use compat
try:
return read_wrapper(
lambda f: pc.load(f, encoding=encoding, compat=False))
# compat pickle
except Exception: # noqa: E722
return read_wrapper(
lambda f: pc.load(f, encoding=encoding, compat=True))
def try_read(path, encoding=None):
# try with current pickle, if we have a Type Error then
# try with the compat pickle to handle subclass changes
# pass encoding only if its not None as py2 doesn't handle
# the param
try:
with open(path, 'rb') as fh:
return pc.load(fh, encoding=encoding, compat=False)
except:
with open(path, 'rb') as fh:
return pc.load(fh, encoding=encoding, compat=True)
def try_read(path, encoding=None):
# try with current pickle, if we have a Type Error then
# try with the compat pickle to handle subclass changes
# pass encoding only if its not None as py2 doesn't handle
# the param
try:
with open(path, 'rb') as fh:
return pc.load(fh, encoding=encoding, compat=False)
except:
with open(path, 'rb') as fh:
return pc.load(fh, encoding=encoding, compat=True)
def _db_to_dataframe(self):
"""
This is basically pd.read_pickle but allowing in-memory objects and
forcing python 3. The idea is that pandas.read_pickle maintains some semblance
of backward compatibility making this more robust. The code this is derived from
came from pandas 0.18 pandas.io.pickle.
"""
fh = BytesIO(self._dataframe)
encoding = 'latin1'
try:
self.dataframe = pickle.load(fh)
except (Exception) as e:
try:
# reg/patched pickle
self.dataframe = pc.load(fh, encoding=encoding, compat=False)
except:
# compat pickle
self.dataframe = pc.load(fh, encoding=encoding, compat=True)
def try_read(path, encoding=None):
# try with cPickle
# try with current pickle, if we have a Type Error then
# try with the compat pickle to handle subclass changes
# pass encoding only if its not None as py2 doesn't handle
# the param
# cpickle
# GH 6899
try:
return read_wrapper(lambda f: pkl.load(f))
except Exception:
# reg/patched pickle
try:
return read_wrapper(
lambda f: pc.load(f, encoding=encoding, compat=False))
# compat pickle
except:
return read_wrapper(
lambda f: pc.load(f, encoding=encoding, compat=True))
def try_read(path, encoding=None):
# try with cPickle
# try with current pickle, if we have a Type Error then
# try with the compat pickle to handle subclass changes
# pass encoding only if its not None as py2 doesn't handle
# the param
# cpickle
# GH 6899
try:
return read_wrapper(lambda f: pkl.load(f))
except Exception:
# reg/patched pickle
try:
return read_wrapper(
lambda f: pc.load(f, encoding=encoding, compat=False))
# compat pickle
except:
return read_wrapper(
lambda f: pc.load(f, encoding=encoding, compat=True))
def try_read(path, encoding=None):
# try with cPickle
# try with current pickle, if we have a Type Error then
# try with the compat pickle to handle subclass changes
# pass encoding only if its not None as py2 doesn't handle
# the param
# cpickle
# GH 6899
try:
with warnings.catch_warnings(record=True):
# We want to silencce any warnings about, e.g. moved modules.
return read_wrapper(lambda f: pkl.load(f))
except Exception:
# reg/patched pickle
try:
return read_wrapper(
lambda f: pc.load(f, encoding=encoding, compat=False))
# compat pickle
except:
return read_wrapper(
lambda f: pc.load(f, encoding=encoding, compat=True))
def try_read(path, encoding=None):
# try with cPickle
# try with current pickle, if we have a Type Error then
# try with the compat pickle to handle subclass changes
# pass encoding only if its not None as py2 doesn't handle
# the param
# cpickle
# GH 6899
try:
with warnings.catch_warnings(record=True):
# We want to silencce any warnings about, e.g. moved modules.
return read_wrapper(lambda f: pkl.load(f))
except Exception:
# reg/patched pickle
try:
return read_wrapper(
lambda f: pc.load(f, encoding=encoding, compat=False))
# compat pickle
except:
return read_wrapper(
lambda f: pc.load(f, encoding=encoding, compat=True))
def try_read(path, encoding=None):
# try with cPickle
# try with current pickle, if we have a Type Error then
# try with the compat pickle to handle subclass changes
# pass encoding only if its not None as py2 doesn't handle
# the param
# cpickle
# GH 6899
try:
with open(path, 'rb') as fh:
return pkl.load(fh)
except (Exception) as e:
# reg/patched pickle
try:
with open(path, 'rb') as fh:
return pc.load(fh, encoding=encoding, compat=False)
# compat pickle
except:
with open(path, 'rb') as fh:
return pc.load(fh, encoding=encoding, compat=True)
def try_read(path, encoding=None):
# try with cPickle
# try with current pickle, if we have a Type Error then
# try with the compat pickle to handle subclass changes
# pass encoding only if its not None as py2 doesn't handle
# the param
# cpickle
# GH 6899
try:
with open(path, 'rb') as fh:
return pkl.load(fh)
except (Exception) as e:
# reg/patched pickle
try:
with open(path, 'rb') as fh:
return pc.load(fh, encoding=encoding, compat=False)
# compat pickle
except:
with open(path, 'rb') as fh:
return pc.load(fh, encoding=encoding, compat=True)
def read_dataframe_from_s3( bucket, key, encoding=None ):
"""
Returns a dataframe from the given bucket and key (where df on s3 is a pickle)
"""
conn = boto.connect_s3()
with tempfile.SpooledTemporaryFile() as t:
k = Key(conn.get_bucket(bucket))
k.key = key
k.get_contents_to_file( t )
t.seek(0)
try:
try:
pkl.dump(data_frame, t)#from actual to_pickle code
except:
logger = logging.getLogger('write2df')
logger.exception("DFsize : %s" % sizeof_df( data_frame ))
raise
except:
try:
t.seek(0)
return pc.load(t, encoding=encoding, compat=False)
except:
t.seek(0)
return pc.load(t, encoding=encoding, compat=True)
def __call__(self, method, **kwargs):
kwargs["timeout"] = self.request_timeout
request = thrift.St_Query_Req()
request.method_name = method
request.params = msgpack.packb(kwargs)
err, result = None, None
for idx in range(self.request_attempt_count):
try:
file = six.BytesIO()
self.ensure_auth()
response = self.client.query(request)
if response.status:
buffer = response.msg
if six.PY3:
if type(buffer) is str:
buffer = bytes(buffer, "ascii")
buffer = zlib.decompress(buffer)
file.write(buffer)
pickle_encoding = None
if six.PY3:
pickle_encoding = "latin1"
result = pc.load(file, encoding=pickle_encoding)
else:
err = self.get_error(response)
break
except KeyboardInterrupt as e:
self._reset()
err = e
raise
except socket_error as e:
self._reset()
err = e
if idx < self.request_attempt_count - 1:
print("网络错误：'{}', 将进行重试".format(e))
continue
except Exception as e:
self._reset()
err = e
break
finally:
file.close()
if result is None:
if isinstance(err, Exception):
raise err
return result
def compare(self, vf):
# py3 compat when reading py2 pickle
try:
with open(vf,'rb') as fh:
data = pickle.load(fh)
except ValueError as detail:
# we are trying to read a py3 pickle in py2.....
return
# we have a deprecated klass
except TypeError as detail:
from pandas.compat.pickle_compat import load
data = load(vf)
except:
if not compat.PY3:
raise
with open(vf,'rb') as fh:
data = pickle.load(fh, encoding='latin1')
for typ, dv in data.items():
for dt, result in dv.items():
expected = self.data[typ][dt]
if isinstance(expected,Index):
self.assert_(expected.equals(result))
continue
if typ.startswith('sp_'):
comparator = getattr(test_sparse,"assert_%s_equal" % typ)
comparator(result,expected,exact_indices=False)
else:
comparator = getattr(tm,"assert_%s_equal" % typ)
comparator(result,expected)
def compare(self, vf):
# py3 compat when reading py2 pickle
try:
with open(vf, 'rb') as fh:
data = pickle.load(fh)
except ValueError as detail:
# we are trying to read a py3 pickle in py2.....
return
# we have a deprecated klass
except TypeError as detail:
from pandas.compat.pickle_compat import load
data = load(vf)
except:
if not compat.PY3:
raise
with open(vf, 'rb') as fh:
data = pickle.load(fh, encoding='latin1')
for typ, dv in data.items():
for dt, result in dv.items():
expected = self.data[typ][dt]
if isinstance(expected, Index):
self.assert_(expected.equals(result))
continue
if typ.startswith('sp_'):
comparator = getattr(test_sparse, "assert_%s_equal" % typ)
comparator(result, expected, exact_indices=False)
else:
comparator = getattr(tm, "assert_%s_equal" % typ)
comparator(result, expected)
def read_pickle(
filepath_or_buffer: FilePath | ReadPickleBuffer,
compression: CompressionOptions = "infer",
storage_options: StorageOptions = None,
):
"""
Load pickled pandas object (or any object) from file.
.. warning::
Loading pickled data received from untrusted sources can be
unsafe. See `here <https://docs.python.org/3/library/pickle.html>`__.
Parameters
----------
filepath_or_buffer : str, path object, or file-like object
String, path object (implementing ``os.PathLike[str]``), or file-like
object implementing a binary ``readlines()`` function.
.. versionchanged:: 1.0.0
Accept URL. URL is not limited to S3 and GCS.
{decompression_options}
.. versionchanged:: 1.4.0 Zstandard support.
{storage_options}
.. versionadded:: 1.2.0
Returns
-------
unpickled : same type as object stored in file
See Also
--------
DataFrame.to_pickle : Pickle (serialize) DataFrame object to file.
Series.to_pickle : Pickle (serialize) Series object to file.
read_hdf : Read HDF5 file into a DataFrame.
read_sql : Read SQL query or database table into a DataFrame.
read_parquet : Load a parquet object, returning a DataFrame.
Notes
-----
read_pickle is only guaranteed to be backwards compatible to pandas 0.20.3.
Examples
--------
>>> original_df = pd.DataFrame({{"foo": range(5), "bar": range(5, 10)}}) # doctest: +SKIP
>>> original_df # doctest: +SKIP
foo bar
0 0 5
1 1 6
2 2 7
3 3 8
4 4 9
>>> pd.to_pickle(original_df, "./dummy.pkl") # doctest: +SKIP
>>> unpickled_df = pd.read_pickle("./dummy.pkl") # doctest: +SKIP
>>> unpickled_df # doctest: +SKIP
foo bar
0 0 5
1 1 6
2 2 7
3 3 8
4 4 9
""" # noqa: E501
excs_to_catch = (AttributeError, ImportError, ModuleNotFoundError,
TypeError)
with get_handle(
filepath_or_buffer,
"rb",
compression=compression,
is_text=False,
storage_options=storage_options,
) as handles:
# 1) try standard library Pickle
# 2) try pickle_compat (older pandas version) to handle subclass changes
# 3) try pickle_compat with latin-1 encoding upon a UnicodeDecodeError
try:
# TypeError for Cython complaints about object.__new__ vs Tick.__new__
try:
with warnings.catch_warnings(record=True):
# We want to silence any warnings about, e.g. moved modules.
warnings.simplefilter("ignore", Warning)
return pickle.load(handles.handle)
except excs_to_catch:
# e.g.
# "No module named 'pandas.core.sparse.series'"
# "Can't get attribute '__nat_unpickle' on <module 'pandas._libs.tslib"
return pc.load(handles.handle, encoding=None)
except UnicodeDecodeError:
# e.g. can occur for files written in py27; see GH#28645 and GH#31988
return pc.load(handles.handle, encoding="latin-1")
def read_pickle(
filepath_or_buffer: FilePathOrBuffer,
compression: CompressionOptions = "infer",
storage_options: StorageOptions = None,
):
"""
Load pickled pandas object (or any object) from file.
.. warning::
Loading pickled data received from untrusted sources can be
unsafe. See `here <https://docs.python.org/3/library/pickle.html>`__.
Parameters
----------
filepath_or_buffer : str, path object or file-like object
File path, URL, or buffer where the pickled object will be loaded from.
.. versionchanged:: 1.0.0
Accept URL. URL is not limited to S3 and GCS.
compression : {{'infer', 'gzip', 'bz2', 'zip', 'xz', None}}, default 'infer'
If 'infer' and 'path_or_url' is path-like, then detect compression from
the following extensions: '.gz', '.bz2', '.zip', or '.xz' (otherwise no
compression) If 'infer' and 'path_or_url' is not path-like, then use
None (= no decompression).
{storage_options}
.. versionadded:: 1.2.0
Returns
-------
unpickled : same type as object stored in file
See Also
--------
DataFrame.to_pickle : Pickle (serialize) DataFrame object to file.
Series.to_pickle : Pickle (serialize) Series object to file.
read_hdf : Read HDF5 file into a DataFrame.
read_sql : Read SQL query or database table into a DataFrame.
read_parquet : Load a parquet object, returning a DataFrame.
Notes
-----
read_pickle is only guaranteed to be backwards compatible to pandas 0.20.3.
Examples
--------
>>> original_df = pd.DataFrame({{"foo": range(5), "bar": range(5, 10)}})
>>> original_df
foo bar
0 0 5
1 1 6
2 2 7
3 3 8
4 4 9
>>> pd.to_pickle(original_df, "./dummy.pkl")
>>> unpickled_df = pd.read_pickle("./dummy.pkl")
>>> unpickled_df
foo bar
0 0 5
1 1 6
2 2 7
3 3 8
4 4 9
>>> import os
>>> os.remove("./dummy.pkl")
"""
excs_to_catch = (AttributeError, ImportError, ModuleNotFoundError,
TypeError)
with get_handle(
filepath_or_buffer,
"rb",
compression=compression,
is_text=False,
storage_options=storage_options,
) as handles:
# 1) try standard library Pickle
# 2) try pickle_compat (older pandas version) to handle subclass changes
# 3) try pickle_compat with latin-1 encoding upon a UnicodeDecodeError
try:
# TypeError for Cython complaints about object.__new__ vs Tick.__new__
try:
with warnings.catch_warnings(record=True):
# We want to silence any warnings about, e.g. moved modules.
warnings.simplefilter("ignore", Warning)
return pickle.load(
handles.handle) # type: ignore[arg-type]
except excs_to_catch:
# e.g.
# "No module named 'pandas.core.sparse.series'"
# "Can't get attribute '__nat_unpickle' on <module 'pandas._libs.tslib"
return pc.load(handles.handle, encoding=None)
except UnicodeDecodeError:
# e.g. can occur for files written in py27; see GH#28645 and GH#31988
return pc.load(handles.handle, encoding="latin-1")
def read_pickle(path, compression='infer'):
"""
Load pickled pandas object (or any object) from file.
.. warning::
Loading pickled data received from untrusted sources can be
unsafe. See `here <https://docs.python.org/3/library/pickle.html>`__.
Parameters
----------
path : str
File path where the pickled object will be loaded.
compression : {'infer', 'gzip', 'bz2', 'zip', 'xz', None}, default 'infer'
For on-the-fly decompression of on-disk data. If 'infer', then use
gzip, bz2, xz or zip if path ends in '.gz', '.bz2', '.xz',
or '.zip' respectively, and no decompression otherwise.
Set to None for no decompression.
.. versionadded:: 0.20.0
Returns
-------
unpickled : same type as object stored in file
See Also
--------
DataFrame.to_pickle : Pickle (serialize) DataFrame object to file.
Series.to_pickle : Pickle (serialize) Series object to file.
read_hdf : Read HDF5 file into a DataFrame.
read_sql : Read SQL query or database table into a DataFrame.
read_parquet : Load a parquet object, returning a DataFrame.
Examples
--------
>>> original_df = pd.DataFrame({"foo": range(5), "bar": range(5, 10)})
>>> original_df
foo bar
0 0 5
1 1 6
2 2 7
3 3 8
4 4 9
>>> pd.to_pickle(original_df, "./dummy.pkl")
>>> unpickled_df = pd.read_pickle("./dummy.pkl")
>>> unpickled_df
foo bar
0 0 5
1 1 6
2 2 7
3 3 8
4 4 9
>>> import os
>>> os.remove("./dummy.pkl")
"""
path = _stringify_path(path)
f, fh = _get_handle(path, 'rb', compression=compression, is_text=False)
# 1) try with cPickle
# 2) try with the compat pickle to handle subclass changes
# 3) pass encoding only if its not None as py2 doesn't handle the param
try:
with warnings.catch_warnings(record=True):
# We want to silence any warnings about, e.g. moved modules.
warnings.simplefilter("ignore", Warning)
return pkl.load(f)
except Exception: # noqa: E722
try:
return pc.load(f, encoding=None)
except Exception: # noqa: E722
if PY3:
return pc.load(f, encoding='latin1')
raise
finally:
f.close()
for _f in fh:
_f.close()
def _read_pickle(vf, encoding=None, compat=False):
from pandas.compat import pickle_compat as pc
with open(vf,'rb') as fh:
pc.load(fh, encoding=encoding, compat=compat)
def read_pickle(
filepath_or_buffer: FilePathOrBuffer,
compression: CompressionOptions = "infer",
storage_options: StorageOptions = None,
):
"""
Load pickled pandas object (or any object) from file.
.. warning::
Loading pickled data received from untrusted sources can be
unsafe. See `here <https://docs.python.org/3/library/pickle.html>`__.
Parameters
----------
filepath_or_buffer : str, path object or file-like object
File path, URL, or buffer where the pickled object will be loaded from.
.. versionchanged:: 1.0.0
Accept URL. URL is not limited to S3 and GCS.
compression : {'infer', 'gzip', 'bz2', 'zip', 'xz', None}, default 'infer'
If 'infer' and 'path_or_url' is path-like, then detect compression from
the following extensions: '.gz', '.bz2', '.zip', or '.xz' (otherwise no
compression) If 'infer' and 'path_or_url' is not path-like, then use
None (= no decompression).
storage_options : dict, optional
Extra options that make sense for a particular storage connection, e.g.
host, port, username, password, etc., if using a URL that will
be parsed by ``fsspec``, e.g., starting "s3://", "gcs://". An error
will be raised if providing this argument with a local path or
a file-like buffer. See the fsspec and backend storage implementation
docs for the set of allowed keys and values
.. versionadded:: 1.2.0
Returns
-------
unpickled : same type as object stored in file
See Also
--------
DataFrame.to_pickle : Pickle (serialize) DataFrame object to file.
Series.to_pickle : Pickle (serialize) Series object to file.
read_hdf : Read HDF5 file into a DataFrame.
read_sql : Read SQL query or database table into a DataFrame.
read_parquet : Load a parquet object, returning a DataFrame.
Notes
-----
read_pickle is only guaranteed to be backwards compatible to pandas 0.20.3.
Examples
--------
>>> original_df = pd.DataFrame({"foo": range(5), "bar": range(5, 10)})
>>> original_df
foo bar
0 0 5
1 1 6
2 2 7
3 3 8
4 4 9
>>> pd.to_pickle(original_df, "./dummy.pkl")
>>> unpickled_df = pd.read_pickle("./dummy.pkl")
>>> unpickled_df
foo bar
0 0 5
1 1 6
2 2 7
3 3 8
4 4 9
>>> import os
>>> os.remove("./dummy.pkl")
"""
ioargs = get_filepath_or_buffer(
filepath_or_buffer, compression=compression, storage_options=storage_options
)
compression = ioargs.compression
if not isinstance(ioargs.filepath_or_buffer, str) and compression == "infer":
compression = None
f, fh = get_handle(
ioargs.filepath_or_buffer, "rb", compression=compression, is_text=False
)
# 1) try standard library Pickle
# 2) try pickle_compat (older pandas version) to handle subclass changes
# 3) try pickle_compat with latin-1 encoding upon a UnicodeDecodeError
try:
excs_to_catch = (AttributeError, ImportError, ModuleNotFoundError, TypeError)
# TypeError for Cython complaints about object.__new__ vs Tick.__new__
try:
with warnings.catch_warnings(record=True):
# We want to silence any warnings about, e.g. moved modules.
warnings.simplefilter("ignore", Warning)
return pickle.load(f)
except excs_to_catch:
# e.g.
# "No module named 'pandas.core.sparse.series'"
# "Can't get attribute '__nat_unpickle' on <module 'pandas._libs.tslib"
return pc.load(f, encoding=None)
except UnicodeDecodeError:
# e.g. can occur for files written in py27; see GH#28645 and GH#31988
return pc.load(f, encoding="latin-1")
finally:
if f != filepath_or_buffer:
# do not close user-provided file objects GH 35679
f.close()
for _f in fh:
_f.close()
if ioargs.should_close:
assert not isinstance(ioargs.filepath_or_buffer, str)
try:
ioargs.filepath_or_buffer.close()
except ValueError:
pass
def read_pickle(path, compression="infer"):
"""
Load pickled pandas object (or any object) from file.
.. warning::
Loading pickled data received from untrusted sources can be
unsafe. See `here <https://docs.python.org/3/library/pickle.html>`__.
Parameters
----------
path : str
File path where the pickled object will be loaded.
compression : {'infer', 'gzip', 'bz2', 'zip', 'xz', None}, default 'infer'
For on-the-fly decompression of on-disk data. If 'infer', then use
gzip, bz2, xz or zip if path ends in '.gz', '.bz2', '.xz',
or '.zip' respectively, and no decompression otherwise.
Set to None for no decompression.
Returns
-------
unpickled : same type as object stored in file
See Also
--------
DataFrame.to_pickle : Pickle (serialize) DataFrame object to file.
Series.to_pickle : Pickle (serialize) Series object to file.
read_hdf : Read HDF5 file into a DataFrame.
read_sql : Read SQL query or database table into a DataFrame.
read_parquet : Load a parquet object, returning a DataFrame.
Notes
-----
read_pickle is only guaranteed to be backwards compatible to pandas 0.20.3.
Examples
--------
>>> original_df = pd.DataFrame({"foo": range(5), "bar": range(5, 10)})
>>> original_df
foo bar
0 0 5
1 1 6
2 2 7
3 3 8
4 4 9
>>> pd.to_pickle(original_df, "./dummy.pkl")
>>> unpickled_df = pd.read_pickle("./dummy.pkl")
>>> unpickled_df
foo bar
0 0 5
1 1 6
2 2 7
3 3 8
4 4 9
>>> import os
>>> os.remove("./dummy.pkl")
"""
path = _stringify_path(path)
f, fh = _get_handle(path, "rb", compression=compression, is_text=False)
# 1) try standard library Pickle
# 2) try pickle_compat (older pandas version) to handle subclass changes
excs_to_catch = (AttributeError, ImportError)
if PY36:
excs_to_catch += (ModuleNotFoundError,)
try:
with warnings.catch_warnings(record=True):
# We want to silence any warnings about, e.g. moved modules.
warnings.simplefilter("ignore", Warning)
return pickle.load(f)
except excs_to_catch:
# e.g.
# "No module named 'pandas.core.sparse.series'"
# "Can't get attribute '__nat_unpickle' on <module 'pandas._libs.tslib"
return pc.load(f, encoding=None)
except UnicodeDecodeError:
# e.g. can occur for files written in py27; see GH#28645
return pc.load(f, encoding="latin-1")
finally:
f.close()
for _f in fh:
_f.close()
def read_pickle(path, compression='infer'):
"""
Load pickled pandas object (or any object) from file.
.. warning::
Loading pickled data received from untrusted sources can be
unsafe. See `here <https://docs.python.org/3/library/pickle.html>`__.
Parameters
----------
path : str
File path where the pickled object will be loaded.
compression : {'infer', 'gzip', 'bz2', 'zip', 'xz', None}, default 'infer'
For on-the-fly decompression of on-disk data. If 'infer', then use
gzip, bz2, xz or zip if path ends in '.gz', '.bz2', '.xz',
or '.zip' respectively, and no decompression otherwise.
Set to None for no decompression.
.. versionadded:: 0.20.0
Returns
-------
unpickled : same type as object stored in file
See Also
--------
DataFrame.to_pickle : Pickle (serialize) DataFrame object to file.
Series.to_pickle : Pickle (serialize) Series object to file.
read_hdf : Read HDF5 file into a DataFrame.
read_sql : Read SQL query or database table into a DataFrame.
read_parquet : Load a parquet object, returning a DataFrame.
Notes
-----
read_pickle is only guaranteed to be backwards compatible to pandas 0.20.3.
Examples
--------
>>> original_df = pd.DataFrame({"foo": range(5), "bar": range(5, 10)})
>>> original_df
foo bar
0 0 5
1 1 6
2 2 7
3 3 8
4 4 9
>>> pd.to_pickle(original_df, "./dummy.pkl")
>>> unpickled_df = pd.read_pickle("./dummy.pkl")
>>> unpickled_df
foo bar
0 0 5
1 1 6
2 2 7
3 3 8
4 4 9
>>> import os
>>> os.remove("./dummy.pkl")
"""
path = _stringify_path(path)
f, fh = _get_handle(path, 'rb', compression=compression, is_text=False)
# 1) try standard libary Pickle
# 2) try pickle_compat (older pandas version) to handle subclass changes
# 3) try pickle_compat with latin1 encoding
try:
with warnings.catch_warnings(record=True):
# We want to silence any warnings about, e.g. moved modules.
warnings.simplefilter("ignore", Warning)
return pickle.load(f)
except Exception: # noqa: E722
try:
return pc.load(f, encoding=None)
except Exception: # noqa: E722
return pc.load(f, encoding='latin1')
finally:
f.close()
for _f in fh:
_f.close()
def _read_pickle(vf, encoding=None, compat=False):
from pandas.compat import pickle_compat as pc
with open(vf, 'rb') as fh:
pc.load(fh, encoding=encoding, compat=compat)
