def _export(obj, fp, extensions_map, extension, overwrite):
if isinstance(fp, Path):
fp = str(fp) # cheeky conversion to string to reuse existing code
if isinstance(fp, basestring):
path_filepath = _validate_filepath(fp, extension, overwrite)
export_function = _extension_to_export_function(
path_filepath.suffix, extensions_map)
with path_filepath.open('wb') as file_handle:
export_function(obj, file_handle)
else:
# You MUST provide an extension if a file handle is given
if extension is None:
raise ValueError('An export file extension must be provided if a '
'file-like object is passed.')
# Apparently in Python 2.x there is no reliable way to detect something
# that is 'file' like (file handle or a StringIO object or something
# you can read and write to like a file). Therefore, we are going to
# just be really Pythonic about it and just assume we were handed
# a correctly behaving object.
try:
# Follow PIL like behaviour. Check the file handle extension
# and check if matches the given extension
_validate_filepath(fp.name, extension, overwrite)
except AttributeError:
pass
export_function = _extension_to_export_function(
_normalise_extension(extension), extensions_map)
export_function(obj, fp)
def _export(obj, fp, extensions_map, extension, overwrite):
if isinstance(fp, Path):
fp = str(fp) # cheeky conversion to string to reuse existing code
if isinstance(fp, basestring):
path_filepath = _validate_filepath(fp, extension, overwrite)
export_function = _extension_to_export_function(
path_filepath.suffix, extensions_map)
with path_filepath.open('wb') as file_handle:
export_function(obj, file_handle)
else:
# You MUST provide an extension if a file handle is given
if extension is None:
raise ValueError('An export file extension must be provided if a '
'file-like object is passed.')
# Apparently in Python 2.x there is no reliable way to detect something
# that is 'file' like (file handle or a StringIO object or something
# you can read and write to like a file). Therefore, we are going to
# just be really Pythonic about it and just assume we were handed
# a correctly behaving object.
try:
# Follow PIL like behaviour. Check the file handle extension
# and check if matches the given extension
_validate_filepath(fp.name, extension, overwrite)
except AttributeError:
pass
export_function = _extension_to_export_function(
_normalise_extension(extension), extensions_map)
export_function(obj, fp)
def export_pickle(obj, fp, overwrite=False):
r"""
Exports a given collection of Python objects with Pickle.
The ``fp`` argument can be either a `str` or any Python type that acts like
a file.
If ``fp`` is a path, it must have the suffix `.pkl` or `.pkl.gz`. If
`.pkl`, the object will be pickled using Pickle protocol 2 without
compression. If `.pkl.gz` the object will be pickled using Pickle protocol
2 with gzip compression (at a fixed compression level of 3).
Note that a special exception is made for `pathlib.Path` objects - they
are pickled down as a `pathlib.PurePath` so that pickles can be easily
moved between different platforms.
Parameters
----------
obj : ``object``
The object to export.
fp : `str` or `file`-like object
The string path or file-like object to save the object at/into.
overwrite : `bool`, optional
Whether or not to overwrite a file if it already exists.
Raises
------
ValueError
File already exists and ``overwrite`` != ``True``
ValueError
``fp`` is a `file`-like object and ``extension`` is
``None``
ValueError
The provided extension does not match to an existing exporter type
(the output type is not supported).
"""
if isinstance(fp, Path):
fp = str(fp) # cheeky conversion to string to reuse existing code
if isinstance(fp, basestring):
# user provided a path - if it ended .gz we will compress
path_filepath = _validate_filepath(fp, '.pkl', overwrite)
o = gzip_open if path_filepath.suffix == '.gz' else open
with o(str(path_filepath), 'wb') as f:
# force overwrite as True we've already done the check above
_export(obj, f, pickle_types, '.pkl', True)
else:
_export(obj, fp, pickle_types, '.pkl', overwrite)
def export_pickle(obj, fp, overwrite=False, protocol=2):
r"""
Exports a given collection of Python objects with Pickle.
The ``fp`` argument can be either a `Path` or any Python type that acts like
a file.
If ``fp`` is a path, it must have the suffix `.pkl` or `.pkl.gz`. If
`.pkl`, the object will be pickled using Pickle protocol 2 without
compression. If `.pkl.gz` the object will be pickled using Pickle protocol
2 with gzip compression (at a fixed compression level of 3).
Note that a special exception is made for `pathlib.Path` objects - they
are pickled down as a `pathlib.PurePath` so that pickles can be easily
moved between different platforms.
Parameters
----------
obj : ``object``
The object to export.
fp : `Path` or `file`-like object
The string path or file-like object to save the object at/into.
overwrite : `bool`, optional
Whether or not to overwrite a file if it already exists.
protocol : `int`, optional
The Pickle protocol used to serialize the file.
The protocols were introduced in different versions of python, thus
it is recommended to save with the highest protocol version that
your python distribution can support.
The protocol refers to:
========= =========================================================
Protocol Functionality
========= =========================================================
0 Simplest protocol for text mode, backwards compatible.
1 Protocol for binary mode, backwards compatible.
2 Wider support for classes, compatible with python >= 2.3.
3 Support for byte objects, compatible with python >= 3.0.
4 Support for large objects, compatible with python >= 3.4.
========= =========================================================
Raises
------
ValueError
File already exists and ``overwrite`` != ``True``
ValueError
``fp`` is a `file`-like object and ``extension`` is
``None``
ValueError
The provided extension does not match to an existing exporter type
(the output type is not supported).
"""
if isinstance(fp, basestring):
fp = Path(fp) # cheeky conversion to Path to reuse existing code
if isinstance(fp, Path):
# user provided a path - if it ended .gz we will compress
path_filepath = _validate_filepath(fp, overwrite)
extension = _parse_and_validate_extension(path_filepath, None,
pickle_types)
o = gzip_open if extension[-3:] == '.gz' else open
with o(str(path_filepath), 'wb') as f:
# force overwrite as True we've already done the check above
_export(obj, f, pickle_types, extension, True, protocol=protocol)
else:
_export(obj, fp, pickle_types, '.pkl', overwrite, protocol=protocol)
def export_pickle(obj, fp, overwrite=False, protocol=2):
r"""
Exports a given collection of Python objects with Pickle.
The ``fp`` argument can be either a `Path` or any Python type that acts like
a file.
If ``fp`` is a path, it must have the suffix `.pkl` or `.pkl.gz`. If
`.pkl`, the object will be pickled using the selected Pickle protocol.
If `.pkl.gz` the object will be pickled using the selected Pickle
protocol with gzip compression (at a fixed compression level of 3).
Note that a special exception is made for `pathlib.Path` objects - they
are pickled down as a `pathlib.PurePath` so that pickles can be easily
moved between different platforms.
Parameters
----------
obj : ``object``
The object to export.
fp : `Path` or `file`-like object
The string path or file-like object to save the object at/into.
overwrite : `bool`, optional
Whether or not to overwrite a file if it already exists.
protocol : `int`, optional
The Pickle protocol used to serialize the file.
The protocols were introduced in different versions of python, thus
it is recommended to save with the highest protocol version that
your python distribution can support.
The protocol refers to:
========= =========================================================
Protocol Functionality
========= =========================================================
0 Simplest protocol for text mode, backwards compatible.
1 Protocol for binary mode, backwards compatible.
2 Wider support for classes, compatible with python >= 2.3.
3 Support for byte objects, compatible with python >= 3.0.
4 Support for large objects, compatible with python >= 3.4.
========= =========================================================
Raises
------
ValueError
File already exists and ``overwrite`` != ``True``
ValueError
``fp`` is a `file`-like object and ``extension`` is
``None``
ValueError
The provided extension does not match to an existing exporter type
(the output type is not supported).
"""
exporter_kwargs = {'protocol': protocol}
if isinstance(fp, basestring):
fp = Path(fp) # cheeky conversion to Path to reuse existing code
if isinstance(fp, Path):
# user provided a path - if it ended .gz we will compress
path_filepath = _validate_filepath(fp, overwrite)
extension = _parse_and_validate_extension(path_filepath, None,
pickle_types)
o = gzip_open if extension[-3:] == '.gz' else open
with o(str(path_filepath), 'wb') as f:
# force overwrite as True we've already done the check above
_export(obj, f, pickle_types, extension, True,
exporter_kwargs=exporter_kwargs)
else:
_export(obj, fp, pickle_types, '.pkl', overwrite,
exporter_kwargs=exporter_kwargs)
