def put(self, obj):
"""Serialize and hash something pickleable, returning a unique key to retrieve it later.
NB: pickle by default memoizes objects by id and pickle repeated objects by references,
for example, (A, A) uses less space than (A, A'), A and A' are equal but not identical.
For content addressability we need equality. Use `fast` mode to turn off memo.
Longer term see https://github.com/pantsbuild/pants/issues/2969
"""
try:
with closing(StringIO.StringIO()) as buf:
pickler = pickle.Pickler(buf, protocol=self._protocol)
pickler.fast = 1
pickler.dump(obj)
blob = buf.getvalue()
# Hash the blob and store it if it does not exist.
key = Key.create(blob)
if key not in self._objects:
self._objects[key] = obj
except Exception as e:
# Unfortunately, pickle can raise things other than PickleError instances. For example it
# will raise ValueError when handed a lambda; so we handle the otherwise overly-broad
# `Exception` type here.
raise SerializationError('Failed to pickle {}: {}'.format(obj, e),
e)
return key
def _try_pickle(obj):
try:
pickle.dumps(obj, protocol=pickle.HIGHEST_PROTOCOL)
except Exception as e:
# Unfortunately, pickle can raise things other than PickleError instances. For example it
# will raise ValueError when handed a lambda; so we handle the otherwise overly-broad
# `Exception` type here.
raise SerializationError('Failed to pickle {}: {}'.format(obj, e))