def override_pickler_hooks(extend=True):
""" Extends the dill library hooks into that of the standard pickler library.
If false all hooks that dill overrides will be removed.
If true dill hooks will be injected into the pickler library dispatch_table.
"""
dill.extend(extend)
def serializeFn(self):
import dill as pickle
#see https://github.com/SeldonIO/alibi/issues/447#issuecomment-881552005
from dill import extend
extend(use_dill=False)
byterep = pickle.dumps(self.fn)
byterep = autoclass("java.nio.ByteBuffer").wrap(byterep)
return byterep
def new_callable_wmodel(byterep):
import dill as pickle
from dill import extend
#see https://github.com/SeldonIO/alibi/issues/447#issuecomment-881552005
extend(use_dill=False)
fn = pickle.loads(byterep)
#we need to prevent these functions from being GCd.
global SAVED_FNS
SAVED_FNS.append(fn)
from .batchretrieve import _function2wmodel
callback, wmodel = _function2wmodel(fn)
SAVED_FNS.append(callback)
#print("Stored lambda fn %s and callback in SAVED_FNS, now %d stored" % (str(fn), len(SAVED_FNS)))
return wmodel
def test_callable_wmodel_dunders(self):
testPosting = pt.autoclass(
"org.terrier.structures.postings.BasicPostingImpl")(0, 1)
from pyterrier.batchretrieve import _function2wmodel
lambdafn = lambda keyFreq, posting, entryStats, collStats: posting.getFrequency(
)
callback, wmodel = _function2wmodel(lambdafn)
from pyterrier.bootstrap import javabytebuffer2array
byterep = javabytebuffer2array(wmodel.scoringClass.serializeFn())
import dill as pickle
from dill import extend
#see https://github.com/SeldonIO/alibi/issues/447#issuecomment-881552005
extend(use_dill=False)
fn = pickle.loads(byterep)
self.assertEqual(
lambdafn(1, testPosting, None, None),
fn(1, testPosting, None, None),
)
wmodel.__getstate__()
rtr = wmodel.__reduce__()
#check the byte array is picklable
pickle.dumps(rtr[1][0])
#check object is picklable
pickle.dumps(wmodel)
#check can be unpickled too
wmodel2 = pickle.loads(pickle.dumps(wmodel))
score1 = wmodel.score(testPosting)
score2 = wmodel2.score(testPosting)
self.assertEqual(score1, score2)
#check newly unpickled can still be pickled
pickle.dumps(wmodel2)
wmodel3 = pickle.loads(pickle.dumps(wmodel2))
score3 = wmodel3.score(testPosting)
self.assertEqual(score1, score3)
import warnings
from typing import Callable, Dict, Iterator, Optional, Tuple, TypeVar
from torch.utils.data import IterDataPipe, functional_datapipe
from torch.utils.data.datapipes.dataframe import dataframe_wrapper as df_wrapper
T_co = TypeVar('T_co', covariant=True)
try:
import dill
# XXX: By default, dill writes the Pickler dispatch table to inject its
# own logic there. This globally affects the behavior of the standard library
# pickler for any user who transitively depends on this module!
# Undo this extension to avoid altering the behavior of the pickler globally.
dill.extend(use_dill=False)
DILL_AVAILABLE = True
except ImportError:
DILL_AVAILABLE = False
@functional_datapipe('filter')
class FilterIterDataPipe(IterDataPipe[T_co]):
r""" :class:`FilterIterDataPipe`.
Iterable DataPipe to filter elements from datapipe according to filter_fn.
Args:
datapipe: Iterable DataPipe being filtered
filter_fn: Customized function mapping an element to a boolean.
fn_args: Positional arguments for `filter_fn`
