def read_dataset(data_dir):
pickle_filename = "humanseg.pickle"
pickle_filepath = os.path.join(data_dir, pickle_filename)
if not os.path.exists(pickle_filepath):
pickle_filename = "humanseg.pickle"
pickle_filepath = os.path.join(data_dir, pickle_filename)
result = create_image_lists(data_dir)
print ("Pickling ...")
with open(pickle_filepath, 'wb') as f:
pickle.dump(result, f)
else:
print ("Found pickle file!")
open(pickle_filepath, 'a').close()
scores = {}
try:
with open(pickle_filepath, "rb") as file:
unpickler = pickle.Unpickler(file)
scores = unpickler.load()
if not isinstance(scores, dict):
scores = {}
except EOFError:
return {}
with open(pickle_filepath, 'rb') as f:
result = pickle.load(f)
training_records = result['training']
validation_records = result['val']
del result
# print(training_records[:10])
return training_records, validation_records
def python_memcache_deserializer(key, value, flags):
if flags == 0:
return value
elif flags & FLAG_TEXT:
return value.decode('utf8')
elif flags & FLAG_INTEGER:
return int(value)
elif flags & FLAG_LONG:
if six.PY3:
return int(value)
else:
return long_type(value)
elif flags & FLAG_PICKLE:
try:
buf = BytesIO(value)
unpickler = pickle.Unpickler(buf)
return unpickler.load()
except Exception:
logging.info('Pickle error', exc_info=True)
return None
return value
def _load_old_object(directory): # type: (str) -> Tuple[str, Dict[str, List[bytes]]] with open(os.path.join(directory, 'tmp', 'old_dn'), 'rb') as fd: p = pickle.Unpickler(fd) (old_dn, old_object) = p.load() return (old_dn, old_object)
def _load(f, constructors=None):
'''
constructors: A dictionary mapping strings to callables, which are
consulted as additional constructors during unpickling. This is useful
for interposing an alternative constructor, or for supporting class
names which have been changed. This is implemented via the cPickle
`Unpickler`'s `find_global` attribute, which is documented here,
albeit quite densely:
https://docs.python.org/2/library/pickle.html#subclassing-unpicklers
'''
from six.moves import cPickle as pickle
from baiji.serialization.util.importlib import class_from_str
def unpickler_find_global(module_name, class_name):
fully_qualified_name = '{}.{}'.format(module_name, class_name)
try:
return constructors[fully_qualified_name]
except KeyError:
# It would be tempting to delegate to the original `find_global`.
# Unfortunately, `Unpickler` does not expose the default
# implementation. Thanks, Obama.
return class_from_str(fully_qualified_name)
unpickler = pickle.Unpickler(f)
if constructors:
unpickler.find_global = unpickler_find_global
return unpickler.load()
def verify(self, key, tolerance=0.0):
"""
Verify data associated with the given key, with the specified tolerance. The
key is internally cast to a string.
"""
key = str(key)
if not key in self.__filenames:
raise CheckpointException("Missing checkpoint with key %s" % key)
if not isinstance(tolerance, float) or tolerance < 0.0:
raise InvalidArgumentException(
"tolerance must be a non-negative float")
handle = open(self.__filename(key), "rb")
pickler = pickle.Unpickler(handle)
c_cs = pickler.load()
try:
id_map = self.__id_map[key]
for c_id in c_cs:
c = id_map[c_id]
self._Checkpointer__verify(c, c_cs[c_id], tolerance=tolerance)
dolfin.info("Verified checkpoint with key %s" % key)
except CheckpointException as e:
dolfin.info(str(e))
raise CheckpointException(
"Failed to verify checkpoint with key %s" % key)
return
def restore(self, key, cs=None):
"""
Restore Constant s and Function s with the given key. If cs is supplied,
only restore Constant s and Function s found in cs. The key is internally
cast to a string.
"""
key = str(key)
if not key in self.__filenames:
raise CheckpointException("Missing checkpoint with key %s" % key)
if not cs is None:
cs = self._Checkpointer__check_cs(cs)
cs = [c.id() for c in cs]
handle = open(self.__filename(key), "rb")
pickler = pickle.Unpickler(handle)
c_cs = pickler.load()
if cs is None:
cs = list(c_cs.keys())
id_map = self.__id_map[key]
for c_id in cs:
c = id_map[c_id]
self._Checkpointer__unpack(c, c_cs[c_id])
return
def _unpickle_iterable(pickle_fh):
with pickle_fh:
unpklr = pickle.Unpickler(pickle_fh)
try:
while True:
yield unpklr.load()
except EOFError:
pass
def load_old(old):
# type: (dict) -> dict
if os.path.exists(DOVECOT_OLD_PICKLE):
with open(DOVECOT_OLD_PICKLE, "rb") as fd:
p = pickle.Unpickler(fd)
old = p.load()
os.unlink(DOVECOT_OLD_PICKLE)
return old
else:
return old
def test_broken_pickle_with_shared(self):
saves = []
def pers_save(obj):
if isinstance(obj, numpy.ndarray):
saves.append(obj)
return len(saves) - 1
else:
return None
def pers_load(id):
return saves[id]
a = numpy.random.rand(4, 5)
b = numpy.random.rand(5, 4)
x = theano.tensor.matrix()
y = theano.shared(b)
f = theano.function([x], theano.tensor.dot(x, y))
from theano.compat import BytesIO
fp = BytesIO()
p = pickle.Pickler(fp, 2)
p.persistent_id = pers_save
try:
p.dump(f)
except NotImplementedError as e:
if exc_message(e).startswith('DebugMode is not picklable'):
return
else:
raise
fp2 = BytesIO(fp.getvalue())
fp.close()
p = pickle.Unpickler(fp2)
p.persistent_load = pers_load
f2 = p.load()
fp2.close()
def load_datasets(pickle_filename):
with open(pickle_filename, 'rb') as f:
return (pickle.Unpickler(f).load())
def loads(cls, s):
obj = pickle.Unpickler(BytesIO(s))
obj.find_global = cls.find_class
return obj.load()
def handler(dn, new, old, command):
# type: (str, Optional[Dict[str, List[bytes]]], Optional[Dict[str, List[bytes]]], str) -> None
if os.path.exists(FETCHMAIL_OLD_PICKLE):
with open(FETCHMAIL_OLD_PICKLE, 'r') as fd:
p = pickle.Unpickler(fd)
old = p.load()
os.unlink(FETCHMAIL_OLD_PICKLE)
if command == 'r':
with open(FETCHMAIL_OLD_PICKLE, 'w+') as fd:
os.chmod(FETCHMAIL_OLD_PICKLE, 0o600)
p = pickle.Pickler(fd)
old = p.dump(old)
p.clear_memo()
flist = load_rc(fn_fetchmailrc)
if old and not new and not command == 'r':
# object has been deleted ==> remove entry from rc file
flist = objdelete(flist, old)
write_rc(flist, fn_fetchmailrc)
elif old and new and details_complete(old) and not details_complete(new):
# data is now incomplete ==> remove entry from rc file
flist = objdelete(flist, old)
write_rc(flist, fn_fetchmailrc)
elif new and details_complete(new):
# obj has been created or modified
passwd = None
if old:
# old exists ==> object has been modified ==> get old password and remove object entry from rc file
passwd = get_pw_from_rc(flist, old['uid'][0].decode('UTF-8'))
flist = objdelete(flist, old)
if not details_complete(new, incl_password=True):
if only_password_reset(old, new):
ud.debug(ud.LISTENER, ud.INFO,
'fetchmail: password has been reset - nothing to do')
# only password has been reset ==> nothing to do
return
# new obj does not contain password
if passwd:
# passwd has been set in old ==> use old password
ud.debug(ud.LISTENER, ud.INFO, 'fetchmail: using old password')
objappend(flist, new, passwd)
write_rc(flist, fn_fetchmailrc)
else:
ud.debug(
ud.LISTENER, ud.ERROR,
'fetchmail: user "%s": no password set in old and new' %
new['uid'][0])
else:
# new obj contains password ==> use new password
objappend(flist, new)
write_rc(flist, fn_fetchmailrc)
ud.debug(ud.LISTENER, ud.INFO, 'fetchmail: using new password')
configRegistry = univention.config_registry.ConfigRegistry()
configRegistry.load()
listener.setuid(0)
try:
lo = univention.uldap.getMachineConnection()
modlist = [('univentionFetchmailPasswd',
new['univentionFetchmailPasswd'][0], b"")]
lo.modify(dn, modlist)
ud.debug(ud.LISTENER, ud.INFO,
'fetchmail: reset password successfully')
except Exception as exc:
ud.debug(
ud.LISTENER, ud.ERROR,
'fetchmail: cannot reset password in LDAP (%s): %s' %
(dn, exc))
finally:
listener.unsetuid()
