def Write(self):
"""Write search indexes to the index file.
This method is a no-op if index_file is set to None.
"""
if not self.__index_file:
return
descriptor, tmp_filename = tempfile.mkstemp(
dir=os.path.dirname(self.__index_file))
tmpfile = os.fdopen(descriptor, 'wb')
pickler = pickle.Pickler(tmpfile, protocol=1)
pickler.fast = True
pickler.dump((self._VERSION, self.__indexes))
tmpfile.close()
self.__index_file_lock.acquire()
try:
try:
os.rename(tmp_filename, self.__index_file)
except OSError:
os.remove(self.__index_file)
os.rename(tmp_filename, self.__index_file)
finally:
self.__index_file_lock.release()
def checkpoint(self, key, cs):
"""
Store, with the supplied key, the supplied Constant s and Function s. The
key is internally cast to a string.
"""
key = str(key)
if key in self.__filenames:
raise CheckpointException(
"Attempting to overwrite checkpoint with key %s" % key)
cs = self._Checkpointer__check_cs(cs)
c_cs = OrderedDict()
id_map = {}
for c in cs:
c_id = c.id()
c_cs[c_id] = self._Checkpointer__pack(c)
id_map[c_id] = c
filename = self.__filename(key)
handle = open(filename, "wb")
pickler = pickle.Pickler(handle, -1)
pickler.dump(c_cs)
self.__filenames[key] = filename
self.__id_map[key] = id_map
return
def _python_memcache_serializer(key, value, pickle_version=None):
flags = 0
value_type = type(value)
# Check against exact types so that subclasses of native types will be
# restored as their native type
if value_type is bytes:
pass
elif value_type is six.text_type:
flags |= FLAG_TEXT
value = value.encode('utf8')
elif value_type is int:
flags |= FLAG_INTEGER
value = "%d" % value
elif six.PY2 and value_type is long_type:
flags |= FLAG_LONG
value = "%d" % value
else:
flags |= FLAG_PICKLE
output = BytesIO()
pickler = pickle.Pickler(output, pickle_version)
pickler.dump(value)
value = output.getvalue()
return value, flags
def __WritePickled(self, obj, filename):
"""Pickles the object and writes it to the given file.
"""
if not filename or filename == '/dev/null' or not obj:
return
descriptor, tmp_filename = tempfile.mkstemp(
dir=os.path.dirname(filename))
tmpfile = os.fdopen(descriptor, 'wb')
pickler = pickle.Pickler(tmpfile, protocol=1)
pickler.fast = True
pickler.dump(obj)
tmpfile.close()
self.__file_lock.acquire()
try:
try:
os.rename(tmp_filename, filename)
except OSError:
try:
os.remove(filename)
except:
pass
os.rename(tmp_filename, filename)
finally:
self.__file_lock.release()
def save_old(old):
# type: (dict) -> None
with open(DOVECOT_OLD_PICKLE, "wb+") as fd:
os.chmod(DOVECOT_OLD_PICKLE, 0o600)
p = pickle.Pickler(fd)
p.dump(old)
p.clear_memo()
def func(f):
if use_cpickle:
p = cPickle.Pickler(f, protocol=protocol)
else:
p = PicklerWithWarning(f, protocol=protocol)
p.persistent_id = persistent_id(zip_file)
p.dump(obj)
def _dump_object_to_file(self, object_data):
filename = self._generate_filename()
with open(filename, 'wb+') as fd:
os.chmod(filename, 0o600)
p = pickle.Pickler(fd)
p.dump(object_data)
p.clear_memo()
def postrun(): # type: () -> None global s4_init_mode global group_objects global connector_needs_restart if s4_init_mode: listener.setuid(0) try: s4_init_mode = False for ob in group_objects: for directory in dirs: filename = os.path.join(directory, "%f" % time.time()) with open(filename, 'wb+') as fd: os.chmod(filename, 0o600) p = pickle.Pickler(fd) p.dump(ob) p.clear_memo() del group_objects group_objects = [] finally: listener.unsetuid() if connector_needs_restart: _restart_connector() connector_needs_restart = False
def _dump_share_and_policy_result(dn, share_object, policy_result):
filename = os.path.join(SHARE_CACHE_DIR, dn)
with open(filename, 'wb+') as fd:
os.chmod(filename, 0o600)
p = pickle.Pickler(fd)
p.dump((dn, share_object, policy_result))
p.clear_memo()
def _save_old_object(directory, dn, old): # type: (str, str, Optional[Dict[str, List[bytes]]]) -> None filename = os.path.join(directory, 'tmp', 'old_dn') with open(filename, 'wb+') as fd: os.chmod(filename, 0o600) p = pickle.Pickler(fd) p.dump((dn, old)) p.clear_memo()
def _dump_changes_to_file_and_check_file(directory, dn, new, old, old_dn): # type: (str, str, Optional[Dict[str, List[bytes]]], Optional[Dict[str, List[bytes]]], Optional[str]) -> None ob = (dn, new, old, old_dn) tmpdir = os.path.join(directory, 'tmp') filename = '%f' % (time.time(),) filepath = os.path.join(tmpdir, filename) with open(filepath, 'wb+') as fd: os.chmod(filepath, 0o600) p = pickle.Pickler(fd) p.dump(ob) p.clear_memo() # prevent a race condition between the pickle file is only partly written to disk and then read # by moving it to the final location after it is completely written to disk shutil.move(filepath, os.path.join(directory, filename))
def python_memcache_serializer(key, value):
flags = 0
if isinstance(value, str):
pass
elif isinstance(value, int):
flags |= FLAG_INTEGER
value = "%d" % value
elif long_type is not None and isinstance(value, long_type):
flags |= FLAG_LONG
value = "%d" % value
else:
flags |= FLAG_PICKLE
output = BytesIO()
pickler = pickle.Pickler(output, 0)
pickler.dump(value)
value = output.getvalue()
return value, flags
def test_broken_pickle_with_shared(self):
saves = []
def pers_save(obj):
if isinstance(obj, numpy.ndarray):
saves.append(obj)
return len(saves) - 1
else:
return None
def pers_load(id):
return saves[id]
a = numpy.random.rand(4, 5)
b = numpy.random.rand(5, 4)
x = theano.tensor.matrix()
y = theano.shared(b)
f = theano.function([x], theano.tensor.dot(x, y))
from theano.compat import BytesIO
fp = BytesIO()
p = pickle.Pickler(fp, 2)
p.persistent_id = pers_save
try:
p.dump(f)
except NotImplementedError as e:
if exc_message(e).startswith('DebugMode is not picklable'):
return
else:
raise
fp2 = BytesIO(fp.getvalue())
fp.close()
p = pickle.Unpickler(fp2)
p.persistent_load = pers_load
f2 = p.load()
fp2.close()
def _pickle_iterable(filename, iterable):
with open(filename, 'wb') as pickle_fh:
pklr = pickle.Pickler(pickle_fh, _PICKLE_PROTOCOL)
for entry in iterable:
pklr.dump(entry)
pklr.clear_memo()
def __init__(self, *args, **kwargs):
self._pickler = pickle.Pickler(-1)
self._pickler.fast = 1
super(PickleMixinP2, self).__init__(*args, **kwargs)
def run(self):
try:
self.iterator = FileStorageIterator(*self.iterator_args)
except:
logger.exception(self.peer)
self.iterator = None
self.callFromThread(self.cfr_close)
return
if self.closed:
self.callFromThread(self.cfr_close)
return
if self.stopped:
# make sure our iterator gets stopped, as stopProducing might
# have been called while we were creating the iterator.
self.iterator.stop()
picklerf = BytesIO()
pickler = cPickle.Pickler(picklerf, 1)
pickler.fast = 1
def dump(data):
picklerf.seek(0)
picklerf.truncate()
pickler.dump(data)
return picklerf.getvalue()
self.md5 = md5(self.start_tid)
blob_block_size = 1 << 16
try:
for trans in self.iterator:
self.write(
dump(('T', (trans.tid, trans.status, trans.user,
trans.description, trans._extension))))
for record in trans:
if record.data and is_blob_record(record.data):
try:
fname = self.storage.loadBlob(
record.oid, record.tid)
f = open(fname, 'rb')
except (IOError, ZODB.POSException.POSKeyError):
pass
else:
f.seek(0, 2)
blob_size = f.tell()
blocks, r = divmod(blob_size, blob_block_size)
if r:
blocks += 1
self.write(
dump(('B', (record.oid, record.tid,
record.version, record.data_txn,
long(blocks)))))
self.write(record.data or '')
f.seek(0)
while blocks > 0:
data = f.read(blob_block_size)
if not data:
raise AssertionError("Too much blob data")
blocks -= 1
self.write(data)
f.close()
continue
self.write(
dump(('S', (record.oid, record.tid, record.version,
record.data_txn))))
self.write(record.data or b'')
self.write(dump(('C', (self.md5.digest(), ))))
except Exception as exc:
logger.exception(self.peer)
self.iterator = None
self.callFromThread(self.cfr_close)
def save_caption_vectors_products(self, train_ratio=0.8):
target, one_hot_targets, n_target = get_one_hot_targets(self.category_path)
train_list = []
with open(self.parse_data_path) as cap_f:
for i, line in enumerate(cap_f):
row = line.strip().split('\t')
asin = row[0]
categories = row[1]
title = row[2]
imgid = asin+'.jpg'
onehot_cate = one_hot_encode_str_lbl(categories, target, one_hot_targets)
train_list.append((imgid, onehot_cate, title))
if i % self.n_log_print == 0:
print(i, train_list[-1])
chunk_size = self.chunk_size
datasize = len(train_list)
self.logger.info('data size: %d' % datasize)
n_chunk = datasize // chunk_size + 1
chunk_list = []
for index in range(n_chunk):
start_index = index * chunk_size
end_index = start_index + chunk_size
chunk = train_list[start_index:end_index]
chunk_list.append((index, chunk))
for chunk in chunk_list:
index = chunk[0]
chunk = chunk[1]
st = time.time()
temp_list = []
for i, (key, cate_vec, title) in enumerate(chunk):
title_vec = self.parser.text2vec(title)
temp_list.append((key, cate_vec, title_vec))
chunk_path = (os.path.join(self.temp_dir_path, 'products_tv_{}.pkl'.format(index)))
f_out = open(chunk_path, 'wb')
p = cPickle.Pickler(f_out)
p.dump(temp_list)
p.clear_memo()
f_out.close()
self.logger.info("%s done: %d" % (chunk_path, time.time() - st))
if not os.path.isdir(self.train_dir_path):
os.makedirs(self.train_dir_path)
all_train = train_ratio >= 1.0
all_dev = train_ratio == 0.0
train_indices, train_size = self.get_train_indices(datasize, train_ratio)
dev_size = datasize - train_size
if all_dev:
train_size = 1
dev_size = datasize
if all_train:
dev_size = 1
train_size = datasize
data_fout = h5py.File(os.path.join(self.train_dir_path, 'data.h5py'), 'w')
train = data_fout.create_group('train')
dev = data_fout.create_group('dev')
self.create_dataset(train, train_size, n_target)
self.create_dataset(dev, dev_size, n_target)
sample_idx = 0
dataset = {'train': train, 'dev': dev}
num_samples = {'train': 0, 'dev': 0}
chunk = {'train': self.init_chunk(chunk_size, n_target),
'dev': self.init_chunk(chunk_size, n_target)}
# make h5file
for input_chunk_idx in range(n_chunk):
path = os.path.join(self.temp_dir_path, 'products_tv_{}.pkl'.format(input_chunk_idx))
print('processing %s ...' % path)
data = cPickle.loads(open(path, 'rb').read())
for data_idx, (img_idx, cate_vec, title_vec) in enumerate(data):
is_train = train_indices[sample_idx + data_idx]
if all_dev:
is_train = False
if all_train:
is_train = True
c = chunk['train'] if is_train else chunk['dev']
idx = c['num']
c['docvec'][idx] = title_vec
c['cate'][idx] = cate_vec
c['asin'].append(np.string_(img_idx))
c['num'] += 1
for t in ['train', 'dev']:
if chunk[t]['num'] >= chunk_size:
self.copy_chunk(dataset[t], chunk[t], num_samples[t])
num_samples[t] += chunk[t]['num']
chunk[t] = self.init_chunk(chunk_size, n_target)
sample_idx += len(data)
for t in ['train', 'dev']:
if chunk[t]['num'] > 0:
self.copy_chunk(dataset[t], chunk[t], num_samples[t])
num_samples[t] += chunk[t]['num']
for div in ['train', 'dev']:
ds = dataset[div]
size = num_samples[div]
ds['cate'].resize((size, n_target))
ds['docvec'].resize((size, self.doc_vec_size))
data_fout.close()
def handler(dn, new, old, command):
# type: (str, Optional[Dict[str, List[bytes]]], Optional[Dict[str, List[bytes]]], str) -> None
if os.path.exists(FETCHMAIL_OLD_PICKLE):
with open(FETCHMAIL_OLD_PICKLE, 'r') as fd:
p = pickle.Unpickler(fd)
old = p.load()
os.unlink(FETCHMAIL_OLD_PICKLE)
if command == 'r':
with open(FETCHMAIL_OLD_PICKLE, 'w+') as fd:
os.chmod(FETCHMAIL_OLD_PICKLE, 0o600)
p = pickle.Pickler(fd)
old = p.dump(old)
p.clear_memo()
flist = load_rc(fn_fetchmailrc)
if old and not new and not command == 'r':
# object has been deleted ==> remove entry from rc file
flist = objdelete(flist, old)
write_rc(flist, fn_fetchmailrc)
elif old and new and details_complete(old) and not details_complete(new):
# data is now incomplete ==> remove entry from rc file
flist = objdelete(flist, old)
write_rc(flist, fn_fetchmailrc)
elif new and details_complete(new):
# obj has been created or modified
passwd = None
if old:
# old exists ==> object has been modified ==> get old password and remove object entry from rc file
passwd = get_pw_from_rc(flist, old['uid'][0].decode('UTF-8'))
flist = objdelete(flist, old)
if not details_complete(new, incl_password=True):
if only_password_reset(old, new):
ud.debug(ud.LISTENER, ud.INFO,
'fetchmail: password has been reset - nothing to do')
# only password has been reset ==> nothing to do
return
# new obj does not contain password
if passwd:
# passwd has been set in old ==> use old password
ud.debug(ud.LISTENER, ud.INFO, 'fetchmail: using old password')
objappend(flist, new, passwd)
write_rc(flist, fn_fetchmailrc)
else:
ud.debug(
ud.LISTENER, ud.ERROR,
'fetchmail: user "%s": no password set in old and new' %
new['uid'][0])
else:
# new obj contains password ==> use new password
objappend(flist, new)
write_rc(flist, fn_fetchmailrc)
ud.debug(ud.LISTENER, ud.INFO, 'fetchmail: using new password')
configRegistry = univention.config_registry.ConfigRegistry()
configRegistry.load()
listener.setuid(0)
try:
lo = univention.uldap.getMachineConnection()
modlist = [('univentionFetchmailPasswd',
new['univentionFetchmailPasswd'][0], b"")]
lo.modify(dn, modlist)
ud.debug(ud.LISTENER, ud.INFO,
'fetchmail: reset password successfully')
except Exception as exc:
ud.debug(
ud.LISTENER, ud.ERROR,
'fetchmail: cannot reset password in LDAP (%s): %s' %
(dn, exc))
finally:
listener.unsetuid()
