def Write(self): """Write search indexes to the index file. This method is a no-op if index_file is set to None. """ if not self.__index_file: return descriptor, tmp_filename = tempfile.mkstemp( dir=os.path.dirname(self.__index_file)) tmpfile = os.fdopen(descriptor, 'wb') pickler = pickle.Pickler(tmpfile, protocol=1) pickler.fast = True pickler.dump((self._VERSION, self.__indexes)) tmpfile.close() self.__index_file_lock.acquire() try: try: os.rename(tmp_filename, self.__index_file) except OSError: os.remove(self.__index_file) os.rename(tmp_filename, self.__index_file) finally: self.__index_file_lock.release()
def checkpoint(self, key, cs): """ Store, with the supplied key, the supplied Constant s and Function s. The key is internally cast to a string. """ key = str(key) if key in self.__filenames: raise CheckpointException( "Attempting to overwrite checkpoint with key %s" % key) cs = self._Checkpointer__check_cs(cs) c_cs = OrderedDict() id_map = {} for c in cs: c_id = c.id() c_cs[c_id] = self._Checkpointer__pack(c) id_map[c_id] = c filename = self.__filename(key) handle = open(filename, "wb") pickler = pickle.Pickler(handle, -1) pickler.dump(c_cs) self.__filenames[key] = filename self.__id_map[key] = id_map return
def _python_memcache_serializer(key, value, pickle_version=None): flags = 0 value_type = type(value) # Check against exact types so that subclasses of native types will be # restored as their native type if value_type is bytes: pass elif value_type is six.text_type: flags |= FLAG_TEXT value = value.encode('utf8') elif value_type is int: flags |= FLAG_INTEGER value = "%d" % value elif six.PY2 and value_type is long_type: flags |= FLAG_LONG value = "%d" % value else: flags |= FLAG_PICKLE output = BytesIO() pickler = pickle.Pickler(output, pickle_version) pickler.dump(value) value = output.getvalue() return value, flags
def __WritePickled(self, obj, filename): """Pickles the object and writes it to the given file. """ if not filename or filename == '/dev/null' or not obj: return descriptor, tmp_filename = tempfile.mkstemp( dir=os.path.dirname(filename)) tmpfile = os.fdopen(descriptor, 'wb') pickler = pickle.Pickler(tmpfile, protocol=1) pickler.fast = True pickler.dump(obj) tmpfile.close() self.__file_lock.acquire() try: try: os.rename(tmp_filename, filename) except OSError: try: os.remove(filename) except: pass os.rename(tmp_filename, filename) finally: self.__file_lock.release()
def save_old(old): # type: (dict) -> None with open(DOVECOT_OLD_PICKLE, "wb+") as fd: os.chmod(DOVECOT_OLD_PICKLE, 0o600) p = pickle.Pickler(fd) p.dump(old) p.clear_memo()
def func(f): if use_cpickle: p = cPickle.Pickler(f, protocol=protocol) else: p = PicklerWithWarning(f, protocol=protocol) p.persistent_id = persistent_id(zip_file) p.dump(obj)
def _dump_object_to_file(self, object_data): filename = self._generate_filename() with open(filename, 'wb+') as fd: os.chmod(filename, 0o600) p = pickle.Pickler(fd) p.dump(object_data) p.clear_memo()
def postrun(): # type: () -> None global s4_init_mode global group_objects global connector_needs_restart if s4_init_mode: listener.setuid(0) try: s4_init_mode = False for ob in group_objects: for directory in dirs: filename = os.path.join(directory, "%f" % time.time()) with open(filename, 'wb+') as fd: os.chmod(filename, 0o600) p = pickle.Pickler(fd) p.dump(ob) p.clear_memo() del group_objects group_objects = [] finally: listener.unsetuid() if connector_needs_restart: _restart_connector() connector_needs_restart = False
def _dump_share_and_policy_result(dn, share_object, policy_result): filename = os.path.join(SHARE_CACHE_DIR, dn) with open(filename, 'wb+') as fd: os.chmod(filename, 0o600) p = pickle.Pickler(fd) p.dump((dn, share_object, policy_result)) p.clear_memo()
def _save_old_object(directory, dn, old): # type: (str, str, Optional[Dict[str, List[bytes]]]) -> None filename = os.path.join(directory, 'tmp', 'old_dn') with open(filename, 'wb+') as fd: os.chmod(filename, 0o600) p = pickle.Pickler(fd) p.dump((dn, old)) p.clear_memo()
def _dump_changes_to_file_and_check_file(directory, dn, new, old, old_dn): # type: (str, str, Optional[Dict[str, List[bytes]]], Optional[Dict[str, List[bytes]]], Optional[str]) -> None ob = (dn, new, old, old_dn) tmpdir = os.path.join(directory, 'tmp') filename = '%f' % (time.time(),) filepath = os.path.join(tmpdir, filename) with open(filepath, 'wb+') as fd: os.chmod(filepath, 0o600) p = pickle.Pickler(fd) p.dump(ob) p.clear_memo() # prevent a race condition between the pickle file is only partly written to disk and then read # by moving it to the final location after it is completely written to disk shutil.move(filepath, os.path.join(directory, filename))
def python_memcache_serializer(key, value): flags = 0 if isinstance(value, str): pass elif isinstance(value, int): flags |= FLAG_INTEGER value = "%d" % value elif long_type is not None and isinstance(value, long_type): flags |= FLAG_LONG value = "%d" % value else: flags |= FLAG_PICKLE output = BytesIO() pickler = pickle.Pickler(output, 0) pickler.dump(value) value = output.getvalue() return value, flags
def test_broken_pickle_with_shared(self): saves = [] def pers_save(obj): if isinstance(obj, numpy.ndarray): saves.append(obj) return len(saves) - 1 else: return None def pers_load(id): return saves[id] a = numpy.random.rand(4, 5) b = numpy.random.rand(5, 4) x = theano.tensor.matrix() y = theano.shared(b) f = theano.function([x], theano.tensor.dot(x, y)) from theano.compat import BytesIO fp = BytesIO() p = pickle.Pickler(fp, 2) p.persistent_id = pers_save try: p.dump(f) except NotImplementedError as e: if exc_message(e).startswith('DebugMode is not picklable'): return else: raise fp2 = BytesIO(fp.getvalue()) fp.close() p = pickle.Unpickler(fp2) p.persistent_load = pers_load f2 = p.load() fp2.close()
def _pickle_iterable(filename, iterable): with open(filename, 'wb') as pickle_fh: pklr = pickle.Pickler(pickle_fh, _PICKLE_PROTOCOL) for entry in iterable: pklr.dump(entry) pklr.clear_memo()
def __init__(self, *args, **kwargs): self._pickler = pickle.Pickler(-1) self._pickler.fast = 1 super(PickleMixinP2, self).__init__(*args, **kwargs)
def run(self): try: self.iterator = FileStorageIterator(*self.iterator_args) except: logger.exception(self.peer) self.iterator = None self.callFromThread(self.cfr_close) return if self.closed: self.callFromThread(self.cfr_close) return if self.stopped: # make sure our iterator gets stopped, as stopProducing might # have been called while we were creating the iterator. self.iterator.stop() picklerf = BytesIO() pickler = cPickle.Pickler(picklerf, 1) pickler.fast = 1 def dump(data): picklerf.seek(0) picklerf.truncate() pickler.dump(data) return picklerf.getvalue() self.md5 = md5(self.start_tid) blob_block_size = 1 << 16 try: for trans in self.iterator: self.write( dump(('T', (trans.tid, trans.status, trans.user, trans.description, trans._extension)))) for record in trans: if record.data and is_blob_record(record.data): try: fname = self.storage.loadBlob( record.oid, record.tid) f = open(fname, 'rb') except (IOError, ZODB.POSException.POSKeyError): pass else: f.seek(0, 2) blob_size = f.tell() blocks, r = divmod(blob_size, blob_block_size) if r: blocks += 1 self.write( dump(('B', (record.oid, record.tid, record.version, record.data_txn, long(blocks))))) self.write(record.data or '') f.seek(0) while blocks > 0: data = f.read(blob_block_size) if not data: raise AssertionError("Too much blob data") blocks -= 1 self.write(data) f.close() continue self.write( dump(('S', (record.oid, record.tid, record.version, record.data_txn)))) self.write(record.data or b'') self.write(dump(('C', (self.md5.digest(), )))) except Exception as exc: logger.exception(self.peer) self.iterator = None self.callFromThread(self.cfr_close)
def save_caption_vectors_products(self, train_ratio=0.8): target, one_hot_targets, n_target = get_one_hot_targets(self.category_path) train_list = [] with open(self.parse_data_path) as cap_f: for i, line in enumerate(cap_f): row = line.strip().split('\t') asin = row[0] categories = row[1] title = row[2] imgid = asin+'.jpg' onehot_cate = one_hot_encode_str_lbl(categories, target, one_hot_targets) train_list.append((imgid, onehot_cate, title)) if i % self.n_log_print == 0: print(i, train_list[-1]) chunk_size = self.chunk_size datasize = len(train_list) self.logger.info('data size: %d' % datasize) n_chunk = datasize // chunk_size + 1 chunk_list = [] for index in range(n_chunk): start_index = index * chunk_size end_index = start_index + chunk_size chunk = train_list[start_index:end_index] chunk_list.append((index, chunk)) for chunk in chunk_list: index = chunk[0] chunk = chunk[1] st = time.time() temp_list = [] for i, (key, cate_vec, title) in enumerate(chunk): title_vec = self.parser.text2vec(title) temp_list.append((key, cate_vec, title_vec)) chunk_path = (os.path.join(self.temp_dir_path, 'products_tv_{}.pkl'.format(index))) f_out = open(chunk_path, 'wb') p = cPickle.Pickler(f_out) p.dump(temp_list) p.clear_memo() f_out.close() self.logger.info("%s done: %d" % (chunk_path, time.time() - st)) if not os.path.isdir(self.train_dir_path): os.makedirs(self.train_dir_path) all_train = train_ratio >= 1.0 all_dev = train_ratio == 0.0 train_indices, train_size = self.get_train_indices(datasize, train_ratio) dev_size = datasize - train_size if all_dev: train_size = 1 dev_size = datasize if all_train: dev_size = 1 train_size = datasize data_fout = h5py.File(os.path.join(self.train_dir_path, 'data.h5py'), 'w') train = data_fout.create_group('train') dev = data_fout.create_group('dev') self.create_dataset(train, train_size, n_target) self.create_dataset(dev, dev_size, n_target) sample_idx = 0 dataset = {'train': train, 'dev': dev} num_samples = {'train': 0, 'dev': 0} chunk = {'train': self.init_chunk(chunk_size, n_target), 'dev': self.init_chunk(chunk_size, n_target)} # make h5file for input_chunk_idx in range(n_chunk): path = os.path.join(self.temp_dir_path, 'products_tv_{}.pkl'.format(input_chunk_idx)) print('processing %s ...' % path) data = cPickle.loads(open(path, 'rb').read()) for data_idx, (img_idx, cate_vec, title_vec) in enumerate(data): is_train = train_indices[sample_idx + data_idx] if all_dev: is_train = False if all_train: is_train = True c = chunk['train'] if is_train else chunk['dev'] idx = c['num'] c['docvec'][idx] = title_vec c['cate'][idx] = cate_vec c['asin'].append(np.string_(img_idx)) c['num'] += 1 for t in ['train', 'dev']: if chunk[t]['num'] >= chunk_size: self.copy_chunk(dataset[t], chunk[t], num_samples[t]) num_samples[t] += chunk[t]['num'] chunk[t] = self.init_chunk(chunk_size, n_target) sample_idx += len(data) for t in ['train', 'dev']: if chunk[t]['num'] > 0: self.copy_chunk(dataset[t], chunk[t], num_samples[t]) num_samples[t] += chunk[t]['num'] for div in ['train', 'dev']: ds = dataset[div] size = num_samples[div] ds['cate'].resize((size, n_target)) ds['docvec'].resize((size, self.doc_vec_size)) data_fout.close()
def handler(dn, new, old, command): # type: (str, Optional[Dict[str, List[bytes]]], Optional[Dict[str, List[bytes]]], str) -> None if os.path.exists(FETCHMAIL_OLD_PICKLE): with open(FETCHMAIL_OLD_PICKLE, 'r') as fd: p = pickle.Unpickler(fd) old = p.load() os.unlink(FETCHMAIL_OLD_PICKLE) if command == 'r': with open(FETCHMAIL_OLD_PICKLE, 'w+') as fd: os.chmod(FETCHMAIL_OLD_PICKLE, 0o600) p = pickle.Pickler(fd) old = p.dump(old) p.clear_memo() flist = load_rc(fn_fetchmailrc) if old and not new and not command == 'r': # object has been deleted ==> remove entry from rc file flist = objdelete(flist, old) write_rc(flist, fn_fetchmailrc) elif old and new and details_complete(old) and not details_complete(new): # data is now incomplete ==> remove entry from rc file flist = objdelete(flist, old) write_rc(flist, fn_fetchmailrc) elif new and details_complete(new): # obj has been created or modified passwd = None if old: # old exists ==> object has been modified ==> get old password and remove object entry from rc file passwd = get_pw_from_rc(flist, old['uid'][0].decode('UTF-8')) flist = objdelete(flist, old) if not details_complete(new, incl_password=True): if only_password_reset(old, new): ud.debug(ud.LISTENER, ud.INFO, 'fetchmail: password has been reset - nothing to do') # only password has been reset ==> nothing to do return # new obj does not contain password if passwd: # passwd has been set in old ==> use old password ud.debug(ud.LISTENER, ud.INFO, 'fetchmail: using old password') objappend(flist, new, passwd) write_rc(flist, fn_fetchmailrc) else: ud.debug( ud.LISTENER, ud.ERROR, 'fetchmail: user "%s": no password set in old and new' % new['uid'][0]) else: # new obj contains password ==> use new password objappend(flist, new) write_rc(flist, fn_fetchmailrc) ud.debug(ud.LISTENER, ud.INFO, 'fetchmail: using new password') configRegistry = univention.config_registry.ConfigRegistry() configRegistry.load() listener.setuid(0) try: lo = univention.uldap.getMachineConnection() modlist = [('univentionFetchmailPasswd', new['univentionFetchmailPasswd'][0], b"")] lo.modify(dn, modlist) ud.debug(ud.LISTENER, ud.INFO, 'fetchmail: reset password successfully') except Exception as exc: ud.debug( ud.LISTENER, ud.ERROR, 'fetchmail: cannot reset password in LDAP (%s): %s' % (dn, exc)) finally: listener.unsetuid()