if os.path.exists(cache_file) and os.path.getmtime( cache_file) > os.path.getmtime(os.path.join(_curpath, "dict.txt")): print >> sys.stderr, "loading model from cache" try: trie, FREQ, total, min_freq = marshal.load(open(cache_file, 'rb')) load_from_cache_fail = False except: load_from_cache_fail = True if load_from_cache_fail: trie, FREQ, total = gen_trie(os.path.join(_curpath, "dict.txt")) FREQ = dict([(k, float(v) / total) for k, v in FREQ.iteritems()]) #normalize min_freq = min(FREQ.itervalues()) print >> sys.stderr, "dumping model to file cache" marshal.dump((trie, FREQ, total, min_freq), open(cache_file, 'wb')) print >> sys.stderr, "loading model cost ", time.time() - t1, "seconds." print >> sys.stderr, "Trie has been built succesfully." def __cut_all(sentence): N = len(sentence) i, j = 0, 0 p = trie while i < N: c = sentence[j] if c in p: p = p[c] if '' in p: yield sentence[i:j + 1]
#Marshal module contains functions that can read and write Python values in a binary format. import marshal # Python code to demonstrate serialization file1 = open("file1.txt","w") bytes = marshal.dump([1,2,3,4,5],file1) print bytes file1.close() # Python code to demonstrate de-serialization file2 = open("file1.txt","r") bytes1 = marshal.load(file2) print bytes1 file2.close()
print(x) if x not in cur: cur[x] = {} print(tree) print(cur[x]) cur = cur[x] print(cur) print(cur.keys()) print(tree) if 'g' not in tree: print('1') if 'b' not in tree: print('2') return tree if __name__ == '__main__': import argparse parser = argparse.ArgumentParser( description='Build search tree from dictionary.') parser.add_argument('input_file', help='Input file name.') parser.add_argument('output_file', help='Output file name.') args = parser.parse_args() tree = gen(args.input_file) with open(args.output_file, 'wb') as f: pickle.dump(tree, f)
def dump_stats(self, file): import marshal f = open(file, 'wb') self.create_stats() marshal.dump(self.stats, f) f.close()
def initialize(self, dictionary=None): #系统初始化主函数 if dictionary: abs_path = _get_abs_path(dictionary) if self.dictionary == abs_path and self.initialized: return else: self.dictionary = abs_path self.initialized = False else: abs_path = self.dictionary with self.lock: try: with DICT_WRITING[abs_path]: pass except KeyError: pass if self.initialized: return default_logger.debug("Building prefix dict from %s ..." % (abs_path or 'the default dictionary')) t1 = time.time() if self.cache_file: cache_file = self.cache_file # default dictionary elif abs_path == DEFAULT_DICT: cache_file = "jieba.cache" # custom dictionary else: cache_file = "jieba.u%s.cache" % md5( abs_path.encode('utf-8', 'replace')).hexdigest() cache_file = os.path.join( self.tmp_dir or tempfile.gettempdir(), cache_file) # prevent absolute path in self.cache_file tmpdir = os.path.dirname(cache_file) load_from_cache_fail = True if os.path.isfile(cache_file) and (abs_path == DEFAULT_DICT or os.path.getmtime(cache_file) > os.path.getmtime(abs_path)): default_logger.debug( "Loading model from cache %s" % cache_file) try: with open(cache_file, 'rb') as cf: self.FREQ, self.total = marshal.load(cf) load_from_cache_fail = False except Exception: load_from_cache_fail = True if load_from_cache_fail: wlock = DICT_WRITING.get(abs_path, threading.RLock()) DICT_WRITING[abs_path] = wlock with wlock: self.FREQ, self.total = self.gen_pfdict(self.get_dict_file()) default_logger.debug( "Dumping model to file cache %s" % cache_file) try: # prevent moving across different filesystems fd, fpath = tempfile.mkstemp(dir=tmpdir) with os.fdopen(fd, 'wb') as temp_cache_file: marshal.dump( (self.FREQ, self.total), temp_cache_file) _replace_file(fpath, cache_file) except Exception: default_logger.exception("Dump cache file failed.") try: del DICT_WRITING[abs_path] except KeyError: pass self.initialized = True default_logger.debug( "Loading model cost %.3f seconds." % (time.time() - t1)) default_logger.debug("Prefix dict has been built succesfully.")
def dump_code_to_file(code, file): file.write(MAGIC) wr_long(file, long(time.time())) # noqa marshal.dump(code, file) file.flush()
def update_pyc(filename, new_path): """Updates the filenames stored in pyc files.""" with open(filename, 'rb') as f: magic = f.read(MAGIC_LENGTH) try: code = marshal.load(f) except Exception: print('Error in %s' % filename) raise def _make_code(code, filename, consts): if sys.version_info[0] == 2: # pragma: no cover (PY2) arglist = [ code.co_argcount, code.co_nlocals, code.co_stacksize, code.co_flags, code.co_code, tuple(consts), code.co_names, code.co_varnames, filename, code.co_name, code.co_firstlineno, code.co_lnotab, code.co_freevars, code.co_cellvars, ] else: # pragma: no cover (PY3) arglist = [ code.co_argcount, code.co_kwonlyargcount, code.co_nlocals, code.co_stacksize, code.co_flags, code.co_code, tuple(consts), code.co_names, code.co_varnames, filename, code.co_name, code.co_firstlineno, code.co_lnotab, code.co_freevars, code.co_cellvars, ] return CodeType(*arglist) def _process(code): consts = [] for const in code.co_consts: if type(const) is CodeType: const = _process(const) consts.append(const) if new_path != code.co_filename or consts != list(code.co_consts): code = _make_code(code, new_path, consts) return code new_code = _process(code) if new_code is not code: debug('B %s' % filename) with open(filename, 'wb') as f: f.write(magic) marshal.dump(new_code, f)
#wd = webdriver.Chrome() #base = 'https://iris.ucl.ac.uk/iris/search' #wd.get(base) #select = Select(wd.find_element_by_id('meta_x_phrase_sand')) #select.select_by_visible_text('Researchers') #wd.find_element_by_id('btnMainAdvSearch').click() #sleep(30); base = 'https://iris.ucl.ac.uk/iris/search/funnelbackResults?query=&collection=iris-meta&form=results&meta_x_phrase_sand=iris-researchers&f.Result+Categories%7cx=%2ciris+researchers&start_rank={}&query_and=&query_phrase=&query_not=&meta_t=&meta_O=&meta_d1day=&meta_d1month=&meta_d1year=&meta_d2day=&meta_d2month=&meta_d2year=&meta_N=&meta_S=&sort=' b = 'https://iris.ucl.ac.uk/iris/browse/profile?' # all = [] url_list = [] ''' for i in range(9585): if i % 10 == 1: all.append(base.format(i)) #marshal.dump(all, 'all') #[:2] for u in all: rsp = requests.get(u, verify=False) soup = BeautifulSoup(rsp.text, 'html.parser') for link in soup.find_all('a'): if link.get('href').find('upi=') != -1: if link.get('href') not in url_list: url_list.append(link.get('href').split('?')[1]) url_file = open("url_list", 'wb')
def ecriture_obstacles(chemin,x,y,obstaclage): obstacles= [(random.randint(0,x-1),random.randint(0,y-1)) for mur in range(int(x*y*obstaclage))] fichier=open(chemin,"wb") marshal.dump(obstacles,fichier) fichier.close()
for ip in product(*zip(py_array,py_short)): initial_py = "".join(ip) py_freq[initial_py] = py_freq.get(initial_py,0) + max(int(log(freq)),1) if not initial_py in p2c: p2c[initial_py] = [] p2c[initial_py].append((freq,word)) lines = None p2c = dict( ( k,tuple( w[1] for w in sorted(v,reverse=True) ) ) for k,v in p2c.iteritems()) total = sum(chn_freq.itervalues()) chn_freq = dict( (k,log(float(v)/total)) for k,v in chn_freq.iteritems() ) py_freq = dict( (k,log(float(v)/total)) for k,v in py_freq.iteritems() ) min_freq = min(py_freq.values()) with file(CACHE_PATH,'wb') as cache_file: marshal.dump((p2c,chn_freq,py_freq,min_freq),cache_file) print "init cost: ", time.time() - t_start #end init single_letters = set(ascii_lowercase) print "data loaded." def word_rank(word): if word in single_letters: return min_freq-1.0 elif len(word)==1: return 0.0 return py_freq.get(word,min_freq*20.0)
def dump(value,file): m.dump(value,file)
def save_kindex(self): '''将 k-gram 索引保存到硬盘''' print(self.kindex) kgram_file = open(self.kindex_name, "wb") marshal.dump(self.kindex, kgram_file) kgram_file.close()
def dump(self, file="config.bin"): f = open(file, 'w') marshal.dump(self.byKey, f) #l = lambda y,x, f = f, d = marshal.dump: d(x,f) #reduce(l,self.byId) f.close()
def create_py3_base_library(libzip_filename, graph): """ Package basic Python modules into .zip file. The .zip file with basic modules is necessary to have on PYTHONPATH for initializing libpython3 in order to run the frozen executable with Python 3. """ # Construct regular expression for matching modules that should be bundled # into base_library.zip. # Excluded are plain 'modules' or 'submodules.ANY_NAME'. # The match has to be exact - start and end of string not substring. regex_modules = '|'.join([r'(^%s$)' % x for x in PY3_BASE_MODULES]) regex_submod = '|'.join([r'(^%s\..*$)' % x for x in PY3_BASE_MODULES]) regex_str = regex_modules + '|' + regex_submod module_filter = re.compile(regex_str) try: # Remove .zip from previous run. if os.path.exists(libzip_filename): os.remove(libzip_filename) logger.debug('Adding python files to base_library.zip') # Class zipfile.PyZipFile is not suitable for PyInstaller needs. with zipfile.ZipFile(libzip_filename, mode='w') as zf: zf.debug = 3 # Sort the graph nodes by identifier to ensure repeatable builds graph_nodes = list(graph.flatten()) graph_nodes.sort(key=lambda item: item.identifier) for mod in graph_nodes: if type(mod) in (modulegraph.SourceModule, modulegraph.Package): # Bundling just required modules. if module_filter.match(mod.identifier): st = os.stat(mod.filename) timestamp = int(st.st_mtime) size = st.st_size & 0xFFFFFFFF # Name inside the archive. The ZIP format # specification requires forward slashes as # directory separator. # TODO use .pyo suffix if optimize flag is enabled. if type(mod) is modulegraph.Package: new_name = mod.identifier.replace('.', '/') \ + '/__init__.pyc' else: new_name = mod.identifier.replace('.', '/') \ + '.pyc' # Write code to a file. # This code is similar to py_compile.compile(). with io.BytesIO() as fc: # Prepare all data in byte stream file-like object. fc.write(BYTECODE_MAGIC) if is_py37: # Additional bitfield according to PEP 552 # 0b01 means hash based but don't check the hash fc.write(struct.pack('<I', 0b01)) with open(mod.filename, 'rb') as fs: source_bytes = fs.read() source_hash = importlib_source_hash( source_bytes) fc.write(source_hash) else: fc.write(struct.pack('<II', timestamp, size)) marshal.dump(mod.code, fc) # Use a ZipInfo to set timestamp for deterministic build info = zipfile.ZipInfo(new_name) zf.writestr(info, fc.getvalue()) except Exception as e: logger.error('base_library.zip could not be created!') raise
def create_py3_base_library(libzip_filename, graph): """ Package basic Python modules into .zip file. The .zip file with basic modules is necessary to have on PYTHONPATH for initializing libpython3 in order to run the frozen executable with Python 3. """ # TODO Replace this function with something better or something from standard Python library. # Helper functions. def _write_long(f, x): """ Write a 32-bit int to a file in little-endian order. """ f.write( bytes([ x & 0xff, (x >> 8) & 0xff, (x >> 16) & 0xff, (x >> 24) & 0xff ])) # Construct regular expression for matching modules that should be bundled # into base_library.zip. # Excluded are plain 'modules' or 'submodules.ANY_NAME'. # The match has to be exact - start and end of string not substring. regex_modules = '|'.join([r'(^%s$)' % x for x in PY3_BASE_MODULES]) regex_submod = '|'.join([r'(^%s\..*$)' % x for x in PY3_BASE_MODULES]) regex_str = regex_modules + '|' + regex_submod module_filter = re.compile(regex_str) try: # Remove .zip from previous run. if os.path.exists(libzip_filename): os.remove(libzip_filename) logger.debug('Adding python files to base_library.zip') # Class zipfile.PyZipFile is not suitable for PyInstaller needs. with zipfile.ZipFile(libzip_filename, mode='w') as zf: zf.debug = 3 for mod in graph.flatten(): if type(mod) in (modulegraph.SourceModule, modulegraph.Package): # Bundling just required modules. if module_filter.match(mod.identifier): st = os.stat(mod.filename) timestamp = int(st.st_mtime) size = st.st_size & 0xFFFFFFFF # Name inside a zip archive. # TODO use .pyo suffix if optimize flag is enabled. if type(mod) is modulegraph.Package: new_name = mod.identifier.replace( '.', os.sep) + os.sep + '__init__' + '.pyc' else: new_name = mod.identifier.replace('.', os.sep) + '.pyc' # Write code to a file. # This code is similar to py_compile.compile(). with io.BytesIO() as fc: # Prepare all data in byte stream file-like object. fc.write(BYTECODE_MAGIC) _write_long(fc, timestamp) _write_long(fc, size) marshal.dump(mod.code, fc) zf.writestr(new_name, fc.getvalue()) except Exception as e: logger.error('base_library.zip could not be created!') raise
def main(args=None): from os.path import exists, join, dirname, isdir if args == None: args = sys.argv[1:] logging.basicConfig() options, args = parser.parse_args(args) if options.lock_file: import zc.lockfile lock = zc.lockfile.LockFile(options.lock_file) else: lock = None fudge = options.clock_fudge_factor encoding = options.file_system_encoding had_index = False s3 = {} if options.index: if not options.ignore_index and exists(options.index): with open(options.index) as f: s3 = marshal.load(f) had_index = True index = {} else: index = None src_path, bucket_name = args if '/' in bucket_name: bucket_name, bucket_prefix = bucket_name.split('/', 1) else: bucket_prefix = '' len_bucket_prefix = len(bucket_prefix) fs = {} queue = Queue.Queue(maxsize=999) put = queue.put cloudfront = options.cloudfront invalidations = [] generate_index_html = options.generate_index_html GENERATE = object() INDEX_HTML = "index.html" def worker(base_path): mtime = path = 0 while 1: try: mtime, queued_path = queue.get() path = queued_path if path is None: return key = boto.s3.key.Key(bucket) if mtime is None: # delete try: try: key.key = bucket_prefix + path key.delete() except Exception: logger.exception('deleting %r, retrying' % key.key) time.sleep(9) key.key = bucket_prefix + path key.delete() except Exception: if index is not None: # Failed to delete. Put the key back so we # try again later index[queued_path] = 1 raise elif mtime is GENERATE: (path, s3mtime) = path fspath = join(base_path, path.encode(encoding)) if exists(fspath): # Someone created a file since we decided to # generate one. continue fspath = dirname(fspath) data = "Index of " + path[:-len(INDEX_HTML) - 1] data = [ "<!-- generated -->", "<html><head><title>%s</title></head><body>" % data, "<h1>%s</h1><table>" % data, "<tr><th>Name</th><th>Last modified</th><th>Size</th>" "</tr>", ] for name in sorted(os.listdir(fspath)): if name.startswith('.'): continue # don't index dot files name_path = join(fspath, name) if isdir(name_path): name = name + '/' size = '-' else: size = os.stat(name_path).st_size mtime = time.ctime(os.stat(name_path).st_mtime) name = name.decode(encoding) data.append('<tr><td><a href="%s">%s</a></td>\n' ' <td>%s</td><td>%s</td></tr>' % (name, name, mtime, size)) data.append("</table></body></html>\n") data = '\n'.join(data) digest = hashlib.md5(data.encode(encoding)).hexdigest() if digest != s3mtime: # Note that s3mtime is either a previous # digest or it's 0 (cus path wasn't in s3) or # it's an s3 upload time. The test above # works in all of these cases. key.key = bucket_prefix + path key.set_metadata('generated', 'true') try: key.set_contents_from_string( data, headers={'Content-Type': 'text/html'}, ) except Exception: logger.exception( 'uploading generated %r, retrying' % path) time.sleep(9) key.set_contents_from_string( data, headers={'Content-Type': 'text/html'}, ) if s3mtime: # update (if it was add, mtime would be 0) if cloudfront: invalidations.append(path) if index is not None: index[path] = digest else: # upload try: if had_index: # We only store mtimes to the nearest second. # We don't have a fudge factor, so there's a # chance that someone might update the file in # the same second, so we check if a second has # passed and sleep if it hasn't. now = time_time_from_sixtuple( time.gmtime(time.time())) if not now > mtime: time.sleep(1) key.key = bucket_prefix + path path = join(base_path, path) try: key.set_contents_from_filename( path.encode(encoding)) except Exception: logger.exception('uploading %r %r, retrying' % (mtime, path)) time.sleep(9) key.set_contents_from_filename( path.encode(encoding)) except Exception: if index is not None: # Upload failed. Remove from index so we # try again later (if the path is still # around). index.pop(queued_path) raise except Exception: logger.exception('processing %r %r' % (mtime, path)) finally: queue.task_done() workers = [thread(worker, src_path) for i in range(options.worker_threads)] # As we build up the 2 dicts, we try to identify cases we can # eliminate right away, or cases we can begin handling, so we can # avoid accumulating them, and also so we can start processing # sooner. def listfs(path, base): for name in sorted(os.listdir(path)): pname = join(path, name) rname = join(base, name) if isdir(pname): listfs(pname, rname) if generate_index_html and not exists(join(pname, INDEX_HTML)): key = rname.decode(encoding) + '/' + INDEX_HTML # We don't short circuit by checking s3 here. # We'll do that at the end. fs[key] = -1 else: try: mtime = time_time_from_sixtuple( time.gmtime(os.stat(pname).st_mtime)) except OSError: logger.exception("bad file %r" % rname) continue key = rname.decode(encoding) if index is not None: index[key] = mtime if key in s3: # We can go ahead and do the check s3mtime = s3.pop(key) if (isinstance(s3mtime, basestring) # generated or mtime > s3mtime): put((mtime, key)) if cloudfront: invalidations.append(key) else: fs[key] = mtime fs_thread = thread(listfs, src_path, '') s3conn = boto.s3.connection.S3Connection() bucket = s3conn.get_bucket(bucket_name) if not had_index: @thread def s3_thread(): for key in bucket.list(bucket_prefix): s3mtime = time_time_from_sixtuple(parse_time( key.last_modified)) # subtract a fudge factor to account for crappy clocks and bias # caused by delat between start of upload and # computation of last_modified. s3mtime -= fudge path = key.key[len_bucket_prefix:] if path in fs: mtime = fs.pop(path) if mtime > s3mtime: put((mtime, path)) if cloudfront: invalidations.append(path) elif mtime == -1: # generate marker. Put it back. fs[path] = -1 else: s3[path] = s3mtime s3_thread.join() fs_thread.join() for (path, mtime) in fs.iteritems(): s3mtime = s3.pop(path, 0) if mtime == -1: # We generate unconditionally, because the content # is dynamic. We pass along the old s3mtime, which might # be an old digest to see if we actually have to update s3. put((GENERATE, (path, s3mtime))) else: if mtime > s3mtime: put((mtime, path)) if s3mtime: # update (if it was add, mtime would be 0) if cloudfront: invalidations.append(path) if not options.no_delete: for path in s3: put((None, path)) if cloudfront: invalidations.append(path) queue.join() if index is not None: with open(options.index, 'w') as f: marshal.dump(index, f) if cloudfront: cfconn = boto.connect_cloudfront() time.sleep( 9) # give a little extra type for the s3 updates to propigate for start in range(0, len(invalidations), 1000): cfconn.create_invalidation_request( cloudfront, [bucket_prefix + p for p in invalidations[start:start + 1000]]) if lock is not None: lock.close() for _ in workers: put((None, None)) for w in workers: w.join()
from bx.misc.seekbzip2 import SeekableBzip2File parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('bz2', help='bz2 dump file') parser.add_argument('bz2t', help='bzip-table file') parser.add_argument('output', help='destination file') parser.add_argument('--offsets-only', action='store_true') args = parser.parse_args() index = {} dump = SeekableBzip2File(args.bz2, args.bz2t) offset = 0 try: for line in dump: if line == ' <page>\n': start = offset elif line.startswith(' <title>'): title = line[11:-9] index[title] = start offset = dump.tell() finally: dump.close() if args.offsets_only: index = tuple(index) with open(args.output, 'wb') as f: marshal.dump(index, f)
codestring = codestring + '\n' try: codeobject = ompc.compile(codestring, dfile or file,'exec') except Exception, err: py_exc = MCompileError(err.__class__,err.args,dfile or file) if doraise: raise py_exc else: sys.stderr.write(py_exc.msg + '\n') return if cfile is None: cfile = file + (__debug__ and 'c' or 'o') fc = open(cfile, 'wb') fc.write('\0\0\0\0') wr_long(fc, timestamp) marshal.dump(codeobject, fc) fc.flush() fc.seek(0, 0) fc.write(MAGIC) fc.close() set_creator_type(cfile) def main(args=None): """Compile several source files. The files named in 'args' (or on the command line, if 'args' is not specified) are compiled and the resulting bytecode is cached in the normal manner. This function does not search a directory structure to locate source files; it only compiles files named explicitly.
def writePyc(code, path): f = open(path, "wb") f.write(MAGIC) f.write("\0" * 4) # don't bother about a time stamp marshal.dump(code, f) f.close()
def save(self): if self.usecache and self.cache: self.canonicalize_filenames() import marshal with open(self.cache, 'wb') as cache: marshal.dump(self.cexecuted, cache)
def initialize(*args): global trie, FREQ, total, min_freq, initialized if len(args) == 0: dictionary = DICTIONARY else: dictionary = args[0] with DICT_LOCK: if initialized: return if trie: del trie trie = None _curpath = os.path.normpath( os.path.join(os.getcwd(), os.path.dirname(__file__))) abs_path = os.path.join(_curpath, dictionary) logger.debug("Building Trie..., from %s" % abs_path) t1 = time.time() if abs_path == os.path.join(_curpath, "dict.txt"): #defautl dictionary cache_file = os.path.join(tempfile.gettempdir(), "jieba.cache") else: #customer dictionary cache_file = os.path.join( tempfile.gettempdir(), "jieba.user." + str(hash(abs_path)) + ".cache") load_from_cache_fail = True if os.path.exists(cache_file) and os.path.getmtime( cache_file) > os.path.getmtime(abs_path): logger.debug("loading model from cache %s" % cache_file) try: trie, FREQ, total, min_freq = marshal.load( open(cache_file, 'rb')) load_from_cache_fail = False except: load_from_cache_fail = True if load_from_cache_fail: trie, FREQ, total = gen_trie(abs_path) FREQ = dict([(k, log(float(v) / total)) for k, v in FREQ.iteritems()]) #normalize min_freq = min(FREQ.itervalues()) logger.debug("dumping model to file cache %s" % cache_file) try: tmp_suffix = "." + str(random.random()) with open(cache_file + tmp_suffix, 'wb') as temp_cache_file: marshal.dump((trie, FREQ, total, min_freq), temp_cache_file) if os.name == 'nt': import shutil replace_file = shutil.move else: replace_file = os.rename replace_file(cache_file + tmp_suffix, cache_file) except: logger.error("dump cache file failed.") logger.exception("") initialized = True logger.debug("loading model cost %s seconds." % (time.time() - t1)) logger.debug("Trie has been built succesfully.")
def build_archive(self, libpath, delete_existing_resources=False): """Build the archive containing the Python library. """ if self.options.bundle_files <= 1: # Add pythonXY.dll as resource into the library file # # XXX We should add a flag to the exe so that it does not try to load pythonXY.dll # from the file system. # XXX XXX XXX with UpdateResources( libpath, delete_existing=delete_existing_resources) as resource: with open(pydll, "rb") as ifi: pydll_bytes = ifi.read() # We do not need to replace the winver string resource # in the python dll since it will be loaded via # MemoryLoadLibrary, and so python cannot find the # string resources anyway. if self.options.verbose > 1: print("Add resource %s/%s(%d bytes) to %s" % (os.path.basename(pydll), 1, len(pydll_bytes), libpath)) resource.add(type=os.path.basename(pydll), name=1, value=pydll_bytes) if self.options.optimize: bytecode_suffix = OPTIMIZED_BYTECODE_SUFFIXES[0] else: bytecode_suffix = DEBUG_BYTECODE_SUFFIXES[0] if self.options.compress: compression = zipfile.ZIP_DEFLATED else: compression = zipfile.ZIP_STORED # Create a zipfile and append it to the library file arc = zipfile.ZipFile(libpath, "a", compression=compression) # The same modules may be in self.ms.modules under different # keys; we only need one of them in the archive. for mod in set(self.mf.modules.values()): if mod.__code__: path = mod.__dest_file__ stream = io.BytesIO() stream.write(imp.get_magic()) if sys.version_info >= (3, 7, 0): stream.write(b"\0\0\0\0") # null flags stream.write(b"\0\0\0\0") # null timestamp stream.write(b"\0\0\0\0") # null size marshal.dump(mod.__code__, stream) arc.writestr(path, stream.getvalue()) elif hasattr(mod, "__file__"): assert mod.__file__.endswith(EXTENSION_TARGET_SUFFIX) if self.options.bundle_files <= 2: # put .pyds into the archive arcfnm = mod.__name__.replace( ".", "\\") + EXTENSION_TARGET_SUFFIX if self.options.verbose > 1: print("Add %s to %s" % (os.path.basename(mod.__file__), libpath)) arc.write(mod.__file__, arcfnm) else: # The extension modules will be copied into # dlldir. To be able to import it without dlldir # being on sys.path, create a loader module and # put that into the archive. pydfile = mod.__name__ + EXTENSION_TARGET_SUFFIX if self.options.verbose > 1: print("Add Loader for %s to %s" % (os.path.basename(mod.__file__), libpath)) loader = LOAD_FROM_DIR.format(pydfile) code = compile(loader, "<loader>", "exec", optimize=self.options.optimize) if hasattr(mod, "__path__"): path = mod.__name__.replace( ".", "\\") + "\\__init__" + bytecode_suffix else: path = mod.__name__.replace(".", "\\") + bytecode_suffix stream = io.BytesIO() stream.write(imp.get_magic()) if sys.version_info >= (3, 7, 0): stream.write(b"\0\0\0\0") # null flags stream.write(b"\0\0\0\0") # null timestamp stream.write(b"\0\0\0\0") # null size marshal.dump(code, stream) arc.writestr(path, stream.getvalue()) if self.options.bundle_files == 0: # put everything into the arc files = self.mf.all_dlls() elif self.options.bundle_files in (1, 2): # put only extension dlls into the arc files = self.mf.extension_dlls() else: arc.close() return for src in files: if self.options.verbose > 1: print("Add DLL %s to %s" % (os.path.basename(src), libpath)) arc.write(src, os.path.basename(src)) arc.close()
def marshal_dump(code, f): if isinstance(f, file): marshal.dump(code, f) else: f.write(marshal.dumps(code))
import parser, marshal, os, __future__ DUMPFILE = 'this_is_the_marshal_file' def reallycompile(tuples_or_src, filename, mode, flag_names): if type(tuples_or_src) is str: flags = 0 if 'nested_scopes' in flag_names: flags |= __future__.CO_NESTED if 'generators' in flag_names: flags |= __future__.CO_GENERATOR_ALLOWED if 'division' in flag_names: flags |= __future__.CO_FUTURE_DIVISION return compile(tuples_or_src, filename, mode, flags) return parser.compileast(parser.tuple2ast(tuples_or_src), filename) if __name__ == '__main__': s = file(DUMPFILE, "rb").read() tup = marshal.loads(s) tuples_or_src, filename, mode, done, flag_names = tup try: code = reallycompile(tuples_or_src, filename, mode, flag_names) except SyntaxError, e: code = e.msg, (e.filename, e.lineno, e.offset, e.text) done = True tup = (code, filename, mode, done, flag_names) marshal.dump(tup, file(DUMPFILE, "wb"))
def dump(path, data): """Serialize the given data and write it to a file at the given path.""" import marshal file = open(path, 'w') marshal.dump(data, file) file.close()
elements = set([g for g in generators]) for generation_index in xrange(generation_limit): generation_start_count = len(elements) for generator in generators: new_elements = [element * generator for element in elements] new_elements.extend([generator * element for element in elements]) for el in new_elements: elements.add(el) print "Generation #" + repr(generation_index), ": ", len(elements) generation_end_count = len(elements) if generation_end_count == generation_start_count: break return elements generators = [State(g) for g in get_generators()] elements = generate_loop(generators) print "Element count:", len(elements) element_values = [e.double for e in elements] file_name = '../data/states_' + str(len(elements))+ '.txt' with open(file_name, 'w') as f: marshal.dump(element_values, f) print "Saved states to " + file_name
def dump_stats(self, file): import marshal with open(file, 'wb') as f: self.create_stats() marshal.dump(self.stats, f)
def handle_item(path, item): marshal.dump((path, item), stdout) return True
def save_toc(self, tocpos): """Default - toc is a dict Gets marshaled to self.lib """ marshal.dump(self.toc, self.lib)
def export_item(self, item): marshal.dump(dict(self._get_serialized_fields(item)), self.file)