def __init__(self, inner, **kwargs): super().__init__(**kwargs) on = "qid" self.inner = inner self.disable = False if CACHE_DIR is None: init() # we take the md5 of the __repr__ of the pipeline to make a unique identifier for the pipeline # all different pipelines should return unique __repr_() values, as these are intended to be # unambiguous trepr = repr(self.inner) if "object at 0x" in trepr: warn( "Cannot cache pipeline %s has a component has not overridden __repr__" % trepr) self.disable = True uid = hashlib.md5(bytes(trepr, "utf-8")).hexdigest() destdir = path.join(CACHE_DIR, uid) os.makedirs(destdir, exist_ok=True) definition_file = path.join(destdir, DEFINITION_FILE) if not path.exists(definition_file): with open(definition_file, "w") as f: f.write(trepr) self.chest = Chest( path=destdir, dump=lambda data, filename: pd.DataFrame.to_pickle(data, filename) if isinstance(data, pd.DataFrame) else pickle.dump( data, filename, protocol=1), load=lambda filehandle: pickle.load(filehandle) if ".keys" in filehandle.name else pd.read_pickle(filehandle)) self.hits = 0 self.requests = 0
def post_frame(ans, params, args): # Analysis! from numpy.linalg import eigh from numpy import argsort ev, vecs = eigh(ans["overlap"]) idx = argsort(ev)[::-1] ans["ev"] = ev[idx] / params["snapshots"] ans["vecs"] = vecs[:, idx] print("Sanity check volume: {:f}".format(ans["volume"])) print( ans["x_min"], ans["x_max"], ans["y_min"], ans["y_max"], ans["z_min"], ans["z_max"], ) print(ans["overlap"]) print("Singular values:") print(ans["ev"]) print(ans["vecs"]) from chest import Chest cpath = '{:s}-chest-{:03d}'.format(args.chest_path, ans["frame"]) c = Chest(path=cpath) for key in ans.keys(): c[ans['time'], key] = ans[key] ans.clear() c.flush() ans["cpath"] = cpath return
def post_frame(ans, params, args): # Analysis! from numpy.linalg import eigh from numpy import argsort ev, vecs = eigh(ans["overlap"]) idx = argsort(ev)[::-1] ans["ev"] = ev[idx] / params["snapshots"] ans["vecs"] = vecs[:,idx] print("Sanity check volume: {:f}".format(ans["volume"])) print(ans["x_min"], ans["x_max"], ans["y_min"], ans["y_max"],ans["z_min"], ans["z_max"],) print(ans["overlap"]) print("Singular values:") print(ans["ev"]) print(ans["vecs"]) from chest import Chest cpath = '{:s}-chest-{:03d}'.format(args.chest_path, ans["frame"]) c = Chest(path=cpath) for key in ans.keys(): c[ans['time'], key] = ans[key] ans.clear() c.flush() ans["cpath"] = cpath return
def construire(phrases): classes = Chest(path="classes", dump=dump, load=load) features = Chest(path="features", dump=dump, load=load) for p in phrases: vecteur = Chest(path=p, dump=dump, load=load) vecteur_log = Chest(path=p, dump=dump, load=load)
def outer_process(job): """ Process to be executed in the outer IPython.parallel map """ # Split the arguments args, params, frame = job # always need these from importlib import import_module MR = import_module(args.mapreduce) # Initialize the MapReduce data with base cases # Returns job list to pass to map jobs = MR.MR_init(args, params, frame) # Copy a base case in which to reduce the results from copy import deepcopy ans = deepcopy(jobs[0][4]) # Map! import time as time_ ttime = time_.time() if args.thread < 2: results = map(inner_process, jobs) else: from multiprocessing import Pool p = Pool(processes=args.thread) results = p.imap_unordered(inner_process, jobs, chunksize=1) if args.verbose: print(' Map took {:f}s on {:d} processes'.format( time_.time() - ttime, args.thread)) # Reduce! ttime = time_.time() for r in results: MR.reduce_(ans, r) if args.thread >= 2: p.close() if args.verbose: print(' Reduce took {:f}s on {:d} processes'.format( time_.time() - ttime, args.thread)) ans["frame"] = frame # Analysis! post = import_module(args.post) post.post_frame(ans, params, args) post.plot_frame(ans, params, args) # Save the results to file! from chest import Chest cpath = '{:s}-chest-{:03d}'.format(args.name, frame) c = Chest(path=cpath) for key in ans.keys(): c[ans['time'], key] = ans[key] c.flush() return cpath
def outer_process(job): """ Process to be executed in the outer IPython.parallel map """ # Split the arguments args, params, frame = job # always need these from importlib import import_module MR = import_module(args.mapreduce) # Initialize the MapReduce data with base cases # Returns job list to pass to map jobs = MR.MR_init(args, params, frame) # Copy a base case in which to reduce the results from copy import deepcopy ans = deepcopy(jobs[0][4]) # Map! import time as time_ ttime = time_.time() if args.thread < 2: results = map(inner_process, jobs) else: from multiprocessing import Pool p = Pool(processes=args.thread) results = p.imap_unordered(inner_process, jobs, chunksize = 1) if args.verbose: print(' Map took {:f}s on {:d} processes'.format(time_.time()-ttime, args.thread)) # Reduce! ttime = time_.time() for r in results: MR.reduce_(ans, r) if args.thread >= 2: p.close() if args.verbose: print(' Reduce took {:f}s on {:d} processes'.format(time_.time()-ttime, args.thread)) ans["frame"] = frame # Analysis! post = import_module(args.post) post.post_frame(ans, params, args) post.plot_frame(ans, params, args) # Save the results to file! from chest import Chest cpath = '{:s}-chest-{:03d}'.format(args.name, frame) c = Chest(path=cpath) for key in ans.keys(): c[ans['time'], key] = ans[key] c.flush() return cpath
def __init__(self, arg1, shape=None, chunksize=(int(1e4), int(1e4)), dtype=None): if shape is None: raise ValueError('Demand shape for now') if not isinstance(arg1, dict): raise TypeError('support other things later') self.shape = shape self.chunksize = chunksize self.the_dict = Chest(arg1)
def post_frame(ans, params, args): from chest import Chest cpath = '{:s}-chest-{:03d}'.format(args.chest_path, ans["frame"]) c = Chest(path=cpath) for key in ans.keys(): c[ans['time'], key] = ans[key] ans.clear() c.flush() ans["cpath"] = cpath return
def main2(): from nltk.corpus import gutenberg from chest import Chest from heapdict import heapdict struc = Chest() texte = gutenberg.raw('carroll-alice.txt') phrases = iter(texte.split('\n')) # print(list(map(lambda x: list(generate_regex(OptimString(point=Point('.'), seq="".join(x)))), everygrams(next(phrases))))) for element in ngrams(next(phrases), 10): chaine = OptimString(point=Point('.'), seq="".join(element)) for x in generate_regex(chaine): # print(str(x)) struc[x] = True print(len(list(struc.keys())))
def __init__(self, name, maxLength, location=os.getcwd()): assert isinstance(maxLength, int), "\'maxLength\' must be an integer!" assert isinstance(location, str) | isinstance(location, PurePath), "\'location\' must be a string or PurePath!" assert isinstance(name, str), "\'name\' must be a string!" assert maxLength >= 1, "\'maxLength\' must be at-least one!" # Private variables self.__maxLength = 0 self.__data = deque([], maxLength) self.__name = name self.__location = location self.__storage = Chest(path=location) # Change dumping/loading method here self.maxLength = maxLength self.load()
def __init__(self, difficulty, player): self.hasMonster = False self.hasBoss = False self.hasChest = False self.isDone = False self.difficulty = difficulty self.settings = Settings() self.goal = self.settings.getGoal() self.player = player self.chest = Chest(self.player) self.monster = Monster() self.inspected = False self.handler = Stringhandler() self.boss = Endboss() self.endboss = self.boss.randomBoss(player)
def set_map(self): self.player = Player(336, 672, self.block, self.block, self.character_frontpng, 0, 4, 0, 0) self.chests = [ Chest(360, 48, self.block, self.block, self.chestpng, 0), Chest(360, 300, self.block, self.block, self.chestpng, 0), ] self.stool = GameObjects(100, 200, self.block, self.block, self.stoolpng, 1) self.enemies = [ Enemy(0, 144, self.block, self.block, self.keypng, 1, 2), Enemy(0, 144 + self.block, self.block, self.block, self.keypng, 1, 2), Enemy(0, 144 - self.block, self.block, self.block, self.keypng, 1, 2), ]
class DOK: def __init__(self, arg1, shape=None, chunksize=(int(1e4), int(1e4)), dtype=None): if shape is None: raise ValueError('Demand shape for now') if not isinstance(arg1, dict): raise TypeError('support other things later') self.shape = shape self.chunksize = chunksize self.the_dict = Chest(arg1) def __getitem__(self, index): """If key=(i,j) is a pair of integers, return the corresponding element. TODO: If either i or j is a slice or sequence, return a new sparse matrix with just these elements. MUSINGS ON THIS: How can we construct a new DOK in a parallel out of core way? We need to create a new Chest with elements of another, moving them through memory. This sounds like a perfect job for dask! """ if ((isinstance(index, tuple)) and (len(index) == 2) and (all(map(lambda x: isinstance(x, int), index)))): return self.the_dict[index] if isinstance(index, slice): pass def __contains__(self, key): return self.the_dict.__contains__(key)
def fft(x, axis=-1, chunksize=2**26, available_memory=(4 * 1024**3), cache=None): """Simple wrapper for DAFT FFT function This function calls the DAFT function, but also performs the computation of the FFT, and returns the result as a numerical array. Parameters ---------- x : array_like Input array, can be complex. axis : int, optional Axis over which to compute the FFT. If not given, the last axis is used. chunksize : int, optional Chunksize to use when splitting up the input array. Default is 2**24, which is about 64MB -- a reasonable target that reduces memory usage. available_memory : int, optional Maximum amount of RAM to use for caching during computation. Defaults to 4*1024**3, which is 4GB. """ if cache is None: from chest import Chest # For more flexible caching cache = Chest(available_memory=available_memory) X_dask = DAFT(x, axis=axis, chunksize=chunksize) return X_dask.compute(cache=cache)
def post_frame(ans, params, args): # Analysis! ans['TAbs'] = max(ans['TMax'], -ans['TMin']) ans['PeCell'] = ans['UAbs']*ans['dx_max']/params['conductivity'] ans['ReCell'] = ans['UAbs']*ans['dx_max']/params['viscosity'] # Mixing height L = params["extent_mesh"][2] - params["root_mesh"][2] h = 0. tmax = np.max(ans["t_proj_z"].to_array()) tmin = np.min(ans["t_proj_z"].to_array()) tzero = (tmax + tmin) / 2 h_cabot = 0. for i in range(ans["t_proj_z"].to_array().shape[0]): if ans["t_proj_z"].to_array()[i] < tzero: h_cabot += (ans["t_proj_z"].to_array()[i] - tmin) else: h_cabot += (tmax - ans["t_proj_z"].to_array()[i]) ans["h"] = h_cabot zs = ans['z_z'].to_array() from utils.my_utils import find_root h_visual = ( find_root(zs, ans["t_proj_z"].to_array(), y0 = tmax - (tmax - tmin)*.01) - find_root(zs, ans["t_proj_z"].to_array(), y0 = tmin + (tmax - tmin)*0.1)) / 2. h_exp = find_root(zs, np.array(ans["t_max_z"].to_array()), y0 = 0.0) ans["H"] = h_visual ans["H_exp"] = h_exp plot_frame(ans, params, args) from interfaces.abstract import AbstractSlice from chest import Chest cpath = '{:s}-chest-{:03d}'.format(args.chest_path, ans["frame"]) c = Chest(path=cpath) for key in ans.keys(): if isinstance(ans[key], AbstractSlice): c[ans['time'], key] = ans[key].to_array() else: c[ans['time'], key] = ans[key] ans.clear() c.flush() ans["cpath"] = cpath return
def max_memory(self, max_memory): # set Cache to protect memory spilling if max_memory is not None: available_memory = max_memory else: available_memory = psutil.virtual_memory().available self.__max_memory = available_memory self.cache = Chest(available_memory=available_memory)
def main(): dico = Chest(path='francais_tatoeba_5bis-char-max') dico2 = defaultdict(set) i = 0 with open( "/Users/korantin/Documents/Projects/Lexiques/francais_col123.txt", 'r') as ba: tmp = ba.read().splitlines() for phrase in tmp: (ind, ln, phrase) = phrase.strip().split('\t') if len(phrase) <= 10: print(phrase) print(i, len(tmp)) estime(phrase, tmp, memoire=dico, d=dico2) i += 1 for x, y in dico2.items(): dico[x] *= (len(tmp) / len(y)) dico.flush() print(dico.items(), sep='\n')
def check_if_collided(self): for enemy in self.enemies: if self.detect_collision(self.player, enemy): self.set_map() return True for chest in self.chests: if self.detect_collision(self.player, chest): Chest(chest.x, chest.y, self.block, self.block, 'crossgame/assets/chest_open.png', 0) self.game_window.blit(chest.image, (chest.x, chest.y)) return False
def generation_room(x_initial, y_initial, room): x = x_initial y = y_initial for i in range(len(room)): for j in range(len(room[0])): if room[i][j] == "f": pf = Floor(x, y, random.randint(0,5)) layer_0.add(pf) if room[i][j] == "w": pf = Wall(x, y, 4) layer_1.add(pf) if room[i][j] == "d": pf = Wall(x, y, 1) layer_2.add(pf) if room[i][j] == "v": pf = Wall(x, y, 2) layer_2.add(pf) if room[i][j] == "x": pf = Wall(x, y, 0) layer_0.add(pf) wall.append(pf) if room[i][j] == "u": pf = Wall(x, y, 3) layer_1.add(pf) r = random.randint(0, 100) if r <= 5: torch = Torch(x, y, 3) layer_1.add(torch) if room[i][j] == "c": r = random.randint(3, 10) if r >= 5: pf = Floor(x, y, 6) layer_0.add(pf) pf = Chest(x, y-6, random.randint(1, 2)) layer_1.add(pf) chest.append(pf) else: pf = Floor(x, y, 6) layer_0.add(pf) if room[i][j] == "z": pf = Monster_spawn(x, y) layer_0.add(pf) r = random.randint(2, 5) for i in range(r): x_ = random.randint(pf.rect.x-32, pf.rect.x+64) y_ = random.randint(pf.rect.y-32, pf.rect.y+64) pf = Monster(x_, y_, random.randint(1, 4)) layer_1.add(pf) layer_monster.add(pf) layer_all_monster_and_layer.add(pf) x += 32 x = x_initial y += 32
def test_cache_options(): try: from chest import Chest except ImportError: return cache = Chest() def inc2(x): assert 'y' in cache return x + 1 with dask.set_options(cache=cache): get_sync({'x': (inc2, 'y'), 'y': 1}, 'x')
def read_geno(bfile, freq_thresh, threads, check=False, max_memory=None, usable_snps=None, normalize=False): # set Cache to protect memory spilling if max_memory is not None: available_memory = max_memory else: available_memory = psutil.virtual_memory().available cache = Chest(available_memory=available_memory) (bim, fam, g) = read_plink(bfile) # read the files using pandas_plink g_std = g.std(axis=1) if check: with ProgressBar(), dask.config.set(pool=ThreadPool(threads)): print('Removing invariant sites') idx = (g_std != 0).compute(cache=cache) g = g[idx, :] bim = bim[idx].copy().reset_index(drop=True) bim.i = bim.index.tolist() del idx gc.collect() if usable_snps is not None: idx = bim[bim.snp.isin(usable_snps)].i.tolist() g = g[idx, :] bim = bim[bim.i.isin(idx)].copy().reset_index(drop=True) bim.i = bim.index.tolist() mafs = g.sum(axis=1) / (2 * n) if freq_thresh > 0 else None # Filter MAF if freq_thresh > 0: print('Filtering MAFs smaller than', freq_thresh) print(' Genotype matrix shape before', g.shape) assert freq_thresh < 0.5 good = (mafs < (1 - float(freq_thresh))) & (mafs > float(freq_thresh)) with ProgressBar(): with dask.config.set(pool=ThreadPool(threads)): good, mafs = dask.compute(good, mafs, cache=cache) g = g[good, :] print(' Genotype matrix shape after', g.shape) bim = bim[good] bim['mafs'] = mafs[good] del good gc.collect() if normalize: mean = g.mean(axis=1) g = (g.T - mean) / g_std else: g = g.T return g, bim, fam
def do_pca(g, n_comp): """ Perform a PCA on the genetic array and return n_comp of it :param g: Genotype array :param n_comp: Number of components sought :return: components array """ cache = Chest(available_memory=available_memory, path=os.getcwd()) pca = PCA(n_components=n_comp) pca = pca.fit_transform(g) return pca.compute(cache=cache)
def __init__(self, n, stocha, obs): self.gold_mines = [] self.forests = [] self.obstacles = [] self.board = [] self.quotas = [False for k in range(NB_RESOURCES)] self.n= n self.time = 0 self.reward = 0 self.stocha = stocha # Board instantiation for i in range(n): for j in range(n): if (i, j) in OBSTACLES and obs: obstacle = Obstacle([i, j]) self.board.append(obstacle) self.obstacles.append(obstacle) elif (i,j) in GOLD_MINES: gold_mine = GoldMine([i,j]) self.board.append(gold_mine) self.gold_mines.append(gold_mine) elif (i,j) in FORESTS: forest = Forest([i,j]) self.board.append(forest) self.forests.append(forest) elif (i,j) == PLAYER: self.player = Player([i,j], NOTHING) self.board.append(FreeTile([i,j])) elif (i,j) == CHEST: self.chest = Chest([i,j]) self.chest_next = True self.board.append(self.chest) else: self.board.append(FreeTile([i,j])) self.gold_mines_next = [False for k in self.gold_mines] self.forests_next = [False for k in self.forests]
def test_cache_options(): try: from chest import Chest except ImportError: return cache = Chest() def inc2(x): assert "y" in cache return x + 1 with dask.config.set(cache=cache): get_sync({"x": (inc2, "y"), "y": 1}, "x")
def exec_dag(dag, num_workers=None, cache_size=1e9, scheduler=dask.threaded.get, show_progress=False): cache = Chest(available_memory=cache_size) if show_progress: with ProgressBar(), dask.set_options(cache=cache, num_workers=num_workers): result = dag.compute(get=scheduler) else: with cache: result = dag.compute(get=scheduler, num_workers=num_workers) return result
def estime(ba, memoire: Chest): """ L'estimation maximale des paramètre se fait par le calcul du powerset sur un ensemble BA (Base d'Apprentissage). On fait donc un tf-idf mais en ayant en vocabulaire les composantes du powerset :param BA: :return: """ len_ba = sum(pow(2, len(x)) for x in ba) with Pool(processes=5) as p: for element in ba: for x in p.map(lambda i: powerset(element, i), range(len(element))): for y in x: if not memoire.get(x): memoire[x] = frequence_brute(y, element, True) for x in memoire: memoire[x] /= len(memoire)
def load_from_archive(names, arch): cs = [] for name in names: cs.append(Chest(path = "{:s}-results".format(name), open = partial(glopen, endpoint=arch), open_many = partial(glopen_many, endpoint=arch), available_memory = 1e12)) scs = [CachedSlict(c) for c in cs] ps = [] for name in names: with glopen( "{:s}.json".format(name), mode='r', endpoint = arch, ) as f: ps.append(json.load(f)) if len(names) == 1: return cs[0], scs[0], ps[0] return cs, scs, ps
def generate_regex(seq: OptimString, memo: Chest = None): file = deque() sortie = deque() file.appendleft(seq) while file: current = file.pop() if not all(x == '.' for x in str(current)): if not memo.get(current): yield current if not isinstance(current.data_pointe[-1], Point): current = current.add_point() file.appendleft(current) for _ in range( current.control.get(current.get_point)[-1] + 1, len(current.data)): current = current.deplace_point() file.appendleft(current)
def enter(): global boy, ground, background, spikeList, Hp_Bar, Monster_Bear_List, Monster_Mage_List, ChestList, CHicken,\ GUI, FlatFormList, FLAG background = Background() boy = Boy(background) Hp_Bar = HP_BAR() ground = [Ground(i, background) for i in range(len(Ground.groundList))] Monster_Bear_List = [ Monster_bear(i, background) for i in range(len(Monster_bear.posList)) ] Monster_Mage_List = [ Monster_mage(i) for i in range(len(Monster_mage.posList)) ] spikeList = [Spike(i, background) for i in range(len(Spike.spikeList))] ChestList = [Chest(i, background) for i in range(len(Chest.chestList))] GUI = Gui() FlatFormList = [ Flatform(i, background) for i in range(len(Flatform.flatFormList)) ] FLAG = Flag(background) game_world.add_object(background, 0) for i in range(len(ground)): game_world.add_object(ground[i], 0) game_world.add_object(boy, 2) game_world.add_object(Hp_Bar, 1) for i in range(len(Monster_Bear_List)): game_world.add_object(Monster_Bear_List[i], 1) for i in range(len(Monster_Mage_List)): game_world.add_object(Monster_Mage_List[i], 1) for i in range(len(ChestList)): game_world.add_object(ChestList[i], 1) for i in range(len(spikeList)): game_world.add_object(spikeList[i], 1) game_world.add_object(GUI, 1) for i in range(len(FlatFormList)): game_world.add_object((FlatFormList[i]), 1) game_world.add_object(FLAG, 1) background.set_center_object(boy) boy.set_background(background)
def fft_to_hdf5(x, filename, axis=-1, chunksize=2**26, available_memory=(4 * 1024**3), cache=None): """Simple wrapper for DAFT FFT function that writes to HDF5 This function calls the DAFT function, but also performs the computation of the FFT, and outputs the result into the requested HDF5 file Parameters ---------- x : array_like Input array, can be complex. filename : string Relative or absolute path to HDF5 file. If this string contains a colon, the preceding part is taken as the filename, while the following part is taken as the dataset group name. The default group name is 'X'. axis : int, optional Axis over which to compute the FFT. If not given, the last axis is used. chunksize : int, optional Chunksize to use when splitting up the input array. Default is 2**24, which is about 64MB -- a reasonable target that reduces memory usage. available_memory : int, optional Maximum amount of RAM to use for caching during computation. Defaults to 4*1024**3, which is 4GB. """ from h5py import File from dask import set_options from dask.array import store if cache is None: from chest import Chest # For more flexible caching cache = Chest(available_memory=available_memory) if ':' in filename: filename, groupname = filename.split(':') else: groupname = 'X' X_dask = DAFT(x, axis=axis, chunksize=chunksize) with set_options(cache=cache): with File(filename, 'w') as f: output = f.create_dataset(groupname, shape=X_dask.shape, dtype=X_dask.dtype) store(X_dask, output) return
required=True, help="directory where to find SQL SELECT statements," " one single statement per file. Can be configured with monthly_check.source" ) parser.add_argument( '-o', '--output', conf_key="monthly_check.output", required=True, help= "Where to write the report. Can be configured with monthly_check.output." ) parser.add_argument( '-m', '--month', conf_key="monthly_check.month", default=month_date.previous(), type=month_date.from_str, help= "6 digits, 4 for the year then 2 for the month (ex.: 201901 is Jan 2019)." " Default value is last month.") CursorProvider.add_arguments_to(parser) args = parser.parse_args() Chest(vars(args)) logging.basicConfig(level=config["logging"]["level"], format=config["logging"]["format"]) monthly_check(args.month, args.source, args.output)
# -*- coding: utf-8 -*- __author__ = 'Bruno Konrad' import sys from chest import Chest from chest.chest_dropbox import DropboxAuthorizator, DropboxStorager if __name__ == "__main__": if len(sys.argv) == 3: dropboxStorager = DropboxStorager() dropboxAuthorizator = DropboxAuthorizator() chest = Chest(sys.argv, dropboxStorager) chest.set_authorizator(dropboxAuthorizator) chest.start() else: print("Você precisa dizer o arquivo que deseja salvar e em que pasta no Dropbox.")
from chest import Chest c = Chest() # Acts like a normal dictionary c['x'] = [1, 2, 3] print c['x']
# Set up the frame arguments from mapcombine import outer_process jobs = [[args, params, i] for i in range(args.frame, args.frame_end+1)] # schedule the frames, one IPython process each # if only one process or parallel not set, use normal map import time start_time = time.time() if len(jobs) > 1 and args.parallel: from IPython.parallel import Client p = Client(profile='mpi') stuff = p.load_balanced_view().map_async(outer_process, jobs) else: stuff = map(outer_process, jobs) # insert new results into the out-of-core dictionary (Chest) nelm = params["shape_mesh"][0] * params["shape_mesh"][1] * params["shape_mesh"][2] from chest import Chest for i, res in enumerate(stuff): c1 = Chest(path=res['cpath']) c = Chest(path=args.chest_path) c.update(c1) c.flush() c1.drop() # Print a progress update run_time = time.time() - start_time print("Processed {:d}th frame after {:f}s ({:f} eps)".format(i, run_time, (i+1)*nelm/run_time))