def pickle(obj, filename, protocol=0): f = None try: f = open(filename, "wb") p = Pickler(f, protocol) p.dump(obj) f.close() f = None # print "Pickled", filename finally: if f: f.close()
def sync(self): res = {} with dbm.open(self.db, self.flag) as db: for k, v in self.dict.items(): f = io.BytesIO() p = Pickler(f, protocol=self._protocol) p.dump(v) db[k] = f.getvalue() try: db.sync() except AttributeError: pass
def save_train_samples(self, iteration): folder = self.config.checkpoint if not os.path.exists(folder): os.makedirs(folder) filename = os.path.join( folder, self.get_checkpoint_file(iteration) + ".samples" ) with open(filename, "wb+") as f: Pickler(f).dump(self.train_samples_history) f.closed if self.config.delete_old_samples: self.delete_train_samples(iteration - 1)
def download_products(self): """ During the time given to the ProgressBar object, the download_products function allows you to download the products from the OpenFoodFacts database by saving only the tables and fields specified by the DESCRIPTION variable. Yields: dict: Contains two dictionaries named 'product' and 'category' corresponding to the names of the tables for the database. Each dictionary has the name of the field and its associated value in the database. """ # progress = Progressbar(5*60*1000) progress = Progressbar(60 * 1000) datas = None fp = open("data_products.bin", "wb") pck = Pickler(fp, HIGHEST_PROTOCOL) self.page = 1 datas = WrapAPI.load_products_from_opf(self.page, page_size=self.page_size) while datas: for prod in datas: p = {'product': {}, 'category': {}} for k, v in WrapAPI.DESCRIPTION['product'].items(): if k == 'energy': p['product']['energy'] = \ int(prod['nutriments']['energy']) elif k in prod.keys(): p['product'][v] = "%s" % prod[k].replace('"', '\"') k = 'categories' v = WrapAPI.DESCRIPTION['category'][k] if k in prod.keys(): p['category'][v] = "%s" % prod[k].replace('"', '\"') try: if p['product']['pname'] and p['category']['cname']: pck.dump(p) except KeyError: pass if progress(): self.page += 1 datas = WrapAPI.load_products_from_opf( self.page, page_size=self.page_size) else: datas = None fp.close()
def _tuer_sauvegarde(self): """ Efface la sauvegarde. Devient inefficace si le mode de sauvegarde change. """ try: with open(adresse_fichier_sauvegarde, 'wb') as fichier_sauvegarde: pick = Pickler(fichier_sauvegarde) pick.dump(None) except: print("Erreur lors de l'enregistrement du fichier")
def delete_service(self,name): # ouverture en lecture binaire with open('portfolio/model/data', "rb") as fic: record = Unpickler(fic) liste = record.load() for el in liste: if el.name == name: liste.remove(el) break # ouvreture en ecriture binaire with open('portfolio/model/data', "wb") as fic: record = Pickler(fic) record.dump(liste)
def pickle_entry(self, entry, zb_index): """Pickles an entry in its corresponding .entry file""" # Create entry file entry_file_path = self.get_entry_file_path(zb_index) file = open(entry_file_path, 'wb') # Write to file pickler = Pickler(file) pickler.dump(entry) # Close file file.close()
def save_train_examples(self, iteration): """ 保存训练数据(board, pi, v)为 @params iteration:迭代次数,保存数据为:checkpoint_iteration.pth.tar.examples """ # folder = args.checkpoint :'./temp/' folder = self.args.checkpoint if not os.path.exists(folder): os.makedirs(folder) file_name = os.path.join(folder, 'checkpoint_' + str(iteration) + '.pth.tar' + ".examples") with open(file_name, "wb+") as f: Pickler(f).dump(self.batch) f.closed
def save_game(current_map): """saves game to "save.txt" Arguments: current_map {list} -- current game map state """ try: file = open("save.txt", 'rb') except IOError: log.info("unable to save game") else: Pickler(file).dump(current_map) log.info("game saved!")
def convert(trace_directory: str, output_file_path: str) -> int: """ Convert CTF trace to pickle file. :param trace_directory: the trace directory :param output_file_path: the path to the output file that will be created :return: the number of events written to the output file """ with open(output_file_path, 'wb') as f: p = Pickler(f, protocol=4) count = ctf_to_pickle(trace_directory, p) return count
def saveTrainExamples(self, iteration=None): folder = self.args.checkpoint if not os.path.exists(folder): os.makedirs(folder) filename = os.path.join( folder, self.getCheckpointFile(iteration) + ".examples") lock = FileLock(filename + ".lock") with lock: with open(filename, "wb+") as f: Pickler(f).dump(self.trainExamplesHistory) f.closed
def _writeSessionFile(self, sessionFile, loadedData): f = None try: f = open(sessionFile, mode='wb') Pickler(f, protocol=pickle.HIGHEST_PROTOCOL).dump(loadedData) except Exception as e: print('Warning when read session file %s' % e) finally: if f: f.close()
def getSize(obj): """Calculate the size as cheap as possible """ # Try the cheap variants first. # Actually the checks ensure the code never fails but beeing sure # is better. try: # check if to return zero (length is zero) if len(obj) == 0: return 0 except: pass try: # check if ``IStreamableReference`` if IStreamableReference.providedBy(obj): size = obj.getSize() if size is not None: return size except: pass try: # string if isinstance(obj, str): return len(obj) except: pass try: # file like object methods = dir(obj) if "seek" in methods and "tell" in methods: currentPos = obj.tell() obj.seek(0, 2) size = obj.tell() obj.seek(currentPos) return size except: pass try: # fallback: pickling the object stream = StringIO() p = Pickler(stream, 1) p.dump(obj) size = stream.tell() except: size = None return size
def playGame(self, verbose=False): """ Executes one episode of a game. Returns: either winner: player who won the game (1 if player1, -1 if player2) or draw result returned from the game that is neither 1, -1, nor 0. """ players = [self.player2, None, self.player1] curPlayer = 1 board = self.game.getInitBoard() it = 0 actions = [] while self.game.getGameEnded(board, curPlayer)==0: it+=1 if verbose: assert(self.display) print("Turn ", str(it), "Player ", str(curPlayer)) self.display(board) action = players[curPlayer+1](self.game.getCanonicalForm(board, curPlayer), curPlayer) # valids = self.game.getValidMoves(self.game.getCanonicalForm(board, curPlayer),curPlayer) # don't know why it works when we just comment this out... # if valids[action] == 0: # print("\nArena bug occured in turn " + str(it) + ":\naction: " # + str(self.game.action_conversion__index_to_explicit(action)) # + ", turn player: " + str(Player(curPlayer))) # print("board:\n" + str(board)) # assert 1. in self.game.getValidMoves(board, curPlayer) # assert valids[action] > 0 # return None board.print_game_over_reason = False if self.replay: board.print_game_over_reason = True board, curPlayer = self.game.getNextState(board, curPlayer, action) board.print_game_over_reason = False actions.append(action) if verbose: assert(self.display) print("Game over: Turn ", str(it), "Result ", str(self.game.getGameEnded(board, 1))) self.display(board) if not self.replay: if not os.path.exists('./arena_replays/'): os.makedirs('./arena_replays/') with open("./arena_replays/arena_replay_" + str(self.game_id).zfill(3) + ".taflreplay", "wb+") as f: Pickler(f).dump(actions) self.game_id += 1 return self.game.getGameEnded(board, 1)
def logup(self, client: soc): """infos to send : username, pwd only that """ self.client_logup = client # One send tok2 = TokenTime(taille=20) username = self.client.recv(1024).decode("utf-8") time.sleep(0.02) pwd = self.client.recv(1024).decode("utf-8") if username not in self.accounts: self.accounts[username] = { "pwd": pwd, "admin": False, "token": tok2.getToken() } with open("connexion.login", "wb") as file: picklefile = Pickler(file) picklefile.dump(self.accounts) self.client_logup.send(b"201") time.sleep(0.1) self.client_logup.send(self.accounts[username]["token"].encode()) self.connected.append(username) elif username in self.accounts: self.client_logup.send(b"202") time.sleep(0.02) self.login(client) with open("connexion.login", "wb") as file: picklefile = Pickler(file) picklefile.dump(self.accounts)
def serializeToFile(filename, obj): """ This function serializes 'object' into 'filename'. Attention: If the specified file exists it will be overwritten. """ from pickle import Pickler from ToolBOSCore.External.atomicfile import AtomicFile Any.requireIsTextNonEmpty(filename) with AtomicFile(filename, 'wb') as f: # Pickle uses binary streams Pickler(f).dump(obj)
def AsyncTrainNetwork(game, args, trainhistory): #set gpu gpus = args.setGPU.split(',') os.environ["CUDA_VISIBLE_DEVICES"] = gpus[0] #create network for training nnet = nn(game, args.displaybar) try: nnet.load_checkpoint(folder=args.checkpoint, filename='best.pth.tar') #print("Retrain best model") except: pass #---load history file--- modelFile = os.path.join(args.checkpoint, "trainhistory.pth.tar") examplesFile = modelFile + ".examples" if not os.path.isfile(examplesFile): print(examplesFile) else: print("File with trainExamples found. Read it.") with open(examplesFile, "rb") as f: for i in Unpickler(f).load(): trainhistory.append(i) f.closed #---------------------- #---delete if over limit--- if len(trainhistory) > args.numItersForTrainExamplesHistory: print("len(trainExamplesHistory) =", len(trainhistory), " => remove the oldest trainExamples") del trainhistory[len(trainhistory) - 1] #------------------- #---extend history--- trainExamples = [] for e in trainhistory: trainExamples.extend(np.array(e)) #for e in trainhistory[:10]: # print(e) #---save history--- folder = args.checkpoint if not os.path.exists(folder): os.makedirs(folder) filename = os.path.join(folder, 'trainhistory.pth.tar' + ".examples") with open(filename, "wb+") as f: Pickler(f).dump(trainhistory) f.closed #------------------ nnet.train(trainExamples) nnet.save_checkpoint(folder=args.checkpoint, filename='train.pth.tar') #print(trainExamples[0][0].transpose(), trainExamples[0][2]) print(len(trainExamples))
def saveTrainExamples(self, iteration): folder = self.args.checkpoint if not os.path.exists(folder): os.makedirs(folder) s3_key_name = self.getCheckpointFile( iteration) + "." + self.args.generator_id + ".examples" filename = os.path.join( folder, self.getCheckpointFile(iteration) + "." + self.args.generator_id + ".examples") with open(filename, "wb+") as f: Pickler(f).dump(self.trainExamplesHistory) f.closed print(self.write_to_s3(s3_key_name, open(filename, "rb").read()))
def save_data(self): """ Sauvegarde les données à la fin de la partie. """ data = { "pot": self.pot, "random_number": self.random_number, "choice_number": self.choice_number, "bet_value": self.bet_value, "winnings": self.winnings } with open("./data.dat", 'wb') as data_file: data_pickle = Pickler(data_file) data_pickle.dump(data)
def test_save_load_list(): ListClass = build_composite_list( input_type=Tuple(MatrixContinuousDense(), CategoricalVector()), output_type=List(CategoricalVector()), ) algorithm = ListClass(DummyAlgorithm) fp = BytesIO() Pickler(fp).dump(algorithm) fp.seek(0) algorithm2 = Unpickler(fp).load() assert repr(algorithm) == repr(algorithm2)
def pickle_book(self, book, zb_index): """Pickles a book in the books' respective directory (zero-based index)""" if zb_index not in range(self.book_total): raise NonExistentBookException else: # Generates path src/bookshelf/aAD89k/aAD89k.bk dump_path = self.get_book_path(zb_index) dump_file_name = ''.join([self.books[zb_index][0], BOOK_FILE_EXT]) dump_file_path = os.path.join(dump_path, dump_file_name) dump_file = open(dump_file_path, "wb") pickler = Pickler(dump_file) pickler.dump(book) # Close file dump_file.close()
def saveTrainExamples( self, iteration): #save training examples (self.trainExamplesHistory to #args['checkpoint'] folder with name of self.getCheckpointFile with given iteration. folder = self.args['checkpoint'] if not os.path.exists( folder ): #if folder specified by args['checkpoint'] does not exist, then make it. os.makedirs(folder) filename = os.path.join( folder, self.getCheckpointFile(iteration) + ".examples") with open(filename, "wb+") as f: Pickler(f).dump(self.trainExamplesHistory) f.closed
def createPickleChild(self, name, child): if not os.path.isdir(self.path): resource.Resource.putChild(self, name, child) # xxx use a file-extension-to-save-function dictionary instead if type(child) == type(""): fl = open(os.path.join(self.path, name), 'wb') fl.write(child) else: if '.' not in name: name = name + '.trp' fl = open(os.path.join(self.path, name), 'wb') from pickle import Pickler pk = Pickler(fl) pk.dump(child) fl.close()
def doCheck(self): import glob pkgDirList = glob.glob(self.startDir + '/src/[A-Z]*/*') errMap = {} for pkg in pkgDirList: if not os.path.isdir(pkg): continue pkg = re.sub('^' + self.startDir + '/src/', '', pkg) missing = self.checkPkg(pkg) if missing: errMap[pkg] = missing from pickle import Pickler summFile = open('libchk.pkl', 'wb') pklr = Pickler(summFile, protocol=2) pklr.dump(errMap) summFile.close()
def save_training_examples(self, iteration, checkpoint_folder, trainExamplesHistory): logging.info("Checking if checkpoint folder exists.") if not os.path.exists(checkpoint_folder): logging.debug("Making checkpoint folder.") os.makedirs(checkpoint_folder) logging.info("Saving examples to checkpoint for iter: " + str(iteration)) filename = os.path.join( checkpoint_folder, self.get_examples_checkpoint_file(iteration) + ".examples") with open(filename, "wb+") as f: Pickler(f).dump(trainExamplesHistory)
def logToFile(stuff, filename, verbose=True): ''' Store one item (e.g. state list or networkx graph) to file. ''' filename = os.path.normcase(filename) directory = os.path.dirname(filename) if not os.path.exists(directory): os.makedirs(directory) f = open(filename, 'wb') p = Pickler(f, protocol=2) p.dump(stuff) f.close() if verbose: total = len(stuff) print("Written %i items to pickled binary file: %s" % (total, filename)) return filename
def enregistrer_score(adresse_fichier_score, joueur, nouveau_score): """Ouvre le fichier des scores, s'il existe, selon le joueur, le score est chargé et envoyé, sinon on renvoie 0. Se charge également de charger entièrement le fichier des scores""" global score try: with open(adresse_fichier_score, 'wb') as fichier_score: #Définition du score score[joueur] = score.get(joueur, 0) + nouveau_score #Enregistrement du socre pick = Pickler(fichier_score) pick.dump(score) except IOError: print("Erreur lors de l'enregistrement du fichier") return score[joueur]
def save(self, path): grille = np.zeros((9, 9), dtype=int) modif = np.zeros((9, 9), dtype=bool) possibilites = [] for i in range(9): possibilites.append([]) for j in range(9): grille[i, j] = self.blocs[i, j].get_val() modif[i, j] = self.blocs[i, j].is_modifiable() possibilites[i].append(self.blocs[i, j].get_possibilites()) with open(path, "wb") as fich: p = Pickler(fich) p.dump(grille) p.dump(modif) p.dump(possibilites) p.dump(self.chrono) p.dump(self.level)
def saveTrainExamples(self, iteration): ''' :param iteration: the number of current iteration which will be used to name the example file in an organised way :return: nothing Save the example file in an organised way in order to provide a way of reusing the examples in case of unexpected failure ''' folder = self.args.trainExampleCheckpoint if not os.path.exists(folder): os.makedirs(folder) filename = os.path.join(folder, self.getCheckpointFile(iteration) + ".examples") with open(filename, "wb+") as f: Pickler(f).dump(self.trainExamplesHistory) f.close()
def edit_map(*args, **kwargs): new_maze = list() new_line = str() i = 0 confirm = True has_entrance = 0 has_exit = 0 new_name = str(input("Name your new maze?\r\n")) print("To edit {}, please enter line by line your maze, the press Q when over".format(new_name)) print("Make sure you have one entrance (E) and one Exit (U)") while new_line != "Q": new_line = str(input("Enter line {}\r\n".format(i))) if "R" in new_line: has_entrance += 1 if "U" in new_line: has_exit += 1 if new_line.upper() == "Q": break new_maze.append(new_line) i += 1 if has_entrance != 1 or has_exit != 1: print("You can only have one entrance {} and one exit {}, quit program".format(has_entrance, has_exit)) exit() file_name = "{}.text".format(new_name) print("Map to save {}".format(file_name)) if os.path.isfile(file_name): ret = str(input("Are you sure you want to erase map {}? (Y/N)\r\n".format(file_name))) while ret.upper() not in ("Y", "N"): ret = str(input("Are you sure you want to erase save {}? (Y/N)\r\n".format(file_name))) if ret.upper() == "N": confirm = False else: confirm = True if confirm: with open(file_name, 'wb') as save: my_pickler = Pickler(save) my_pickler.dump({new_name: new_maze}) print("Map {} saved".format(file_name)) exit()