def pickle(obj, filename, protocol=0):
f = None
try:
f = open(filename, "wb")
p = Pickler(f, protocol)
p.dump(obj)
f.close()
f = None
# print "Pickled", filename
finally:
if f:
f.close()
def sync(self):
res = {}
with dbm.open(self.db, self.flag) as db:
for k, v in self.dict.items():
f = io.BytesIO()
p = Pickler(f, protocol=self._protocol)
p.dump(v)
db[k] = f.getvalue()
try:
db.sync()
except AttributeError:
pass
def save_train_samples(self, iteration):
folder = self.config.checkpoint
if not os.path.exists(folder):
os.makedirs(folder)
filename = os.path.join(
folder, self.get_checkpoint_file(iteration) + ".samples"
)
with open(filename, "wb+") as f:
Pickler(f).dump(self.train_samples_history)
f.closed
if self.config.delete_old_samples:
self.delete_train_samples(iteration - 1)
def download_products(self):
"""
During the time given to the ProgressBar object, the
download_products function allows you to download the products from
the OpenFoodFacts database by saving only the tables and fields
specified by the DESCRIPTION variable.
Yields:
dict:
Contains two dictionaries named 'product' and 'category'
corresponding to the names of the tables for the database.
Each dictionary has the name of the field and its
associated value in the database.
"""
# progress = Progressbar(5*60*1000)
progress = Progressbar(60 * 1000)
datas = None
fp = open("data_products.bin", "wb")
pck = Pickler(fp, HIGHEST_PROTOCOL)
self.page = 1
datas = WrapAPI.load_products_from_opf(self.page,
page_size=self.page_size)
while datas:
for prod in datas:
p = {'product': {}, 'category': {}}
for k, v in WrapAPI.DESCRIPTION['product'].items():
if k == 'energy':
p['product']['energy'] = \
int(prod['nutriments']['energy'])
elif k in prod.keys():
p['product'][v] = "%s" % prod[k].replace('"', '\"')
k = 'categories'
v = WrapAPI.DESCRIPTION['category'][k]
if k in prod.keys():
p['category'][v] = "%s" % prod[k].replace('"', '\"')
try:
if p['product']['pname'] and p['category']['cname']:
pck.dump(p)
except KeyError:
pass
if progress():
self.page += 1
datas = WrapAPI.load_products_from_opf(
self.page, page_size=self.page_size)
else:
datas = None
fp.close()
def _tuer_sauvegarde(self):
"""
Efface la sauvegarde.
Devient inefficace si le mode de sauvegarde change.
"""
try:
with open(adresse_fichier_sauvegarde, 'wb') as fichier_sauvegarde:
pick = Pickler(fichier_sauvegarde)
pick.dump(None)
except:
print("Erreur lors de l'enregistrement du fichier")
def delete_service(self,name):
# ouverture en lecture binaire
with open('portfolio/model/data', "rb") as fic:
record = Unpickler(fic)
liste = record.load()
for el in liste:
if el.name == name:
liste.remove(el)
break
# ouvreture en ecriture binaire
with open('portfolio/model/data', "wb") as fic:
record = Pickler(fic)
record.dump(liste)
def pickle_entry(self, entry, zb_index):
"""Pickles an entry in its corresponding .entry file"""
# Create entry file
entry_file_path = self.get_entry_file_path(zb_index)
file = open(entry_file_path, 'wb')
# Write to file
pickler = Pickler(file)
pickler.dump(entry)
# Close file
file.close()
def save_train_examples(self, iteration):
"""
保存训练数据（board, pi, v）为
@params iteration:迭代次数，保存数据为:checkpoint_iteration.pth.tar.examples
"""
# folder = args.checkpoint ：'./temp/'
folder = self.args.checkpoint
if not os.path.exists(folder):
os.makedirs(folder)
file_name = os.path.join(folder, 'checkpoint_' + str(iteration) + '.pth.tar' + ".examples")
with open(file_name, "wb+") as f:
Pickler(f).dump(self.batch)
f.closed
def save_game(current_map):
"""saves game to "save.txt"
Arguments:
current_map {list} -- current game map state
"""
try:
file = open("save.txt", 'rb')
except IOError:
log.info("unable to save game")
else:
Pickler(file).dump(current_map)
log.info("game saved!")
def convert(trace_directory: str, output_file_path: str) -> int:
"""
Convert CTF trace to pickle file.
:param trace_directory: the trace directory
:param output_file_path: the path to the output file that will be created
:return: the number of events written to the output file
"""
with open(output_file_path, 'wb') as f:
p = Pickler(f, protocol=4)
count = ctf_to_pickle(trace_directory, p)
return count
def saveTrainExamples(self, iteration=None):
folder = self.args.checkpoint
if not os.path.exists(folder):
os.makedirs(folder)
filename = os.path.join(
folder,
self.getCheckpointFile(iteration) + ".examples")
lock = FileLock(filename + ".lock")
with lock:
with open(filename, "wb+") as f:
Pickler(f).dump(self.trainExamplesHistory)
f.closed
def _writeSessionFile(self, sessionFile, loadedData):
f = None
try:
f = open(sessionFile, mode='wb')
Pickler(f, protocol=pickle.HIGHEST_PROTOCOL).dump(loadedData)
except Exception as e:
print('Warning when read session file %s' % e)
finally:
if f:
f.close()
def getSize(obj):
"""Calculate the size as cheap as possible
"""
# Try the cheap variants first.
# Actually the checks ensure the code never fails but beeing sure
# is better.
try:
# check if to return zero (length is zero)
if len(obj) == 0:
return 0
except:
pass
try:
# check if ``IStreamableReference``
if IStreamableReference.providedBy(obj):
size = obj.getSize()
if size is not None:
return size
except:
pass
try:
# string
if isinstance(obj, str):
return len(obj)
except:
pass
try:
# file like object
methods = dir(obj)
if "seek" in methods and "tell" in methods:
currentPos = obj.tell()
obj.seek(0, 2)
size = obj.tell()
obj.seek(currentPos)
return size
except:
pass
try:
# fallback: pickling the object
stream = StringIO()
p = Pickler(stream, 1)
p.dump(obj)
size = stream.tell()
except:
size = None
return size
def playGame(self, verbose=False):
"""
Executes one episode of a game.
Returns:
either
winner: player who won the game (1 if player1, -1 if player2)
or
draw result returned from the game that is neither 1, -1, nor 0.
"""
players = [self.player2, None, self.player1]
curPlayer = 1
board = self.game.getInitBoard()
it = 0
actions = []
while self.game.getGameEnded(board, curPlayer)==0:
it+=1
if verbose:
assert(self.display)
print("Turn ", str(it), "Player ", str(curPlayer))
self.display(board)
action = players[curPlayer+1](self.game.getCanonicalForm(board, curPlayer), curPlayer)
# valids = self.game.getValidMoves(self.game.getCanonicalForm(board, curPlayer),curPlayer)
# don't know why it works when we just comment this out...
# if valids[action] == 0:
# print("\nArena bug occured in turn " + str(it) + ":\naction: "
# + str(self.game.action_conversion__index_to_explicit(action))
# + ", turn player: " + str(Player(curPlayer)))
# print("board:\n" + str(board))
# assert 1. in self.game.getValidMoves(board, curPlayer)
# assert valids[action] > 0
# return None
board.print_game_over_reason = False
if self.replay:
board.print_game_over_reason = True
board, curPlayer = self.game.getNextState(board, curPlayer, action)
board.print_game_over_reason = False
actions.append(action)
if verbose:
assert(self.display)
print("Game over: Turn ", str(it), "Result ", str(self.game.getGameEnded(board, 1)))
self.display(board)
if not self.replay:
if not os.path.exists('./arena_replays/'):
os.makedirs('./arena_replays/')
with open("./arena_replays/arena_replay_" + str(self.game_id).zfill(3) + ".taflreplay", "wb+") as f:
Pickler(f).dump(actions)
self.game_id += 1
return self.game.getGameEnded(board, 1)
def logup(self, client: soc):
"""infos to send :
username, pwd
only that
"""
self.client_logup = client
# One send
tok2 = TokenTime(taille=20)
username = self.client.recv(1024).decode("utf-8")
time.sleep(0.02)
pwd = self.client.recv(1024).decode("utf-8")
if username not in self.accounts:
self.accounts[username] = {
"pwd": pwd,
"admin": False,
"token": tok2.getToken()
}
with open("connexion.login", "wb") as file:
picklefile = Pickler(file)
picklefile.dump(self.accounts)
self.client_logup.send(b"201")
time.sleep(0.1)
self.client_logup.send(self.accounts[username]["token"].encode())
self.connected.append(username)
elif username in self.accounts:
self.client_logup.send(b"202")
time.sleep(0.02)
self.login(client)
with open("connexion.login", "wb") as file:
picklefile = Pickler(file)
picklefile.dump(self.accounts)
def serializeToFile(filename, obj):
"""
This function serializes 'object' into 'filename'.
Attention: If the specified file exists it will be overwritten.
"""
from pickle import Pickler
from ToolBOSCore.External.atomicfile import AtomicFile
Any.requireIsTextNonEmpty(filename)
with AtomicFile(filename, 'wb') as f: # Pickle uses binary streams
Pickler(f).dump(obj)
def AsyncTrainNetwork(game, args, trainhistory):
#set gpu
gpus = args.setGPU.split(',')
os.environ["CUDA_VISIBLE_DEVICES"] = gpus[0]
#create network for training
nnet = nn(game, args.displaybar)
try:
nnet.load_checkpoint(folder=args.checkpoint, filename='best.pth.tar')
#print("Retrain best model")
except:
pass
#---load history file---
modelFile = os.path.join(args.checkpoint, "trainhistory.pth.tar")
examplesFile = modelFile + ".examples"
if not os.path.isfile(examplesFile):
print(examplesFile)
else:
print("File with trainExamples found. Read it.")
with open(examplesFile, "rb") as f:
for i in Unpickler(f).load():
trainhistory.append(i)
f.closed
#----------------------
#---delete if over limit---
if len(trainhistory) > args.numItersForTrainExamplesHistory:
print("len(trainExamplesHistory) =", len(trainhistory),
" => remove the oldest trainExamples")
del trainhistory[len(trainhistory) - 1]
#-------------------
#---extend history---
trainExamples = []
for e in trainhistory:
trainExamples.extend(np.array(e))
#for e in trainhistory[:10]:
# print(e)
#---save history---
folder = args.checkpoint
if not os.path.exists(folder):
os.makedirs(folder)
filename = os.path.join(folder, 'trainhistory.pth.tar' + ".examples")
with open(filename, "wb+") as f:
Pickler(f).dump(trainhistory)
f.closed
#------------------
nnet.train(trainExamples)
nnet.save_checkpoint(folder=args.checkpoint, filename='train.pth.tar')
#print(trainExamples[0][0].transpose(), trainExamples[0][2])
print(len(trainExamples))
def saveTrainExamples(self, iteration):
folder = self.args.checkpoint
if not os.path.exists(folder):
os.makedirs(folder)
s3_key_name = self.getCheckpointFile(
iteration) + "." + self.args.generator_id + ".examples"
filename = os.path.join(
folder,
self.getCheckpointFile(iteration) + "." + self.args.generator_id +
".examples")
with open(filename, "wb+") as f:
Pickler(f).dump(self.trainExamplesHistory)
f.closed
print(self.write_to_s3(s3_key_name, open(filename, "rb").read()))
def save_data(self):
"""
Sauvegarde les données à la fin de la partie.
"""
data = {
"pot": self.pot,
"random_number": self.random_number,
"choice_number": self.choice_number,
"bet_value": self.bet_value,
"winnings": self.winnings
}
with open("./data.dat", 'wb') as data_file:
data_pickle = Pickler(data_file)
data_pickle.dump(data)
def test_save_load_list():
ListClass = build_composite_list(
input_type=Tuple(MatrixContinuousDense(), CategoricalVector()),
output_type=List(CategoricalVector()),
)
algorithm = ListClass(DummyAlgorithm)
fp = BytesIO()
Pickler(fp).dump(algorithm)
fp.seek(0)
algorithm2 = Unpickler(fp).load()
assert repr(algorithm) == repr(algorithm2)
def pickle_book(self, book, zb_index):
"""Pickles a book in the books' respective directory (zero-based index)"""
if zb_index not in range(self.book_total):
raise NonExistentBookException
else:
# Generates path src/bookshelf/aAD89k/aAD89k.bk
dump_path = self.get_book_path(zb_index)
dump_file_name = ''.join([self.books[zb_index][0], BOOK_FILE_EXT])
dump_file_path = os.path.join(dump_path, dump_file_name)
dump_file = open(dump_file_path, "wb")
pickler = Pickler(dump_file)
pickler.dump(book)
# Close file
dump_file.close()
def saveTrainExamples(
self,
iteration): #save training examples (self.trainExamplesHistory to
#args['checkpoint'] folder with name of self.getCheckpointFile with given iteration.
folder = self.args['checkpoint']
if not os.path.exists(
folder
): #if folder specified by args['checkpoint'] does not exist, then make it.
os.makedirs(folder)
filename = os.path.join(
folder,
self.getCheckpointFile(iteration) + ".examples")
with open(filename, "wb+") as f:
Pickler(f).dump(self.trainExamplesHistory)
f.closed
def createPickleChild(self, name, child):
if not os.path.isdir(self.path):
resource.Resource.putChild(self, name, child)
# xxx use a file-extension-to-save-function dictionary instead
if type(child) == type(""):
fl = open(os.path.join(self.path, name), 'wb')
fl.write(child)
else:
if '.' not in name:
name = name + '.trp'
fl = open(os.path.join(self.path, name), 'wb')
from pickle import Pickler
pk = Pickler(fl)
pk.dump(child)
fl.close()
def doCheck(self):
import glob
pkgDirList = glob.glob(self.startDir + '/src/[A-Z]*/*')
errMap = {}
for pkg in pkgDirList:
if not os.path.isdir(pkg): continue
pkg = re.sub('^' + self.startDir + '/src/', '', pkg)
missing = self.checkPkg(pkg)
if missing: errMap[pkg] = missing
from pickle import Pickler
summFile = open('libchk.pkl', 'wb')
pklr = Pickler(summFile, protocol=2)
pklr.dump(errMap)
summFile.close()
def save_training_examples(self, iteration, checkpoint_folder,
trainExamplesHistory):
logging.info("Checking if checkpoint folder exists.")
if not os.path.exists(checkpoint_folder):
logging.debug("Making checkpoint folder.")
os.makedirs(checkpoint_folder)
logging.info("Saving examples to checkpoint for iter: " +
str(iteration))
filename = os.path.join(
checkpoint_folder,
self.get_examples_checkpoint_file(iteration) + ".examples")
with open(filename, "wb+") as f:
Pickler(f).dump(trainExamplesHistory)
def logToFile(stuff, filename, verbose=True):
''' Store one item (e.g. state list or networkx graph) to file. '''
filename = os.path.normcase(filename)
directory = os.path.dirname(filename)
if not os.path.exists(directory):
os.makedirs(directory)
f = open(filename, 'wb')
p = Pickler(f, protocol=2)
p.dump(stuff)
f.close()
if verbose:
total = len(stuff)
print("Written %i items to pickled binary file: %s" %
(total, filename))
return filename
def enregistrer_score(adresse_fichier_score, joueur, nouveau_score):
"""Ouvre le fichier des scores, s'il existe, selon le joueur, le score est chargé et envoyé,
sinon on renvoie 0. Se charge également de charger entièrement le fichier des scores"""
global score
try:
with open(adresse_fichier_score, 'wb') as fichier_score:
#Définition du score
score[joueur] = score.get(joueur, 0) + nouveau_score
#Enregistrement du socre
pick = Pickler(fichier_score)
pick.dump(score)
except IOError:
print("Erreur lors de l'enregistrement du fichier")
return score[joueur]
def save(self, path):
grille = np.zeros((9, 9), dtype=int)
modif = np.zeros((9, 9), dtype=bool)
possibilites = []
for i in range(9):
possibilites.append([])
for j in range(9):
grille[i, j] = self.blocs[i, j].get_val()
modif[i, j] = self.blocs[i, j].is_modifiable()
possibilites[i].append(self.blocs[i, j].get_possibilites())
with open(path, "wb") as fich:
p = Pickler(fich)
p.dump(grille)
p.dump(modif)
p.dump(possibilites)
p.dump(self.chrono)
p.dump(self.level)
def saveTrainExamples(self, iteration):
'''
:param iteration: the number of current iteration which will be used to name the example file in an organised
way
:return: nothing
Save the example file in an organised way in order to provide a way of reusing the examples in case of
unexpected failure
'''
folder = self.args.trainExampleCheckpoint
if not os.path.exists(folder):
os.makedirs(folder)
filename = os.path.join(folder, self.getCheckpointFile(iteration) + ".examples")
with open(filename, "wb+") as f:
Pickler(f).dump(self.trainExamplesHistory)
f.close()
def edit_map(*args, **kwargs):
new_maze = list()
new_line = str()
i = 0
confirm = True
has_entrance = 0
has_exit = 0
new_name = str(input("Name your new maze?\r\n"))
print("To edit {}, please enter line by line your maze, the press Q when over".format(new_name))
print("Make sure you have one entrance (E) and one Exit (U)")
while new_line != "Q":
new_line = str(input("Enter line {}\r\n".format(i)))
if "R" in new_line:
has_entrance += 1
if "U" in new_line:
has_exit += 1
if new_line.upper() == "Q":
break
new_maze.append(new_line)
i += 1
if has_entrance != 1 or has_exit != 1:
print("You can only have one entrance {} and one exit {}, quit program".format(has_entrance, has_exit))
exit()
file_name = "{}.text".format(new_name)
print("Map to save {}".format(file_name))
if os.path.isfile(file_name):
ret = str(input("Are you sure you want to erase map {}? (Y/N)\r\n".format(file_name)))
while ret.upper() not in ("Y", "N"):
ret = str(input("Are you sure you want to erase save {}? (Y/N)\r\n".format(file_name)))
if ret.upper() == "N":
confirm = False
else:
confirm = True
if confirm:
with open(file_name, 'wb') as save:
my_pickler = Pickler(save)
my_pickler.dump({new_name: new_maze})
print("Map {} saved".format(file_name))
exit()
