def init_bdd(self):
liste = list()
liste.append(Service())
#ouvreture en ecriture binaire
with open('portfolio/model/data',"wb") as fic:
record = Pickler(fic)
record.dump(liste)
def create_initial_file(self):
"""Creates the initial empty list bookshelf/bookshelf.dump file"""
init_list = list()
file = open(self.bookshelf_file_path, 'wb')
pcklr = Pickler(file)
pcklr.dump(init_list)
file.close()
def save():
print(Donnees.admin)
print("Test : Saved")
for i in range(Row):
test = self.table.item(i, 1).text()
if test == "True":
test = True
else:
if self.table.item(i, 0).text() == "Shayajs" or self.table.item(i, 0).text() == "uadmin":
test = True
if self.table.item(i, 1).text() == "False":
self.label001.show()
else:
test = False
if test and self.table.item(i, 0).text() not in Donnees.admin:
Donnees.admin.append(self.table.item(i, 0).text())
elif not test and self.table.item(i, 0).text() in Donnees.admin:
Donnees.admin.remove(self.table.item(i, 0).text())
print(Donnees.admin)
with open("./bin/admin.spi", "wb") as adminfile:
a = Pickler(adminfile)
a.dump(Donnees.admin)
b = Online()
b.uploadftp("admin.spi", "./bin/")
def _testTracingOrProfileState(self, do_pickle=False, **kw):
t = tasklet(self.Tasklet)
t(**kw)
t.run()
self.foo()
n = len(self.trace)
self.foo()
n2 = len(self.trace)
self.assertEqual(n, n2)
if do_pickle:
io = StringIO()
p = Pickler(io, -1)
p.persistent_id = self.persistent_id
p.dump(t)
t.remove()
t.bind(None)
p = Unpickler(StringIO(io.getvalue()))
p.persistent_load = self.persistent_load
t = p.load()
p = None
io = None
t.run()
self.foo()
n = len(self.trace)
self.foo()
n2 = len(self.trace)
self.assertEqual(n, n2)
def testDeepCopyCanInvalidate(self):
"""
Tests regression for invalidation problems related to missing
readers and writers values in cloned objects (see
http://mail.zope.org/pipermail/zodb-dev/2008-August/012054.html)
"""
import ZODB.MappingStorage
database = DB(ZODB.blob.BlobStorage(
'blobs', ZODB.MappingStorage.MappingStorage()))
connection = database.open()
root = connection.root()
transaction.begin()
root['blob'] = Blob()
transaction.commit()
stream = StringIO()
p = Pickler(stream, 1)
p.dump(root['blob'])
u = Unpickler(stream)
stream.seek(0)
clone = u.load()
clone._p_invalidate()
# it should also be possible to open the cloned blob
# (even though it won't contain the original data)
clone.open()
# tearDown
database.close()
def deepCopy(obj):
stream = StringIO()
p = Pickler(stream, 1)
p.dump(obj)
stream.seek(0)
u = Unpickler(stream)
return u.load()
def testDeepCopyCanInvalidate(self):
"""
Tests regression for invalidation problems related to missing
readers and writers values in cloned objects (see
http://mail.zope.org/pipermail/zodb-dev/2008-August/012054.html)
"""
import ZODB.MappingStorage
database = DB(
ZODB.blob.BlobStorage('blobs',
ZODB.MappingStorage.MappingStorage()))
connection = database.open()
root = connection.root()
transaction.begin()
root['blob'] = Blob()
transaction.commit()
stream = StringIO()
p = Pickler(stream, 1)
p.dump(root['blob'])
u = Unpickler(stream)
stream.seek(0)
clone = u.load()
clone._p_invalidate()
# it should also be possible to open the cloned blob
# (even though it won't contain the original data)
clone.open()
# tearDown
database.close()
def import_from_file(self, file):
try:
with open(file, "r") as import_file:
imported_sets = import_file.read()
imported_sets = imported_sets.split('\n')
for set in imported_sets:
#Isolate setcode
set = set.split()
selected_code = set[0]
#Recover full name
selected_name = ""
for i in range(1, len(set)):
selected_name += set[i]
if i < len(set) - 1:
selected_name += " "
#Register archetype
if not self.is_name_in_sets(selected_name) and not self.is_code_in_sets(selected_code):
self.add_archetype(selected_name, selected_code)
#Save changes
with open("util/sets", "wb") as set_file:
pick = Pickler(set_file)
pick.dump(self)
except FileNotFoundError:
print("Could not find file", file)
def parseLog(self):
logData = {}
logRE = re.compile('^([1-9][0-9]*\.[0-9]+)[^/]+/step([1-9])_.*\.log$')
for logFile in glob.glob('[1-9]*/step[0-9]*.log'):
m = logRE.match(logFile)
if not m: continue
wf = m.group(1)
step = int(m.group(2)) - 1
if step >= self.StepsPerJob: continue
if not logData.has_key(wf):
logData[wf] = {'events': [], 'failed': [], 'warning': []}
for s in range(0, self.StepsPerJob):
for k in logData[wf].keys():
logData[wf][k].append(-1)
warn = 0
err = 0
rd = 0
inFile = open(logFile)
for line in inFile:
if '%MSG-w' in line: warn += 1
if '%MSG-e' in line: err += 1
if 'Begin processing the ' in line: rd += 1
inFile.close()
logData[wf]['events'][step] = rd
logData[wf]['failed'][step] = err
logData[wf]['warning'][step] = warn
from pickle import Pickler
outFile = open('runTheMatrixMsgs.pkl', 'w')
pklFile = Pickler(outFile)
pklFile.dump(logData)
outFile.close()
return
def take_snapshot(self, object):
file = open(fu.next_snapshot_file(self.basedir), "wb")
logger.debug("Taking snapshot on: " + file.name)
pickler = Pickler(file, pickle.HIGHEST_PROTOCOL)
pickler.dump(object)
file.flush()
file.close()
def __setitem__(self, key, value):
if self.writeback:
self.cache[key] = value
f = BytesIO()
p = Pickler(f, self._protocol)
p.dump(value)
self.dict[key.encode(self.keyencoding)] = f.getvalue()
def encode(self):
buf = ringbuffer.RingBuffer(2**16)
pckler = Pickler(buf, protocol=-1, fix_imports=False)
pckler.dump(self.value)
if buf.freespace == 0:
raise ProtocolError("Data too large.")
return buf.read()
def picklify(objs):
from StringIO import StringIO
from pickle import Pickler
sio = StringIO()
pickler = Pickler(sio)
pickler.dump(objs)
return sio.getvalue()
def parseLog(self):
logData = {}
logRE = re.compile('^.*/([1-9][0-9]*(\.[0-9]+|))_[^/]+/step([1-9])_.*\.log$')
max_steps = 0
for logFile in glob.glob(self.basedir+'/[1-9]*/step[0-9]*.log'):
m = logRE.match(logFile)
if not m: continue
wf = m.group(1)
step = int(m.group(3))
if step>max_steps: max_steps=step
if not logData.has_key(wf):
logData[wf] = {'steps': {}, 'events' : [], 'failed' : [], 'warning' : []}
if not logData[wf]['steps'].has_key(step):
logData[wf]['steps'][step]=logFile
cache_read=0
log_processed=0
for wf in logData:
for k in logData[wf]:
if k == 'steps': continue
for s in range(0, max_steps):
logData[wf][k].append(-1)
index =0
for step in sorted(logData[wf]['steps']):
data = [0, 0, 0]
logFile = logData[wf]['steps'][step]
json_cache = os.path.dirname(logFile)+"/logcache_"+str(step)+".json"
if (not os.path.exists(json_cache)) or (os.path.getmtime(logFile)>os.path.getmtime(json_cache)):
try:
es_parse_log(logFile)
except Exception as e:
print "Sending log information to elasticsearch failed" , str(e)
inFile = open(logFile)
for line in inFile:
if '%MSG-w' in line: data[1]=data[1]+1
if '%MSG-e' in line: data[2]=data[2]+1
if 'Begin processing the ' in line: data[0]=data[0]+1
inFile.close()
jfile = open(json_cache,"w")
json.dump(data,jfile)
jfile.close()
log_processed+=1
else:
jfile = open(json_cache,"r")
data = json.load(jfile)
jfile.close()
cache_read+=1
logData[wf]['events'][index] = data[0]
logData[wf]['failed'][index] = data[2]
logData[wf]['warning'][index] = data[1]
index+=1
del logData[wf]['steps']
print "Log processed: ",log_processed
print "Caches read:",cache_read
from pickle import Pickler
outFile = open(os.path.join(self.outdir,'runTheMatrixMsgs.pkl'), 'w')
pklFile = Pickler(outFile)
pklFile.dump(logData)
outFile.close()
return
class Environment(object):
handle = None
loader = None
writer = None
data = None
def __init__(self):
self.handle = open("environment.pickle", 'w+')
self.loader = Unpickler(self.handle)
self.writer = Pickler(self.handle)
try:
self.data = self.loader.load()
except EOFError:
print "WARNING: Empty environment, creating environment file."
self.data = {}
self.write(self.data)
def write(self, data):
self.writer.dump(data)
def create(dataset, file_handler, keys=None):
if isinstance(file_handler, str):
with open(file_handler, "wb") as file_handler:
return PickledDataset.create(dataset, file_handler, keys=keys)
index = {}
pickler = Pickler(file_handler)
# allocate space for index offset
file_handler.seek(0)
pickler.dump(1 << 65) # 64 bits placeholder
if keys is None:
keys = dataset.list_keys()
for key in keys:
# pickle objects and build index
index[key] = file_handler.tell()
obj = dataset.query_item(key)
pickler.dump(obj)
pickler.memo.clear()
# put index and record offset
index_location = file_handler.tell()
pickler.dump(index)
# put context
context = getattr(dataset, "_context", None)
if context:
pickler.dump({**context})
# put index offset at the beginning of the file
file_handler.seek(0)
index_location ^= 1 << 65
pickler.dump(index_location)
def __setitem__(self, key, value):
if self.writeback:
self.cache[key] = value
f = BytesIO()
p = Pickler(f, self._protocol)
p.dump(value)
self.dict[key.encode(self.keyencoding)] = f.getvalue()
def save(basename, **data):
"""
Save data using cPickle
@type basename : str
@param basename : The name of the file to save the data in, .cpickle
will be appended to the file name if not provided
@param data : The actuall data to be saved.
"""
check_arg(basename, str, 0)
# If zero data size just return
if len(data) == 0:
return
filename = basename if ".cpickle" in basename else basename + ".cpickle"
f = open(filename, "w")
p = Pickler(f)
# Dump the dictionary kwarg
p.dump(data)
f.close()
def saveMX(self, mx):
ruta = os.path.split(sys.argv[0])
abs = os.path.join(ruta[0], "db/matrix.mx")
filemx = open(abs, 'w')
serializer = Pickler(filemx)
serializer.dump(mx)
print 'matrix salvada'
def _testTracingOrProfileState(self, do_pickle=False, **kw):
t = tasklet(self.Tasklet)
t(**kw)
t.run()
self.foo()
n = len(self.trace)
self.foo()
n2 = len(self.trace)
self.assertEqual(n, n2)
if do_pickle:
io = StringIO()
p = Pickler(io, -1)
p.persistent_id = self.persistent_id
p.dump(t)
t.remove()
t.bind(None)
p = Unpickler(StringIO(io.getvalue()))
p.persistent_load = self.persistent_load
t = p.load()
p = None
io = None
t.run()
self.foo()
n = len(self.trace)
self.foo()
n2 = len(self.trace)
self.assertEqual(n, n2)
def picklify(objs): from StringIO import StringIO from pickle import Pickler sio = StringIO() pickler = Pickler(sio) pickler.dump(objs) return sio.getvalue()
def parseLog(self):
logData = {}
logRE = re.compile('^([1-9][0-9]*\.[0-9]+)[^/]+/step([1-9])_.*\.log$')
for logFile in glob.glob(self.basedir+'/[1-9]*/step[0-9]*.log'):
m = logRE.match(logFile)
if not m: continue
wf = m.group(1)
step = int(m.group(2)) - 1
if step >= self.StepsPerJob: continue
if not logData.has_key(wf):
logData[wf] = {'events' : [], 'failed' : [], 'warning' : []}
for s in range(0, self.StepsPerJob):
for k in logData[wf].keys(): logData[wf][k].append(-1)
warn = 0
err = 0
rd = 0
inFile = open(logFile)
for line in inFile:
if '%MSG-w' in line: warn += 1
if '%MSG-e' in line: err += 1
if 'Begin processing the ' in line: rd += 1
inFile.close()
logData[wf]['events'][step] = rd
logData[wf]['failed'][step] = err
logData[wf]['warning'][step] = warn
from pickle import Pickler
outFile = open(os.path.join(self.basedir,'runTheMatrixMsgs.pkl'), 'w')
pklFile = Pickler(outFile)
pklFile.dump(logData)
outFile.close()
return
def write_to_disk(self):
try:
p = Pickler(open(self.file, 'wb'))
p.dump(self.reg)
except PicklingError:
sys.stderr.write(
"ERROR: could not save the registry to path {}".format(
self.file))
def save_positions (self):
""" Save the current character positions in a file """
val = self.file_val()
assert self.positions is not None
with open(self.__data_path + 'current/' + val, 'w+b') as file:
pickler = Pickler(file)
pickler.dump(self.positions)
def enregistrer(self):
"""
Fonction qui enregistre toutes les données de l'utilisateur dans
un fichier save.elh
"""
with open("save.elh", "wb") as fichier_sauvegarde:
pickler = Pickler(fichier_sauvegarde)
pickler.dump(self)
def pickle_outcomes(outcomes, fn):
fh = open(fn, 'w')
p = Pickler(fh)
p.dump(outcomes)
fh.close()
if KEEP_TIMING:
end_t = time()
time_map["pickling"] = end_t - start_t
start_t = time()
def pack_datas(self):
"""
interacts with 'datas' file.
it dumps 'self.pwmanagement._datas_list' in 'datas'
this function is called every time we need to save and store new or modified accounts
"""
with open('datas', 'wb') as datas_file:
pickler_datas = Pickler(datas_file)
pickler_datas.dump(self.pwmanagement._datas_list)
def pickle_outcomes(outcomes, fn):
fh = open(fn, 'w')
p = Pickler(fh)
p.dump(outcomes)
fh.close()
if KEEP_TIMING:
end_t = time()
time_map["pickling"] = end_t - start_t
start_t = time()
def deepcopy(obj):
"""Makes a deep copy of the object using the pickle mechanism.
"""
stream = StringIO()
p = Pickler(stream, 1)
p.dump(aq_base(obj))
stream.seek(0)
u = Unpickler(stream)
return u.load()
def testPickleUnpickle(self):
s = BytesIO()
p = Pickler(s)
p.dump(Allow)
s.seek(0)
u = Unpickler(s)
newAllow = u.load()
self.assertTrue(newAllow is Allow)
def dump(obj):
print "* Dumping..."
setrecursionlimit(10000)
copy_reg.pickle(types.MethodType, _pickle_method, _unpickle_method)
pickle_buffer = StringIO()
pickler = Pickler(pickle_buffer, -1)
pickler.persistent_id = persistent_id
pickler.dump(obj)
print "* Dumped!"
return pickle_buffer
def save(obj, filename):
print "* Saving..."
setrecursionlimit(10000)
copy_reg.pickle(types.MethodType, _pickle_method, _unpickle_method)
out_file = open(filename, "wb")
pickler = Pickler(out_file, -1)
pickler.persistent_id = persistent_id
pickler.dump(obj)
out_file.close()
print "* Saved!"
def save(self, path):
with open(self.fname(path), 'wb') as f:
p = Pickler(f, -1)
p.dump(self)
# remove old *.memory files
save_dir = Path(path).parent
for memfile in save_dir.glob("*.memory"):
basefile = memfile.with_suffix('.meta')
if not basefile.is_file():
memfile.unlink()
def _sauvegarder(self):
"""
Sauvegarde la map.
"""
try:
with open(adresse_fichier_sauvegarde, 'wb') as fichier_sauvegarde:
pick = Pickler(fichier_sauvegarde)
pick.dump(self._map)
except:
print("Erreur lors de l'enregistrement du fichier")
def save_server(self, abs_path=None, save_dir=None, ext=None):
"""Save a new Server."""
if os.path.exists(save_dir) and not os.path.isfile(save_dir):
os.chdir(save_dir)
else:
os.mkdir(save_dir)
os.chdir(save_dir)
with open(self.name + ext, 'wb') as new_file:
pickler = Pickler(new_file)
pickler.dump(self)
os.chdir(abs_path)
def save_state(self):
with open(self.options.results_file, 'w') as f:
dump(self.results, f, indent=4, separators=(',', ': '))
with open(self.statefile, 'wb') as f:
p = Pickler(f)
p.dump(self.target)
p.dump(self.session)
p.dump(self.cleanup_session)
p.dump(self.options)
p.dump(self.visited)
p.dump(self.pending)
def _sauvegarder(self):
"""
Sauvegarde la map.
"""
try:
with open(adresse_fichier_sauvegarde, 'wb') as fichier_sauvegarde:
pick = Pickler(fichier_sauvegarde)
pick.dump(self._map)
except:
print("Erreur lors de l'enregistrement du fichier")
def sync(self):
res = {}
with dbm.open(self.db, self.flag) as db:
for k, v in self.dict.items():
f = io.BytesIO()
p = Pickler(f, protocol=self._protocol)
p.dump(v)
db[k] = f.getvalue()
try:
db.sync()
except AttributeError:
pass
def pickle(obj, filename, protocol=0):
f = None
try:
f = open(filename, "wb")
p = Pickler(f, protocol)
p.dump(obj)
f.close()
f = None
# print "Pickled", filename
finally:
if f:
f.close()
def _dump_blocked(self):
dump = os.path.join(self.get_cache_dir(), "autoqueue_block_cache")
if not self._blocked_artists:
try:
os.remove(dump)
except OSError:
pass
return
with open(dump, 'w') as pickle_file:
pickler = Pickler(pickle_file, -1)
to_dump = (self._blocked_artists, self._blocked_artists_times)
pickler.dump(to_dump)
def pickle(obj, filename, protocol=0):
f = None
try:
f = open(filename, "wb")
p = Pickler(f, protocol)
p.dump(obj)
f.close()
f = None
# print "Pickled", filename
finally:
if f:
f.close()
def sync(self):
res = {}
with dbm.open(self.db, self.flag) as db:
for k, v in self.dict.items():
f = io.BytesIO()
p = Pickler(f, protocol=self._protocol)
p.dump(v)
db[k] = f.getvalue()
try:
db.sync()
except AttributeError:
pass
def _tuer_sauvegarde(self):
"""
Efface la sauvegarde.
Devient inefficace si le mode de sauvegarde change.
"""
try:
with open(adresse_fichier_sauvegarde, 'wb') as fichier_sauvegarde:
pick = Pickler(fichier_sauvegarde)
pick.dump(None)
except:
print("Erreur lors de l'enregistrement du fichier")
def _dump_blocked(self):
dump = os.path.join(
self.get_cache_dir(), "autoqueue_block_cache")
if not self._blocked_artists:
try:
os.remove(dump)
except OSError:
pass
return
with open(dump, 'w') as pickle_file:
pickler = Pickler(pickle_file, -1)
to_dump = (self._blocked_artists, self._blocked_artists_times)
pickler.dump(to_dump)
def createPickleChild(self, name, child):
if not os.path.isdir(self.path):
resource.Resource.putChild(self, name, child)
if type(child) == type(""):
fl = open(os.path.join(self.path, name), 'wb')
fl.write(child)
else:
if '.' not in name:
name = name + '.trp'
fl = open(os.path.join(self.path, name), 'wb')
from pickle import Pickler
pk = Pickler(fl)
pk.dump(child)
fl.close()
def createPickleChild(self, name, child):
if not os.path.isdir(self.path):
resource.Resource.putChild(self, name, child)
# xxx use a file-extension-to-save-function dictionary instead
if type(child) == type(""):
fl = open(os.path.join(self.path, name), 'wb')
fl.write(child)
else:
if '.' not in name:
name = name + '.trp'
fl = open(os.path.join(self.path, name), 'wb')
from pickle import Pickler
pk = Pickler(fl)
pk.dump(child)
fl.close()
def doCheck(self):
import glob
pkgDirList = glob.glob(self.startDir+'/src/[A-Z]*/*')
errMap = {}
for pkg in pkgDirList:
if not os.path.isdir(pkg): continue
pkg = re.sub('^'+self.startDir+'/src/', '', pkg)
missing = self.checkPkg(pkg)
if missing: errMap[pkg] = missing
from pickle import Pickler
summFile = open('libchk.pkl','w')
pklr = Pickler(summFile)
pklr.dump(errMap)
summFile.close()
def doPickling(option,pickleFile,obj=None):
varRootHash = {}
from pickle import Pickler
if os.path.isfile(pickleFile):
os.remove(pickleFile)
print "DEBUG:[Pickling] Removing existing pickleFile: ",pickleFile
f = open(pickleFile,"w")
if option == "CATALOG" :
print "DEBUG:[Pickling] Pickling CATALOG..."
fileSeekHash = obj
print "DEBUG:[doPickling] Before dumping to"+pickleFile+"->\n",stemmedQTList
p = Pickler(f)
p.dump(makecataloghash())
f.close()
else:
print "******ERROR******* Specify correct pickle option"
def dump_brain(self):
"""
Record the configuration of the actor and critic into a binary file.
:return: Nothing
"""
# Open the file for writing in binary mode
with open("Config/actor.cfg", "wb") as cfg_file:
# Instantiate a pickler
pickle = Pickler(cfg_file)
# Dump the configuration
pickle.dump(self._actor)
# Repeat the operation for the critic
with open("Config/critic.cfg", "wb") as cfg_file:
pickle = Pickler(cfg_file)
pickle.dump(self._critic)
def dump(self, obj):
self.inject_addons()
try:
return Pickler.dump(self, obj)
except RuntimeError as e:
if 'recursion' in e.args[0]:
msg = """Could not pickle object as excessively deep recursion required."""
raise pickle.PicklingError(msg)
def save(self, path):
grille = np.zeros((9, 9), dtype=int)
modif = np.zeros((9, 9), dtype=bool)
possibilites = []
for i in range(9):
possibilites.append([])
for j in range(9):
grille[i, j] = self.blocs[i, j].get_val()
modif[i, j] = self.blocs[i, j].is_modifiable()
possibilites[i].append(self.blocs[i, j].get_possibilites())
with open(path, "wb") as fich:
p = Pickler(fich)
p.dump(grille)
p.dump(modif)
p.dump(possibilites)
p.dump(self.chrono)
p.dump(self.level)
def enregistrer_score(adresse_fichier_score, joueur, nouveau_score):
"""Ouvre le fichier des scores, s'il existe, selon le joueur, le score est chargé et envoyé,
sinon on renvoie 0. Se charge également de charger entièrement le fichier des scores"""
global score
try:
with open(adresse_fichier_score, 'wb') as fichier_score:
#Définition du score
score[joueur] = score.get(joueur, 0) + nouveau_score
#Enregistrement du socre
pick = Pickler(fichier_score)
pick.dump(score)
except IOError:
print("Erreur lors de l'enregistrement du fichier")
return score[joueur]
def pickle(self):
outfile=open(self.saveFile,'wb')
pickle = Pickler(outfile)
kinds={"kind":[], "adjective":[], "entity":[]}
for knd in self._kinds:
if knd not in kinds:
kinds[knd]=[]
typ=knd
base=self._kinds[typ].__base__.__name__
while base not in kinds:
kinds[base]=[typ]
typ=base
base=self._kinds[typ].__base__.__name__
kinds[base].append(typ)
pickle.dump(kinds)
globals().update(self._kinds)
pickle.dump((self._entities,self._antecedents,self._names,self._adjectives))
outfile.close()
def stop(self):
try:
for t in self.threads:
try:
t.interrupt()
except:
pass
for t in self.threads:
try:
t.join()
except:
pass
self.inventoryLock.acquire()
with open(os.path.expanduser("~/.ingressbot.pkl"), "wb") as f:
pickler = Pickler(f)
pickler.dump(self.inventory)
finally:
self.inventoryLock.release()
self.logger.info("stopped")
def parseLog(self):
logData = {}
logRE = re.compile('^.*/([1-9][0-9]*\.[0-9]+)[^/]+/step([1-9])_.*\.log$')
max_steps = 0
for logFile in glob.glob(self.basedir+'/[1-9]*/step[0-9]*.log'):
m = logRE.match(logFile)
if not m: continue
wf = m.group(1)
step = int(m.group(2))
if step>max_steps: max_steps=step
if not logData.has_key(wf):
logData[wf] = {'steps': {}, 'events' : [], 'failed' : [], 'warning' : []}
if not logData[wf]['steps'].has_key(step):
logData[wf]['steps'][step]=logFile
for wf in logData:
for k in logData[wf]:
if k == 'steps': continue
for s in range(0, max_steps):
logData[wf][k].append(-1)
index =0
for step in sorted(logData[wf]['steps']):
warn = 0
err = 0
rd = 0
inFile = open(logData[wf]['steps'][step])
for line in inFile:
if '%MSG-w' in line: warn += 1
if '%MSG-e' in line: err += 1
if 'Begin processing the ' in line: rd += 1
inFile.close()
logData[wf]['events'][index] = rd
logData[wf]['failed'][index] = err
logData[wf]['warning'][index] = warn
index+=1
del logData[wf]['steps']
from pickle import Pickler
outFile = open(os.path.join(self.basedir,'runTheMatrixMsgs.pkl'), 'w')
pklFile = Pickler(outFile)
pklFile.dump(logData)
outFile.close()
return
def createPickleChild(self, name, child):
warnings.warn(
"File.createPickleChild is deprecated since Twisted 9.0. "
"Resource persistence is beyond the scope of Twisted Web.",
DeprecationWarning, stacklevel=2)
if not os.path.isdir(self.path):
resource.Resource.putChild(self, name, child)
# xxx use a file-extension-to-save-function dictionary instead
if type(child) == type(""):
fl = open(os.path.join(self.path, name), 'wb')
fl.write(child)
else:
if '.' not in name:
name = name + '.trp'
fl = open(os.path.join(self.path, name), 'wb')
from pickle import Pickler
pk = Pickler(fl)
pk.dump(child)
fl.close()
def tokens_referenced(self):
"""
Return a list of all the tokens that are referenced (i.e. contained in)
this message. Tokens that haven't been assigned an id yet are searched
recursively for tokens. So this method may return fewer results after
the message is sent. This information is used by the game engine to
catch mistakes like forgetting to add a token to the world or keeping a
stale reference to a token after its been removed.
"""
tokens = set()
# Use the pickle machinery to find all the tokens contained at any
# level of this message. When an object is being pickled, the Pickler
# calls its persistent_id() method for each object it encounters. We
# hijack this method to add every Token we encounter to a list.
# This definitely feels like a hacky abuse of the pickle machinery, but
# that notwithstanding this should be quite robust and quite fast.
def persistent_id(obj):
from .tokens import Token
if isinstance(obj, Token):
tokens.add(obj)
# Recursively descend into tokens that haven't been assigned an
# id yet, but not into tokens that have.
return obj.id
from pickle import Pickler
from io import BytesIO
# Use BytesIO to basically ignore the serialized data stream, since we
# only care about visiting all the objects that would be pickled.
pickler = Pickler(BytesIO())
pickler.persistent_id = persistent_id
pickler.dump(self)
return tokens
def process(infile='t10k'):
f = open(infile,'r')
out = open('report.pickle','w')
agents = loadagents()
pickler = Pickler(out)
for line in f.readlines():
n, h = line.strip().split(',',1)
for ua in agents:
fetcher = HeaderFetcher('http://'+h,{'User-Agent':ua})
print n, h, ua
fetcher.fetch()
pickler.dump((n,h,ua,fetcher.report))
for url in fetcher.report:
print 'GET',url
for header in fetcher.report[url]:
print header
print '========================'
print 'done'
out.flush()
out.close()
f.close()
def pack(self, message):
from pickle import Pickler
from io import BytesIO
from .tokens import Token
from .messages import Message, require_message
buffer = BytesIO()
delegate = Pickler(buffer)
def persistent_id(token):
if isinstance(token, Token):
assert isinstance(message, Message), msg("""\
Both Message and ServerResponse objects can be
serialized, but only Messages can contain tokens.""")
assert token.id, msg("""\
Every token should have an id by now. Tokens that are
in the world should always have an id, and tokens that
are being added to the world should've been assigned an
id by Actor.send_message().""")
if token in self.world:
assert token not in message.tokens_to_add(), msg("""\
Actor.send_message() should've refused to send a
message that would add a token that's already in
the world.""")
return token.id
else:
assert token in message.tokens_to_add(), msg("""\
Actor.send_message() should've refused to send a
message referencing tokens that aren't in the world
and aren't being added to the world.""")
return None
delegate.persistent_id = persistent_id
delegate.dump(message)
return buffer.getvalue()
