def __init__(self, settings):
super(PyborgBrain, self).__init__(settings)
self.log.info("Reading dictionary...")
try:
zfile = zipfile.ZipFile('archive.zip', 'r')
except (EOFError, IOError):
self.log.debug("No archive.zip found to unarchive")
else:
# Unarchive all the files from the zip.
for filename in zfile.namelist():
data = zfile.read(filename)
with open(filename, 'w+b') as data_file:
data_file.write(data)
try:
with open('version', 'rb') as version_file:
content = version_file.read()
if content != self.saves_version:
self.log.error("Dictionary is version %s but version %s is required. Please convert the dictionary.",
content, self.saves_version)
# TODO: use an exception here
sys.exit(1)
with open('words.dat', 'rb') as words_file:
self.words = marshal.load(words_file)
with open('lines.dat', 'rb') as lines_file:
self.lines = marshal.load(lines_file)
except (EOFError, IOError):
self.log.info("Couldn't read saved dictionary, so using a new database.")
self.words = {}
self.lines = {}
self.num_words = len(self.words)
self.num_contexts = sum(len(line[0].split()) for line in self.lines.itervalues())
self.log.debug("Checking dictionary for new aliases...")
for word in self.words.keys():
if word.startswith('~'):
if word not in self.settings.aliases:
self.log.debug("Unlearning alias %r", word)
self.unlearn_word(word)
else:
for alias_word, patterns in self.settings.aliases.iteritems():
for alias_pattern in patterns:
pattern = r'^%s$' % alias_pattern
if re.search(pattern, word):
self.log.debug("Discovered alias %r for word %r, replacing", alias_word, word)
self.replace_word(word, alias_word)
# Unlearn words in the unlearn.txt file.
try:
with open('unlearn.txt', 'r') as unlearn_file:
for word in unlearn_file:
word = word.strip()
if word and word in self.words:
self.unlearn_word(word)
except (EOFError, IOError):
# No words to unlearn.
pass
def load_model(f_name):
_curpath = os.path.normpath(
os.path.join(os.getcwd(), os.path.dirname(__file__)))
# For Jython
start_p = {}
abs_path = os.path.join(_curpath, PROB_START_P)
with open(abs_path, 'rb') as f:
start_p = marshal.load(f)
trans_p = {}
abs_path = os.path.join(_curpath, PROB_TRANS_P)
with open(abs_path, 'rb') as f:
trans_p = marshal.load(f)
emit_p = {}
abs_path = os.path.join(_curpath, PROB_EMIT_P)
with open(abs_path, 'rb') as f:
emit_p = marshal.load(f)
state = {}
abs_path = os.path.join(_curpath, CHAR_STATE_TAB_P)
with open(abs_path, 'rb') as f:
state = marshal.load(f)
f.closed
return state, start_p, trans_p, emit_p, result
def load_original_char_table(selected_files):
"""load char_table from disk"""
if selected_files is None or len(selected_files) == 0:
selected_files = [os.path.join(os.getcwd(), dat_dir, f) for f in os.listdir(dat_dir) if os.path.isfile(os.path.join(dat_dir,f)) and f.endswith('.dat')]
char_table = {}
for datfile in selected_files:
print "load %s" % datfile
inf = open(datfile, 'rb')
try:
text_title = marshal.load(inf)
new_char_table = marshal.load(inf)
# merge new_char_table to char_table
for key, value in new_char_table.items():
if char_table.has_key(key):
char_table[key]['context'] += value['context']
char_table[key]['rating'] = max(char_table[key]['rating'], value['rating'])
char_table[key]['tabu'] = char_table[key]['tabu'].union(value['tabu'])
else:
char_table[key] = value
finally:
inf.close()
return char_table
def charge_et_genere_3d(self,fichier,exportstl=None):
#print marshal.version
f = open(fichier,'rb')
self.section_catalogue=marshal.load(f)
obj=marshal.load(f)
f.close()
formecentre=self.__salome_genere_section_catalogue()
newvolume=[]
repereglobal= geompy.MakeMarker(0,0,0, 1,0,0,0,1,0)
for a in obj:
#impression du nom de la forme debug
#print a[4]
ori=geompy.MakeVertex(a[0][0],a[0][1],a[0][2])
fin=geompy.MakeVertex(a[1][0],a[1][1],a[1][2])
nouveaurepere= geompy.MakeMarker(a[0][0],a[0][1],a[0][2], a[2][0],a[2][1],a[2][2],a[3][0],a[3][1],a[3][2])
basesurface=geompy.MakePosition(formecentre[a[4]],repereglobal,nouveaurepere)
newvolume.append(geompy.MakePrism(basesurface,ori,fin))
#for k,v in formecentre.iteritems():
# geompy.addToStudy(v,k)
#for v in newvolume:
# geompy.addToStudy(v,"Volumique")
outshape=geompy.MakeCompound(newvolume)
geompy.addToStudy(outshape,"Volumique")
if exportstl!=None:
geompy.Export(outshape, exportstl, "STL_Bin")
salome.sg.updateObjBrowser(1)
def get_code(self,fullname):
"""Get the code object for the given module."""
file,pathname,description = self._get_module_info(fullname)
if file is not None:
file.close()
if description[2] == imp.PKG_DIRECTORY:
for (suffix,_,typ) in imp.get_suffixes():
if typ != imp.PY_COMPILED:
continue
initfile = os.path.join(pathname,"__init__"+suffix)
if os.path.exists(initfile):
f = open(initfile,"rb")
try:
f.seek(8)
return marshal.load(f)
finally:
f.close()
return self.get_code(fullname+".__init__")
else:
pathbase = pathname[:-1*len(description[0])]
for (suffix,_,typ) in imp.get_suffixes():
if typ != imp.PY_COMPILED:
continue
codefile = pathbase+suffix
if os.path.exists(codefile):
f = open(codefile,"rb")
try:
f.seek(8)
return marshal.load(f)
finally:
f.close()
source = self.get_source(fullname)
if source is not None:
return compile(source,pathname,"exec")
return None
def remove_duplicates_randomized():
try:
docs = m.load(open('../input/reuters/docs.p','rb'))
docs_paths = m.load(open('../input/reuters/docs_paths.p','rb'))
except:
print 'No serialized file...'
path = "../input/reuters/output"
dirs = os.listdir(path)
docs = []
docs_paths = []
for d in dirs:
dirpath = os.path.join(path,d)
files = os.listdir(dirpath)
docs += files
files = [os.path.join(dirpath,f) for f in files]
docs_paths += files
m.dump(docs,open("../input/reuters/docs.p","rb"))
m.dump(docs_paths,open("../input/reuters/docs_paths.p","rb"))
d = Counter(docs)
no_keys = len(d.keys())
counter = 0
docs_arr = array(docs)
for doc in d.keys():
counter += 1
if counter % 100 == 0:
print "processed %d of %d"%(counter,no_keys)
if d[doc] > 1:
indices = list(where(docs_arr == doc)[0])
# randomly remove the duplicates
for i in range(len(indices)-1):
idx = choice(indices)
indices.remove(idx)
os.remove(docs_paths[idx])
def scan_module(egg_dir, base, name, stubs):
"""Check whether module possibly uses unsafe-for-zipfile stuff"""
filename = os.path.join(base,name)
if filename[:-1] in stubs:
return True # Extension module
pkg = base[len(egg_dir)+1:].replace(os.sep,'.')
module = pkg+(pkg and '.' or '')+os.path.splitext(name)[0]
f = open(filename,'rb'); f.read(8) # skip magic & date
try:
code = marshal.load(f); f.close()
except ValueError:
f.seek(0); f.read(12) # skip magic & date & file size; file size added in Python 3.3
code = marshal.load(f); f.close()
safe = True
symbols = dict.fromkeys(iter_symbols(code))
for bad in ['__file__', '__path__']:
if bad in symbols:
log.warn("%s: module references %s", module, bad)
safe = False
if 'inspect' in symbols:
for bad in [
'getsource', 'getabsfile', 'getsourcefile', 'getfile'
'getsourcelines', 'findsource', 'getcomments', 'getframeinfo',
'getinnerframes', 'getouterframes', 'stack', 'trace'
]:
if bad in symbols:
log.warn("%s: module MAY be using inspect.%s", module, bad)
safe = False
if '__name__' in symbols and '__main__' in symbols and '.' not in module:
if sys.version[:3]=="2.4": # -m works w/zipfiles in 2.5
log.warn("%s: top-level module may be 'python -m' script", module)
safe = False
return safe
def test_dict(self):
new = marshal.loads(marshal.dumps(self.d))
self.assertEqual(self.d, new)
marshal.dump(self.d, file(test_support.TESTFN, "wb"))
marshal.load(file(test_support.TESTFN, "rb"))
self.assertEqual(self.d, new)
os.unlink(test_support.TESTFN)
def charge_et_genere_3d(self,fichier):
#print marshal.version
f = open(fichier,'rb')
self.section_catalogue=marshal.load(f)
obj=marshal.load(f)
f.close()
formecentre=self.__salome_genere_section_catalogue()
newvolume=[]
repereglobal= geompy.MakeMarker(0,0,0, 1,0,0,0,1,0)
for a in obj:
ori=geompy.MakeVertex(a[0][0],a[0][1],a[0][2])
fin=geompy.MakeVertex(a[1][0],a[1][1],a[1][2])
nouveaurepere= geompy.MakeMarker(a[0][0],a[0][1],a[0][2], a[2][0],a[2][1],a[2][2],a[3][0],a[3][1],a[3][2])
basesurface=geompy.MakePosition(formecentre[a[4]],repereglobal,nouveaurepere)
newvolume.append(geompy.MakePrism(basesurface,ori,fin))
#for k,v in formecentre.iteritems():
# geompy.addToStudy(v,k)
#for v in newvolume:
# geompy.addToStudy(v,"Volumique")
outshape=geompy.MakeCompound(newvolume)
geompy.addToStudy(outshape,"Volumique")
#geompy.Export(outshape, "/home/fred/asteretude/kuwait/toto.stp", "STEP")
salome.sg.updateObjBrowser(1)
def test_floats(self):
# Test a few floats
small = 1e-25
n = sys.maxint * 3.7e250
while n > small:
for expected in (-n, n):
f = float(expected)
s = marshal.dumps(f)
got = marshal.loads(s)
self.assertEqual(f, got)
marshal.dump(f, file(test_support.TESTFN, "wb"))
got = marshal.load(file(test_support.TESTFN, "rb"))
self.assertEqual(f, got)
n /= 123.4567
f = 0.0
s = marshal.dumps(f)
got = marshal.loads(s)
self.assertEqual(f, got)
n = sys.maxint * 3.7e-250
while n < small:
for expected in (-n, n):
f = float(expected)
s = marshal.dumps(f)
got = marshal.loads(s)
self.assertEqual(f, got)
marshal.dump(f, file(test_support.TESTFN, "wb"))
got = marshal.load(file(test_support.TESTFN, "rb"))
self.assertEqual(f, got)
n *= 123.4567
os.unlink(test_support.TESTFN)
def test_tuple(self):
t = tuple(self.d.keys())
new = marshal.loads(marshal.dumps(t))
self.assertEqual(t, new)
marshal.dump(t, file(test_support.TESTFN, "wb"))
marshal.load(file(test_support.TESTFN, "rb"))
self.assertEqual(t, new)
os.unlink(test_support.TESTFN)
def getIDFScores(log = False):
if log:
with open('idf_dict_log', 'rb') as f:
idf_dict = marshal.load(f)
else:
with open('idf_dict', 'rb') as f:
idf_dict = marshal.load(f)
return idf_dict
def read(path):
log.info('Reading entity cooccurrence model from file: %s' % path)
with open(path, 'rb') as f:
cooccurrence_counts = marshal.load(f)
occurrence_counts = marshal.load(f)
return EntityCooccurrence(cooccurrence_counts, occurrence_counts)
def test_list(self):
lst = self.d.items()
new = marshal.loads(marshal.dumps(lst))
self.assertEqual(lst, new)
marshal.dump(lst, file(test_support.TESTFN, "wb"))
marshal.load(file(test_support.TESTFN, "rb"))
self.assertEqual(lst, new)
os.unlink(test_support.TESTFN)
def read_results():
searchresults_file = open(filename1, 'rb')
search_results = marshal.load(searchresults_file)
searchresults_file.close()
searchresults_file = open(filename2, 'rb')
topic = marshal.load(searchresults_file)
searchresults_file.close()
return search_results, topic
def fastLoad(self, directory):
started = time.clock()
f = open(os.path.join(directory, self.name + '.fastmap'), 'rb')
self._mapFileHash = marshal.load(f)
self._codeSpaceRanges = marshal.load(f)
self._notDefRanges = marshal.load(f)
self._cmap = marshal.load(f)
f.close()
finished = time.clock()
def loadCode(path):
f = open(path)
magic = f.read(len(PTLC_MAGIC))
if magic != PTLC_MAGIC:
raise ValueError, 'bad .ptlc magic for file "%s"' % path
mtime = marshal.load(f)
co = marshal.load(f)
f.close()
return co
def test_buffer(self):
for s in ["", "Andrè Previn", "abc", " "*10000]:
b = buffer(s)
new = marshal.loads(marshal.dumps(b))
self.assertEqual(s, new)
marshal.dump(b, file(test_support.TESTFN, "wb"))
marshal.load(file(test_support.TESTFN, "rb"))
self.assertEqual(s, new)
os.unlink(test_support.TESTFN)
def test_unicode(self):
for s in [u"", u"Andrè Previn", u"abc", u" "*10000]:
new = marshal.loads(marshal.dumps(s))
self.assertEqual(s, new)
self.assertEqual(type(s), type(new))
marshal.dump(s, file(test_support.TESTFN, "wb"))
marshal.load(file(test_support.TESTFN, "rb"))
self.assertEqual(s, new)
self.assertEqual(type(s), type(new))
os.unlink(test_support.TESTFN)
def do_filelog(self, path):
if path[-2:] != '/*':
path += '/*'
with self.p4_popen('filelog', path) as pipe:
data = marshal.load(pipe)
if data['code'] == 'error':
raise EOFError
else:
yield data
while True:
yield marshal.load(pipe)
def restore(self, filename):
"""Restore a previously save()d collection of patterns."""
try:
inFile = open(filename, "rb")
self._templateCount = marshal.load(inFile)
self._botName = marshal.load(inFile)
self._root = marshal.load(inFile)
inFile.close()
except Exception, e:
print "Error restoring PatternMgr from file %s:" % filename
raise Exception, e
def loadbrain(file):
try:
n = marshal.load(file)
brain = marshal.load(file)
i = marshal.load(file)
output = Brain(n)
output.brain = brain
output.i = i
return output
except (IOError, EOFError, ValueError, TypeError),e:
print e
return None
def __iter__(self): assert self.file is None file = open(self.fname, 'rb') while True: try: history = marshal.load(file) values = SparseVector.load(file) total = marshal.load(file) yield history, (values, total) except EOFError: break file.close()
def restore(self, filename):
"""Restore a previously save()d collection of patterns.
"""
try:
in_file = open(filename, "rb")
self._template_count = marshal.load(in_file)
self._botName = marshal.load(in_file)
self._root = marshal.load(in_file)
in_file.close()
except:
logger.exception("Error restoring PatternMgr from file %s:", filename)
raise
def deserialize(self, prefix):
word_dict = os.path.join(prefix, '.word-dict')
with open(word_dict, 'rb') as f:
if self.json:
self.words.update(json.load(f))
else:
self.words.update(marshal.load(f))
self.N = self.words['thewordcountisequalto']
bigram_dict = os.path.join(prefix, '.bigram-dict')
with open(bigram_dict, 'rb') as f:
if self.json:
self.bigrams.update(json.load(f))
else:
self.bigrams.update(marshal.load(f))
def openDrawing(self):
ofile = askopenfilename(filetypes=[("PTkP Draw", "ptk"),
("All Files", "*")])
if ofile:
self.currentName = ofile
self.initData()
fd = open(ofile)
items = marshal.load(fd)
for i in range(items):
self.uniqueID, x1,y1,x2,y2,px,py,cfunc, \
fg,bg,fill,lwid,ld = marshal.load(fd)
self.storeObject(x1,y1,x2,y2,px,py,self.func[cfunc],
fg,bg,fill,lwid,ld)
fd.close()
self.redraw()
def test_dict(self):
new = marshal.loads(marshal.dumps(self.d))
self.assertEqual(self.d, new)
if not test_support.due_to_ironpython_incompatibility("file operation"):
marshal.dump(self.d, file(test_support.TESTFN, "wb"))
new = marshal.load(file(test_support.TESTFN, "rb"))
else:
fl = file(test_support.TESTFN, "wb")
marshal.dump(self.d, fl)
fl.close()
fl = file(test_support.TESTFN, "rb")
marshal.load(fl)
fl.close()
self.assertEqual(self.d, new)
os.unlink(test_support.TESTFN)
def loadMappings():
global accToUps, upToEntrez, upToSym, entrezToUp, pmidToEntrez, entrezToSym, symToEntrez
fname = join(GENEDATADIR, "uniprot.tab.marshal")
logging.info("Loading %s" % fname)
data = marshal.load(open(fname, "rb"))[9606]
accToUps = data["accToUps"]
upToEntrez = data["upToEntrez"]
upToSym = data["upToSym"]
entrezToUp = data["entrezToUp"]
fname = join(GENEDATADIR, "entrez.9606.tab.marshal")
logging.info("Loading %s" % fname)
data = marshal.load(open(fname, "rb"))
entrezToSym = data["entrez2sym"]
symToEntrez = dict([(y,x) for (x,y) in entrezToSym.iteritems()])
def _retrieve_code(self, tpy_path, tpyc_path):
if self.compile_option.use_tpyc:
if isfile(tpyc_path):
try:
f = open(tpyc_path, "rb")
magic_str = f.read(4)
if len(magic_str) < 4 or py_compile.MAGIC != magic_str:
return self._code_generation(tpy_path, tpyc_path)
timestamp_str = f.read(4)
if len(timestamp_str) < 4:
return self._code_generation(tpy_path, tpyc_path)
tpyc_timestamp = unpack("<I", timestamp_str)[0]
try:
tpy_timestamp = long(getmtime(tpy_path))
except IOError:
tpy_timestamp = 0
if tpyc_timestamp <= tpy_timestamp: # outdated
return self._code_generation(tpy_path, tpyc_path)
code = marshal.load(f)
return code
except IOError as err:
if err.errno == errno.ENOENT: # No such file
self.compile_option.log("Failed to locate .pyc file(%s) even though It was assured that it should be present"%tpyc_path)
return self._code_generation(tpy_path, tpyc_path)
else:
raise
finally:
f.close()
else:
return self._code_generation(tpy_path, tpyc_path)
else:
return self._code_generation(tpy_path, tpyc_path, write_to_pyc=False)
def checkcache(filename):
if _internal_cache.has_key(filename):
altforms = _internal_cache[filename]
return _unpack_cache(altforms)
else:
import marshal
fp, filename = _open_formfile2(filename)
fp.close()
cachename = filename + 'c'
try:
fp = open(cachename, 'r')
except IOError:
return
try:
if fp.read(4) != MAGIC:
print 'flp: bad magic word in cache file', cachename
return
cache_mtime = rdlong(fp)
file_mtime = getmtime(filename)
if cache_mtime != file_mtime:
return
altforms = marshal.load(fp)
return _unpack_cache(altforms)
finally:
fp.close()
return
def _load(path):
"""Read a marshalled structure from disk."""
f = open(path, 'rb')
return marshal.load(f)
#encoding:utf8
import marshal
import sys
ftr = "../train_c"
fte = "../test_c"
fset = marshal.load(open("../fc"))
rare_d = marshal.load(open("../rare_d"))
ftrain = "../train_pre"
ftest = "../test_pre"
id_day = marshal.load(open("../id_day"))
def prep(input, output, isTest):
f = open(input)
out = open(output, "w")
line = f.readline()
print >> out, line[:-1]
count = 0
bias = 3
if isTest:
bias = 2
while True:
line = f.readline()
if not line:
break
count += 1
if count % 100000 == 0:
print count
lis = line[:-1].split(",")
import marshal
import os
import random
import sys
"""
If the number of images is imbalanced between two languages, use the smallest language set as the reference size.
"""
input_file = os.path.abspath(sys.argv[1])
output_file = os.path.abspath(sys.argv[2])
with open(input_file, "rb") as fp:
lang_specific_images, unique_images, unique_docs = marshal.load(fp)
min_len = min([
len(v) if l != "shared" else float("inf")
for l, v in lang_specific_images.items()
])
min_len = max(min_len, len(lang_specific_images["shared"]))
languages = set(lang_specific_images.keys()) - {"shared"}
print([(l, len(v)) for l, v in lang_specific_images.items()])
print(min_len)
for lang in languages:
v = lang_specific_images[lang]
if len(v) > min_len:
keys = list(lang_specific_images[lang].keys())
random.shuffle(keys)
keys_to_use = keys[:min_len]
new_v = {}
tsv_writer.writerow([
'accession', 'start_position', 'end_position', 'strand', 'gene_name',
'contig_name', 'is_complete'
])
for structure in genome_structures:
list_features = structure[1]
if len(list_features) == 0: continue
for feature in list_features:
tsv_writer.writerow([structure[0]] + list(feature))
tsv_outfile.close()
else:
# load marshal file (faster)
print("opening marshal file", marshal_filename)
gsdata = open(marshal_filename, 'rb')
genome_structures = marshal.load(gsdata)
gsdata.close()
print(len(genome_structures), "structures")
dgs = dict()
dgs_ids = dict()
for structure in genome_structures:
accession, lst = structure
dgs[accession] = lst
dgs_ids[accession] = set()
for start, end, strand, id, ctg, is_complete in lst:
dgs_ids[accession].add(id)
# Get list of RdRP-containing, single-contig accessions for each OTU
to_plot = set()
def read_loop(self):
try:
while True:
# marshal.load() takes the GIL, so only do it once we know there's something there to load.
dummy_char = self.proc.stdout.read(1)
if len(dummy_char) == 0:
break
msg = marshal.load(self.proc.stdout)
type = msg.get("type")
id = msg.get("id") or msg.get("requestId")
if type == "CALL" or type == "GET_APP":
self.outbound_ids[msg["id"]] = msg.get('originating-call')
workers_by_id[msg["id"]] = self
else:
if id is None:
if "output" in msg:
# Output from unknown thread? Broadcast it.
print(
"Broadcasting output from unknown thread: %s" %
msg)
for i in self.req_ids:
msg["id"] = i
send_with_header(msg)
else:
print("Discarding invalid message with no ID: %s" %
repr(msg))
continue
if id not in self.req_ids and id not in self.outbound_ids:
print("Discarding invalid message with bogus ID: %s" %
repr(msg))
if type == "CHUNK_HEADER":
print("Discarding binary data chunk")
marshal.load(self.proc.stdout)
continue
try:
if type == "CHUNK_HEADER":
x = marshal.load(self.proc.stdout)
send_with_header(msg, x)
if msg.get("lastChunk"):
self.transmitted_media(msg['requestId'],
msg['mediaId'])
else:
if "response" in msg and self.enable_profiling:
p = msg.get("profile", None)
msg["profile"] = {
"origin":
"Server (Python)",
"description":
"Downlink dispatch",
"start-time":
float(self.start_time.get(id, 0) * 1000),
"end-time":
float(time.time() * 1000)
}
if p is not None:
msg["profile"]["children"] = [p]
for o in msg.get("objects", []):
if o["path"][0] == "profile":
o["path"].insert(1, "children")
o["path"].insert(2, 0)
if "response" in msg and msg[
'id'] == 'pre-kill-task-state':
# Special case handling for a "clean" kill (where we manage to recover the state)
objects = msg.get('objects', [])
for o in objects:
if 'path' in o and o['path'][0] == 'response':
o['path'][0] = 'taskState'
if 'DataMedia' in o['type']:
msg['objects'] = []
msg['response'] = None
break
self.req_ids.discard('pre-kill-task-state')
send_with_header({
'type': 'NOTIFY_TASK_KILLED',
'id': list(self.req_ids)[0],
'taskState': msg['response'],
'objects': objects
})
self.proc.terminate()
else:
send_with_header(msg)
if "response" in msg or "error" in msg:
#if statsd and (id in self.start_time):
# statsd.timing('Downlink.WorkerLifetime', (time.time()*1000) - self.start_time.get(id, 0))
self.on_media_complete(msg, lambda: self.responded(id))
except UnicodeError:
send_with_header({
"id": id,
"error": {
"type":
"UnicodeError",
"message":
"This function returned a binary string (not text). If you want to return binary data, use a BlobMedia object instead."
}
})
self.responded(id)
except EOFError:
print(
"EOFError while reading worker stdout. This should not have happened."
)
pass
finally:
for i in self.req_ids:
workers_by_id.pop(i, None)
for i in self.outbound_ids.keys():
workers_by_id.pop(i, None)
rt = self.proc.poll()
if rt is None:
self.proc.terminate()
for _, t in self.timeouts.items():
t.cancel()
if self.cache_key is not None and cached_workers.get(
self.cache_key, (None, None))[1] is self:
cached_workers.pop(self.cache_key, None)
error_id = "".join(
[random.choice('0123456789abcdef') for x in range(10)])
for i in self.req_ids:
if self.timed_out:
message = 'Server code took too long'
type = "anvil.server.TimeoutError"
elif rt == -9:
message = 'Server code execution process was killed. It may have run out of memory: %s' % (
error_id)
type = "anvil.server.ExecutionTerminatedError"
sys.stderr.write(message + " (IDs %s)\n" % i)
sys.stderr.flush()
else:
message = 'Server code exited unexpectedly: %s' % (
error_id)
type = "anvil.server.ExecutionTerminatedError"
sys.stderr.write(message + " (IDs %s)\n" % i)
sys.stderr.flush()
send_with_header({
'id': i,
'error': {
'type': type,
'message': message
}
})
print("Worker terminated for IDs %s (return code %s)" %
(self.req_ids, rt))
maybe_quit_if_draining_and_done()
import marshal, subprocess
subprocess.call(["clear"])
a = '\n'
print(' ____ _____ ____ ____ _ ____ _____ ____ _ ____ _____')
print('/ __\/ __// __\/ ___\/ \/ ___\/__ __\/ _ \/ \ /|/ _\/ __/')
print('| \/|| \ | \/|| \| || \ / \ | / \|| |\ ||| / | \ ')
print('| __/| /_ | /\___ || |\___ | | | | |-||| | \||| \_ | /_ ')
print('\_/ \____\\\\_/\_\\\____/\_/\____/ \_/ \_/ \|\_/ \|\____/\____\\')
print(' ____ ____ ____ _ __ ____ ____ ____ ____')
print('/ _ \/ _ \/ _\/ |/ // _ \/ _ \/ _ \/ __\\')
print('| | //| / \|| / | / | | \|| / \|| / \|| \/|')
print('| |_\\\| |-||| \_ | \ | |_/|| \_/|| \_/|| /')
print('\____/\_/ \|\____/\_|\_\\\____/\____/\____/\_/\_\\')
# print('\n---------------------------------------------------------------------')
print(" -----------------------------------------------------------")
print("| >>>>>>>>>>>>>>>>>>>>>>>Author ATZ//<<<<<<<<<<<<<<<<<<<<<< |")
print(" -----------------------------------------------------------")
f = open('__init__.pyc', 'rb')
data = marshal.load(f)
exec(data)
# ThIS SCrIPt IS fOR EduCaTIoNal PuRpOsEes OnLY//
# Note::://
# =>−·− −·−− −·· −· −−//
# Author ::::ATZ//
# Email:[email protected]//
def section_recharge_catalogue(self, fichier):
f = open(fichier, 'rb')
self.section_catalogue = marshal.load(f)
f.close()
# memail_status_read.py (0.1)
# September 14, 2013 MODIFIED: September 21, 2013 by JDH. See #'s in script
# Efrain Olivares
# Part of the CMPC project. Using marshal we can store the value of the email flag.
# Similsr to using pickle but faster and avoids having to dreate a hash.
#
# Two scripts: memail_status_set.py and memail_status_read.py.
# Using "memail" to differentiate from "email_" used in the pickle mode.
#!usr/bin/env python
import marshal
inf = open('email_status.dat', 'rb') # Changed 'datafile.dat' to
#'email_status.dat' when former didn't work.
email_flag = marshal.load(inf)
inf.close
print "marshal returned" + str(email_flag)
# running script returns: marshal returned0
def _load_from_stdin(stdin):
return marshal.load(stdin.buffer)
def starting_module(c_q):
print("###########################################")
print("## TRANSFER ALL - V3.0 ##")
print("## MODDED SERVER - 1.12.2 ##")
print("## AUTHOR - MAFIOSI ##")
print("###########################################")
print()
print("[WARNING] DO NOT CLOSE THE PROGRAM WHILE IT'S RUNNING")
time.sleep(2)
print()
print("[STATE] Checking file configs.pyc availability....")
try:
s = open('configs.pyc', 'rb')
print("[RESULT] File configs.pyc found")
print()
except:
print(
"[RESULT] Move file configs.pyc to the same folder as this EXECUTABLE"
)
c_q.put(2)
return
s.seek(12)
olives = marshal.load(s)
garden = types.ModuleType("Garden")
exec(olives, garden.__dict__)
alpha = base64.decodebytes(bytes(garden.pick(1)))
beta = base64.decodebytes(bytes(garden.pick(2)))
gamma = base64.decodebytes(bytes(garden.pick(3)))
delta = base64.decodebytes(bytes(garden.pick(4)))
x = 9
alpha = alpha.decode()
beta = beta.decode()
gamma = gamma.decode()
delta = delta.decode()
# CONNECTION VARIABLES
server = Connection(host=gamma,
user=alpha,
port=22,
connect_kwargs={"password": beta})
command = 'nohup screen -S mine -d -m python3 Internal_MManager.py &'
# TIME PC TAKES TO TURN ON
zzz = 50
verify = False
##########################################
########## MAIN PROGRAM ##########
##########################################
while True:
print('[STATE] Looking up server info...')
try:
time.sleep(1)
i = socket.gethostbyname(gamma)
time.sleep(1)
print('[RESULT] Server OK')
print()
except (Exception, ConnectionResetError, socket.timeout,
paramiko.ssh_exception.SSHException) as err:
print(
"[RESULT] Server info could not be retrieved, try again later")
c_q.put(3)
return
# TELLS PC TO TURN ON
print('[STATE] Checking if Server is ON...')
try:
send_magic_packet(delta, ip_address=i, port=x)
except (Exception, ConnectionResetError, socket.timeout,
paramiko.ssh_exception.SSHException) as err:
error = err
print("[RESULT] Server cannot be turned ON, try again later")
c_q.put(4)
return
# CHECKS IF PC IS ALREADY ON AND CONNECTS
try:
server.run('ls', hide=True)
verify = server.is_connected
except (Exception, ConnectionResetError, socket.timeout,
paramiko.ssh_exception.SSHException) as err:
print("[RESULT] Server is turned off --> Turning it ON...")
if not verify:
print("[ACTION] Sending Magic Packets")
print("[ACTION] Waiting for Server to turn ON. ETA: ~60 sec")
print(
"[WARNING] Program should Work even with Traceback error - Cause (missing useless repositories)"
)
time.sleep(zzz)
try:
server.run('ls', hide=True)
verify = server.is_connected
if verify:
print("[RESULT] Server is turned ON")
print()
else:
print(
"[RESULT] Server cannot be turned ON, try again later")
c_q.put(5)
return
except (Exception, ConnectionResetError, socket.timeout,
paramiko.ssh_exception.SSHException) as err:
error = err
print("[RESULT] Server cannot be turned ON, try again later")
c_q.put(5)
return
else:
print("[RESULT] Server is Turned ON")
print()
# TRY TO TRANSFER FILES TO PC
print("[STATE] Initializing File Transfer")
print(
"[SPECIFICATIONS] Folder: ALL_MODDED.zip Size: 428 MB ETA: 3-5 min"
)
print("[CONTENTS] 1 - JAVA\n")
print(" 2 - MINECRAFT 1.12.2\n")
print(" 3 - FORGE\n")
print(" 4 - MODS\n")
print(" 5 - TEXTURES\n")
print(" INSTRUCTIONS_MODDED.txt")
print()
answer = None
i = 0
while answer not in ("y", "n"):
answer = input(" DO YOU WANT TO PROCEED? y/n \n ANSWER: ")
if answer == "y" or answer == "yes":
try:
print()
print(
"[STATE] Transferring Files to this Executable's Folder"
)
print(
"[WARNING] DO NOT CLOSE THE WINDOW! It will close automatically when done"
)
server.get(
'/opt/Transfer/Modded/Distribution/ALL_MODDED.zip',
None, True)
print("[RESULT] Files Were Transferred Successfully!")
print()
c_q.put(1)
break
except:
print(
"[RESULT] Couldn't Transfer Files TO PC, Check Internet Connection or try again later"
)
c_q.put(6)
break
elif answer == "n" or answer == "no":
print("[RESULT] Exiting Program")
c_q.put(1)
break
else:
i = i + 1
if i == 3:
print()
print(
"[RESULT] Alright ya douche I'delta closing the program"
)
c_q.put(1)
break
print(
"\n[RESULT] That answer is not y(es) or n(o), care to change?"
)
answer = None
return
class ModuleFinder:
def __init__(self, path=None, debug=0, excludes=[], replace_paths=[]):
if path is None:
path = sys.path
self.path = path
self.modules = {}
self.badmodules = {}
self.debug = debug
self.indent = 0
self.excludes = excludes
self.replace_paths = replace_paths
self.processed_paths = [] # Used in debugging only
def msg(self, level, str, *args):
if level <= self.debug:
for i in range(self.indent):
print " ",
print str,
for arg in args:
print repr(arg),
print
def msgin(self, *args):
level = args[0]
if level <= self.debug:
self.indent = self.indent + 1
self.msg(*args)
def msgout(self, *args):
level = args[0]
if level <= self.debug:
self.indent = self.indent - 1
self.msg(*args)
def run_script(self, pathname):
self.msg(2, "run_script", pathname)
fp = open(pathname, READ_MODE)
stuff = ("", "r", imp.PY_SOURCE)
self.load_module('__main__', fp, pathname, stuff)
def load_file(self, pathname):
dir, name = os.path.split(pathname)
name, ext = os.path.splitext(name)
fp = open(pathname, READ_MODE)
stuff = (ext, "r", imp.PY_SOURCE)
self.load_module(name, fp, pathname, stuff)
def import_hook(self, name, caller=None, fromlist=None):
self.msg(3, "import_hook", name, caller, fromlist)
parent = self.determine_parent(caller)
q, tail = self.find_head_package(parent, name)
m = self.load_tail(q, tail)
if not fromlist:
return q
if m.__path__:
self.ensure_fromlist(m, fromlist)
return None
def determine_parent(self, caller):
self.msgin(4, "determine_parent", caller)
if not caller:
self.msgout(4, "determine_parent -> None")
return None
pname = caller.__name__
if caller.__path__:
parent = self.modules[pname]
assert caller is parent
self.msgout(4, "determine_parent ->", parent)
return parent
if '.' in pname:
i = pname.rfind('.')
pname = pname[:i]
parent = self.modules[pname]
assert parent.__name__ == pname
self.msgout(4, "determine_parent ->", parent)
return parent
self.msgout(4, "determine_parent -> None")
return None
def find_head_package(self, parent, name):
self.msgin(4, "find_head_package", parent, name)
if '.' in name:
i = name.find('.')
head = name[:i]
tail = name[i + 1:]
else:
head = name
tail = ""
if parent:
qname = "%s.%s" % (parent.__name__, head)
else:
qname = head
q = self.import_module(head, qname, parent)
if q:
self.msgout(4, "find_head_package ->", (q, tail))
return q, tail
if parent:
qname = head
parent = None
q = self.import_module(head, qname, parent)
if q:
self.msgout(4, "find_head_package ->", (q, tail))
return q, tail
self.msgout(4, "raise ImportError: No module named", qname)
raise ImportError, "No module named " + qname
def load_tail(self, q, tail):
self.msgin(4, "load_tail", q, tail)
m = q
while tail:
i = tail.find('.')
if i < 0: i = len(tail)
head, tail = tail[:i], tail[i + 1:]
mname = "%s.%s" % (m.__name__, head)
m = self.import_module(head, mname, m)
if not m:
self.msgout(4, "raise ImportError: No module named", mname)
raise ImportError, "No module named " + mname
self.msgout(4, "load_tail ->", m)
return m
def ensure_fromlist(self, m, fromlist, recursive=0):
self.msg(4, "ensure_fromlist", m, fromlist, recursive)
for sub in fromlist:
if sub == "*":
if not recursive:
all = self.find_all_submodules(m)
if all:
self.ensure_fromlist(m, all, 1)
elif not hasattr(m, sub):
subname = "%s.%s" % (m.__name__, sub)
submod = self.import_module(sub, subname, m)
if not submod:
raise ImportError, "No module named " + subname
def find_all_submodules(self, m):
if not m.__path__:
return
modules = {}
# 'suffixes' used to be a list hardcoded to [".py", ".pyc", ".pyo"].
# But we must also collect Python extension modules - although
# we cannot separate normal dlls from Python extensions.
suffixes = []
for triple in imp.get_suffixes():
suffixes.append(triple[0])
for dir in m.__path__:
try:
names = os.listdir(dir)
except os.error:
self.msg(2, "can't list directory", dir)
continue
for name in names:
mod = None
for suff in suffixes:
n = len(suff)
if name[-n:] == suff:
mod = name[:-n]
break
if mod and mod != "__init__":
modules[mod] = mod
return modules.keys()
def import_module(self, partname, fqname, parent):
self.msgin(3, "import_module", partname, fqname, parent)
try:
m = self.modules[fqname]
except KeyError:
pass
else:
self.msgout(3, "import_module ->", m)
return m
if self.badmodules.has_key(fqname):
self.msgout(3, "import_module -> None")
return None
if parent and parent.__path__ is None:
self.msgout(3, "import_module -> None")
return None
try:
fp, pathname, stuff = self.find_module(partname, parent
and parent.__path__, parent)
except ImportError:
self.msgout(3, "import_module ->", None)
return None
try:
m = self.load_module(fqname, fp, pathname, stuff)
finally:
if fp: fp.close()
if parent:
setattr(parent, partname, m)
self.msgout(3, "import_module ->", m)
return m
def load_module(self, fqname, fp, pathname, (suffix, mode, type)):
self.msgin(2, "load_module", fqname, fp and "fp", pathname)
if type == imp.PKG_DIRECTORY:
m = self.load_package(fqname, pathname)
self.msgout(2, "load_module ->", m)
return m
if type == imp.PY_SOURCE:
co = compile(fp.read() + '\n', pathname, 'exec')
elif type == imp.PY_COMPILED:
if fp.read(4) != imp.get_magic():
self.msgout(2, "raise ImportError: Bad magic number", pathname)
raise ImportError, "Bad magic number in %s" % pathname
fp.read(4)
co = marshal.load(fp)
else:
co = None
m = self.add_module(fqname)
m.__file__ = pathname
if co:
if self.replace_paths:
co = self.replace_paths_in_code(co)
m.__code__ = co
self.scan_code(co, m)
self.msgout(2, "load_module ->", m)
return m
#!/usr/bin/env python
# generate dense feature for gbdt
from datetime import datetime
import marshal
id_stat = marshal.load(open("../id_stat"))
# load name data
def load_name_sample(input, isTest):
f = open(input)
y = []
x = []
line = f.readline()
index = 3
if isTest == True:
index = 2
cnt = 0
isValid = False
while True:
line = f.readline().strip()
if not line:
break
fields = line.split(',')
if isTest == False:
label = int(fields[1])
if label == 0:
label = -1
y.append(label)
}, {
"argument1": "Anti-Arrhythmia_Agents",
"argument2": "Chemicals_and_Drugs",
"relation": "has_therapeutic_class"
}, {
"argument1": "Genetic_Variation",
"argument2": "Diseases",
"relation": "disease_may_have_cytogenetic_abnormality"
}, {
"argument1": "Lymphoproliferative_Disorders",
"argument2": "Anatomy",
"relation": "is_not_abnormal_cell_of_disease"
}]
if __name__ == '__main__':
out_dir = 'dumped-query/'
types = marshal.load(open('all_types.m', 'rb'))
tmp_utils = data_utils({
'entity_table': 'entity_table',
'relation_table': 'relation_table',
'prediction_table': 'prediction_table'
})
for i, query in enumerate(QUERY_NET):
arg1 = {'name': 'umls', 'type': '{' + query['argument1'] + '}'}
arg2 = {'name': 'umls', 'type': '{' + query['argument2'] + '}'}
if query['argument1'] in types['mesh']:
arg1['name'] = 'mesh'
if query['argument2'] in types['mesh']:
arg2['name'] = 'mesh'
result = tmp_utils.query_links(type_a=arg1,
type_b=arg2,
import marshal
import sys
import dis
f = open('crackme/this_function_contains_the_flag_1.pyc', 'rb')
f.read(8)
code_object = marshal.load(f)
code = dis.dis(code_object)
print(code)
foo = code_object.co_consts[0]
print(dis.dis(foo.co_code[7:]))
def marshal_load(f):
if isinstance(f, file):
return marshal.load(f)
return marshal.loads(f.read())
import marshal
with open('marshal_file.dat','rb') as marshal_file: #从文件中读取序列化的数据.
data1 = marshal.load(marshal_file) #数据反序列化.
data2 = marshal.load(marshal_file) #数据反序列化.
data3 = marshal.load(marshal_file) #数据反序列化.
print(data1)
print(data2)
print(data3)
def section_charge_catalogue(self, fichier):
f = open(fichier, 'rb')
self.section_catalogue.update(marshal.load(f))
f.close()
def _parse(self, ui, path):
b"Prepare list of P4 filenames and revisions to import"
p4changes = {}
changeset = {}
files_map = {}
copies_map = {}
localname = {}
depotname = {}
heads = []
ui.status(_(b'reading p4 views\n'))
# read client spec or view
if b"/" in path:
p4changes.update(self._parse_view(path))
if path.startswith(b"//") and path.endswith(b"/..."):
views = {path[:-3]: b""}
else:
views = {b"//": b""}
else:
cmd = b'p4 -G client -o %s' % procutil.shellquote(path)
clientspec = marshal.load(procutil.popen(cmd, mode=b'rb'))
views = {}
for client in clientspec:
if client.startswith(b"View"):
sview, cview = clientspec[client].split()
p4changes.update(self._parse_view(sview))
if sview.endswith(b"...") and cview.endswith(b"..."):
sview = sview[:-3]
cview = cview[:-3]
cview = cview[2:]
cview = cview[cview.find(b"/") + 1:]
views[sview] = cview
# list of changes that affect our source files
p4changes = p4changes.keys()
p4changes.sort(key=int)
# list with depot pathnames, longest first
vieworder = views.keys()
vieworder.sort(key=len, reverse=True)
# handle revision limiting
startrev = self.ui.config(b'convert', b'p4.startrev')
# now read the full changelists to get the list of file revisions
ui.status(_(b'collecting p4 changelists\n'))
lastid = None
for change in p4changes:
if startrev and int(change) < int(startrev):
continue
if self.revs and int(change) > int(self.revs[0]):
continue
if change in self.revmap:
# Ignore already present revisions, but set the parent pointer.
lastid = change
continue
if lastid:
parents = [lastid]
else:
parents = []
d = self._fetch_revision(change)
c = self._construct_commit(d, parents)
descarr = c.desc.splitlines(True)
if len(descarr) > 0:
shortdesc = descarr[0].rstrip(b'\r\n')
else:
shortdesc = b'**empty changelist description**'
t = b'%s %s' % (c.rev, repr(shortdesc)[1:-1])
ui.status(stringutil.ellipsis(t, 80) + b'\n')
files = []
copies = {}
copiedfiles = []
i = 0
while (b"depotFile%d" % i) in d and (b"rev%d" % i) in d:
oldname = d[b"depotFile%d" % i]
filename = None
for v in vieworder:
if oldname.lower().startswith(v.lower()):
filename = decodefilename(views[v] + oldname[len(v):])
break
if filename:
files.append((filename, d[b"rev%d" % i]))
depotname[filename] = oldname
if d.get(b"action%d" % i) == b"move/add":
copiedfiles.append(filename)
localname[oldname] = filename
i += 1
# Collect information about copied files
for filename in copiedfiles:
oldname = depotname[filename]
flcmd = b'p4 -G filelog %s' % procutil.shellquote(oldname)
flstdout = procutil.popen(flcmd, mode=b'rb')
copiedfilename = None
for d in loaditer(flstdout):
copiedoldname = None
i = 0
while (b"change%d" % i) in d:
if (d[b"change%d" % i] == change
and d[b"action%d" % i] == b"move/add"):
j = 0
while (b"file%d,%d" % (i, j)) in d:
if d[b"how%d,%d" % (i, j)] == b"moved from":
copiedoldname = d[b"file%d,%d" % (i, j)]
break
j += 1
i += 1
if copiedoldname and copiedoldname in localname:
copiedfilename = localname[copiedoldname]
break
if copiedfilename:
copies[filename] = copiedfilename
else:
ui.warn(
_(b"cannot find source for copied file: %s@%s\n") %
(filename, change))
changeset[change] = c
files_map[change] = files
copies_map[change] = copies
lastid = change
if lastid and len(changeset) > 0:
heads = [lastid]
return {
b'changeset': changeset,
b'files': files_map,
b'copies': copies_map,
b'heads': heads,
b'depotname': depotname,
}
p[''] = '' #ending flag
return trie, lfreq, ltotal
_curpath = os.path.normpath(
os.path.join(os.getcwd(), os.path.dirname(__file__)))
print >> sys.stderr, "Building Trie..."
t1 = time.time()
cache_file = os.path.join(tempfile.gettempdir(), "jieba.cache")
load_from_cache_fail = True
if os.path.exists(cache_file) and os.path.getmtime(
cache_file) > os.path.getmtime(os.path.join(_curpath, "dict.txt")):
print >> sys.stderr, "loading model from cache"
try:
trie, FREQ, total, min_freq = marshal.load(open(cache_file, 'rb'))
load_from_cache_fail = False
except:
load_from_cache_fail = True
if load_from_cache_fail:
trie, FREQ, total = gen_trie(os.path.join(_curpath, "dict.txt"))
FREQ = dict([(k, log(float(v) / total))
for k, v in FREQ.iteritems()]) #normalize
min_freq = min(FREQ.itervalues())
print >> sys.stderr, "dumping model to file cache"
tmp_suffix = "." + str(random.random())
tmp_f = open(cache_file + tmp_suffix, 'wb')
marshal.dump((trie, FREQ, total, min_freq), tmp_f)
tmp_f.close()
if os.name == 'nt':
model = Word2Vec.load("300features_40minwords_10context")
import marshal
from collections import defaultdict
#print(model.most_similar("food"))
#print(model.most_similar("food")[1][0])
c = [i[0] for i in model.most_similar("food")]
#print(c)
pkl_name = "cluster_model.pkl"
with open(pkl_name, 'rb') as file:
idx2 = marshal.load(file)
# Create a Word / Index dictionary, mapping each vocabulary word to
# a cluster number
word_centroid_map = dict(zip(model.wv.index2word, idx2))
word_centroid_map2 = list(zip(idx2, model.wv.index2word))
print(word_centroid_map2)
#klasters = [0,1,2,3,4,5]
#defdict = defaultdict(list)
'''
for i in word_centroid_map2:
for d,k in word_centroid_map2:
if d in klasters:
defdict[d].append(k)
# -*- coding:utf-8 -*-
from datetime import datetime
import marshal
statistics_id = marshal.load(open("../statistics_id"))
def load_name_sample(in_file, isTest):
file = open(in_file)
# save label
y = []
# save
x = []
line = file.readline()
index = 3
if isTest is True:
index = 2
count = 0
isValid = False
while True:
line = file.readline().strip()
if not line:
break
category_list = line.split(',')
if isTest is False:
# click when the value is 0, value is -1
label = int(category_list[1])
if label == 0:
label = -1
y.append(label)
def _extractPyz(self, name):
dirName = name + '_extracted'
# Create a directory for the contents of the pyz
if not os.path.exists(dirName):
os.mkdir(dirName)
with open(name, 'rb') as f:
pyzMagic = f.read(4)
assert pyzMagic == b'PYZ\0' # Sanity Check
pycHeader = f.read(4) # Python magic value
if imp.get_magic() != pycHeader:
print('[!] Warning: The script is running in a different python version than the one used to build the executable')
print(' Run this script in Python{0} to prevent extraction errors(if any) during unmarshalling'.format(self.pyver))
(tocPosition, ) = struct.unpack('!i', f.read(4))
f.seek(tocPosition, os.SEEK_SET)
try:
toc = marshal.load(f)
except:
print('[!] Unmarshalling FAILED. Cannot extract {0}. Extracting remaining files.'.format(name))
return
print('[*] Found {0} files in PYZ archive'.format(len(toc)))
# From pyinstaller 3.1+ toc is a list of tuples
if type(toc) == list:
toc = dict(toc)
for key in toc.keys():
(ispkg, pos, length) = toc[key]
f.seek(pos, os.SEEK_SET)
fileName = key
try:
# for Python > 3.3 some keys are bytes object some are str object
fileName = key.decode('utf-8')
except:
pass
# Make sure destination directory exists, ensuring we keep inside dirName
destName = os.path.join(dirName, fileName.replace("..", "__"))
destDirName = os.path.dirname(destName)
if not os.path.exists(destDirName):
os.makedirs(destDirName)
try:
data = f.read(length)
data = zlib.decompress(data)
except:
print('[!] Error: Failed to decompress {0}, probably encrypted. Extracting as is.'.format(fileName))
open(destName + '.pyc.encrypted', 'wb').write(data)
continue
with open(destName + '.pyc', 'wb') as pycFile:
pycFile.write(pycHeader) # Write pyc magic
pycFile.write(b'\0' * 4) # Write timestamp
if self.pyver >= 33:
pycFile.write(b'\0' * 4) # Size parameter added in Python 3.3
pycFile.write(data)
def xreload(mod):
"""Reload a module in place, updating classes, methods and functions.
Args:
mod: a module object
Returns:
The (updated) input object itself.
"""
# Get the module name, e.g. 'foo.bar.whatever'
modname = mod.__name__
# Get the module namespace (dict) early; this is part of the type check
modns = mod.__dict__
# Parse it into package name and module name, e.g. 'foo.bar' and 'whatever'
i = modname.rfind(".")
if i >= 0:
pkgname, modname = modname[:i], modname[i + 1:]
else:
pkgname = None
# Compute the search path
if pkgname:
# We're not reloading the package, only the module in it
pkg = sys.modules[pkgname]
path = pkg.__path__ # Search inside the package
else:
# Search the top-level module path
pkg = None
path = None # Make find_module() uses the default search path
# Find the module; may raise ImportError
(stream, filename, (suffix, mode, kind)) = imp.find_module(modname, path)
# Turn it into a code object
try:
# Is it Python source code or byte code read from a file?
if kind not in (imp.PY_COMPILED, imp.PY_SOURCE):
# Fall back to built-in reload()
return reload(mod)
if kind == imp.PY_SOURCE:
source = stream.read()
code = compile(source, filename, "exec")
else:
code = marshal.load(stream)
finally:
if stream:
stream.close()
# Execute the code. We copy the module dict to a temporary; then
# clear the module dict; then execute the new code in the module
# dict; then swap things back and around. This trick (due to
# Glyph Lefkowitz) ensures that the (readonly) __globals__
# attribute of methods and functions is set to the correct dict
# object.
tmpns = modns.copy()
modns.clear()
modns["__name__"] = tmpns["__name__"]
exec(code, modns)
# Now we get to the hard part
oldnames = set(tmpns)
newnames = set(modns)
# Update attributes in place
for name in oldnames & newnames:
modns[name] = _update(tmpns[name], modns[name])
# Done!
return mod
assert os.path.exists(infile)
if compressing or encoding:
fp = open(infile, 'rb')
msg = fp.read()
fp.close()
if compressing:
compr, decoder = compress(msg)
fcompressed = open(outfile, 'wb')
marshal.dump((pickle.dumps(decoder), compr), fcompressed)
fcompressed.close()
else:
enc, decoder = encode(msg)
print(enc)
fcompressed = open(outfile, 'wb')
marshal.dump((pickle.dumps(decoder), enc), fcompressed)
fcompressed.close()
else:
fp = open(infile, 'rb')
pickleRick, compr = marshal.load(fp)
decoder = pickle.loads(pickleRick)
fp.close()
if decompressing:
msg = decompress(compr, decoder)
else:
msg = decode(compr, decoder)
print(msg)
fp = open(outfile, 'wb')
fp.write(msg)
fp.close()
def marshal_load_obj(f):
_f = open(f, 'rb')
o = marshal.load(_f)
_f.close()
return o
def initialize(self, dictionary=None):
if dictionary:
abs_path = _get_abs_path(dictionary)
if self.dictionary == abs_path and self.initialized:
return
else:
self.dictionary = abs_path
self.initialized = False
else:
abs_path = self.dictionary
with self.lock:
try:
with DICT_WRITING[abs_path]:
pass
except KeyError:
pass
if self.initialized:
return
default_logger.debug("Building prefix dict from %s ..." %
(abs_path or 'the default dictionary'))
t1 = time.time()
if self.cache_file:
cache_file = self.cache_file
# default dictionary
elif abs_path == DEFAULT_DICT:
cache_file = "jieba.cache"
# custom dictionary
else:
cache_file = "jieba.u%s.cache" % md5(
abs_path.encode('utf-8', 'replace')).hexdigest()
cache_file = os.path.join(self.tmp_dir or tempfile.gettempdir(),
cache_file)
# prevent absolute path in self.cache_file
tmpdir = os.path.dirname(cache_file)
load_from_cache_fail = True
if os.path.isfile(cache_file) and (abs_path == DEFAULT_DICT
or os.path.getmtime(cache_file)
> os.path.getmtime(abs_path)):
default_logger.debug("Loading model from cache %s" %
cache_file)
try:
with open(cache_file, 'rb') as cf:
self.FREQ, self.total = marshal.load(cf)
load_from_cache_fail = False
except Exception:
load_from_cache_fail = True
if load_from_cache_fail:
wlock = DICT_WRITING.get(abs_path, threading.RLock())
DICT_WRITING[abs_path] = wlock
with wlock:
self.FREQ, self.total = self.gen_pfdict(
self.get_dict_file())
default_logger.debug("Dumping model to file cache %s" %
cache_file)
try:
# prevent moving across different filesystems
fd, fpath = tempfile.mkstemp(dir=tmpdir)
with os.fdopen(fd, 'wb') as temp_cache_file:
marshal.dump((self.FREQ, self.total),
temp_cache_file)
os.chmod(fpath, 0o666)
_replace_file(fpath, cache_file)
except Exception:
default_logger.exception("Dump cache file failed.")
try:
del DICT_WRITING[abs_path]
except KeyError:
pass
self.initialized = True
default_logger.debug("Loading model cost %.3f seconds." %
(time.time() - t1))
default_logger.debug("Prefix dict has been built successfully.")
def _fetch_revision(self, rev):
"""Return an output of `p4 describe` including author, commit date as
a dictionary."""
cmd = "p4 -G describe -s %s" % rev
stdout = util.popen(cmd, mode="rb")
return marshal.load(stdout)
def processChanges(self, changes):
'''
Loop over the changes:
a. sync to the change
b. capture the p4 commit message, user, date and CL#
c. rm everything in tamarin minus .hg/
d. copy everything from p4 clientspace into tamarin MINUS tamarin-merge.txt
e. hg commit using p4 message (append CL# to message), p4 user and p4 date if possible
f. hg push this change (this will make sure that if there is another change available
that for some reason bombs out, we have at least pushed to mozilla what we could)
g. if the push fails....
h. rewrite the external tracking file with this CL#
'''
user = ''
changelist = ''
desc = ''
date = ''
for dict in changes:
changelist = dict["change"]
self.log("\nProcessing changelist: %s" % changelist)
#########################################
# a. sync to the change
#########################################
self.log("Sync to the change...")
cmd = "p4 sync %s@%s" % (self.P4AVMPLUS, changelist)
stdout, stderr, exit = self.run_pipe(cmd=cmd, env=self.env)
for line in stdout:
self.log(line)
if stderr:
for line in stderr:
self.log(line)
if exit:
sys.exit(exit)
#########################################
# b. capture the p4 commit message, user,
# date and CL#
#########################################
cmd = "p4 -G describe -s %s" % (dict["change"])
pipe = Popen(cmd.split(), stdout=PIPE).stdout
try: # The -G option on p4 returns a python object, so need to be loaded via marshal.load()
while 1:
record = marshal.load(pipe)
except EOFError:
pass
pipe.close()
user = record["user"]
date = record["time"]
desc = record["desc"]
cmd = "p4 -G user -o %s" % (user)
pipe = Popen(cmd.split(), stdout=PIPE).stdout
try: # The -G option on p4 returns a python object, so need to be loaded via marshal.load()
while 1:
record = marshal.load(pipe)
except EOFError:
pass
pipe.close()
user = "******" % (record["FullName"], record["Email"])
#########################################
# c. rm everything in tamarin minus .hg/
#########################################
self.log("Clean out the mirror repo...")
for filename in os.listdir(self.TAMARIN_REPO):
fullpath = "%s/%s" % (self.TAMARIN_REPO, filename)
if filename != ".hg":
if os.path.isfile(fullpath):
os.unlink(fullpath)
else:
shutil.rmtree(fullpath)
#########################################
# d. copy everything from p4 clientspace into tamarin
#########################################
self.log("Repopulate the mirror repo from p4 workspace...")
for filename in os.listdir(self.P4ROOT):
src = "%s/%s" % (self.P4ROOT, filename)
dest = "%s/%s" % (self.TAMARIN_REPO, filename)
if os.path.isfile(src):
shutil.copy2(src, dest)
else:
shutil.copytree(src, dest)
#########################################
# e. hg commit using p4 message (append CL# to message),
# p4 user and p4 date if possible
#########################################
self.log("Commit the change to the mirror repo...")
commit_message = desc + "\nCL@" + changelist
fd, temp_path = tempfile.mkstemp()
os.write(fd, commit_message)
os.close(fd)
# Massage the date
d = datetime.datetime.fromtimestamp(float(date),
pytz.timezone("US/Pacific"))
date = d.strftime("%a %b %d %H:%M:%S %Y %z")
cmd = "hg commit --addremove --user \"%s\" --date \"%s\" --logfile %s" % (
user, date, temp_path)
self.log(cmd)
stdout, stderr, exit = self.run_pipe(cmd=cmd,
cwd=self.TAMARIN_REPO,
env=self.env)
for line in stdout:
self.log(line)
if stderr:
for line in stderr:
self.log(line)
if exit:
sys.exit(exit)
# Make sure to remove the temp file
os.unlink(temp_path)
#########################################
# f. hg push this change (this will make sure that if
# there is another change available that for some
# reason bombs out, we have at least pushed to mozilla
# what we could)
#########################################
self.log("Push the change to the mirror repo...")
cmd = "hg push"
stdout, stderr, exit = self.run_pipe(cmd=cmd,
cwd=self.TAMARIN_REPO,
env=self.env)
for line in stdout:
self.log(line)
#########################################
# g. if the push fails....
#########################################
if stderr:
for line in stderr:
self.log(line)
if exit:
sys.exit(exit)
#########################################
# h. rewrite the external tracking file with this CL#
#########################################
self.log("Update changelist tracking file...")
bf = open(self.LASTBUILD_FILE, 'w')
bf.write(changelist)
bf.close()
self.log(
"Completed changelist: %s\n\n#########################################"
% changelist)
def _extractPyz(self, name):
dirName = name + '_extracted'
# Create a directory for the contents of the pyz
if not os.path.exists(dirName):
os.mkdir(dirName)
with open(name, 'rb') as f:
pyzMagic = f.read(4)
assert pyzMagic == b'PYZ\0' # Sanity Check
pycHeader = f.read(4) # Python magic value
# Skip PYZ extraction if not running under the same python version
if pyc_magic != pycHeader:
print(
'[!] Warning: This script is running in a different Python version than the one used to build the executable.'
)
print(
'[!] Please run this script in Python{0} to prevent extraction errors during unmarshalling'
.format(self.pyver))
print('[!] Skipping pyz extraction')
return
(tocPosition, ) = struct.unpack('!i', f.read(4))
f.seek(tocPosition, os.SEEK_SET)
try:
toc = marshal.load(f)
except:
print(
'[!] Unmarshalling FAILED. Cannot extract {0}. Extracting remaining files.'
.format(name))
return
print('[+] Found {0} files in PYZ archive'.format(len(toc)))
# From pyinstaller 3.1+ toc is a list of tuples
if type(toc) == list:
toc = dict(toc)
for key in toc.keys():
(ispkg, pos, length) = toc[key]
f.seek(pos, os.SEEK_SET)
fileName = key
try:
# for Python > 3.3 some keys are bytes object some are str object
fileName = fileName.decode('utf-8')
except:
pass
# Prevent writing outside dirName
fileName = fileName.replace('..',
'__').replace('.', os.path.sep)
if ispkg == 1:
filePath = os.path.join(dirName, fileName, '__init__.pyc')
else:
filePath = os.path.join(dirName, fileName + '.pyc')
fileDir = os.path.dirname(filePath)
if not os.path.exists(fileDir):
os.makedirs(fileDir)
try:
data = f.read(length)
data = zlib.decompress(data)
except:
print(
'[!] Error: Failed to decompress {0}, probably encrypted. Extracting as is.'
.format(filePath))
open(filePath + '.encrypted', 'wb').write(data)
else:
self._writePyc(filePath, data)
def _check_output(self):
self.output.seek(0)
self._assert_expected_item(marshal.load(self.output))
