def run(self, row):
if not row:
print 'Complete'
proceed = False
self.lastrow = row['rowid']
jssdhash = make_sdhash(row['js'])
#de_jssdhash = make_sdhash(row['de_js'])
swfsdhash = make_sdhash(row['swf'])
#actionscriptsdhash = make_sdhash(row['actionscript'])
#self.subs = (jssdhash, de_jssdhash, swfsdhash, actionscriptsdhash, row['rowid'])
self.subs = (jssdhash, swfsdhash, row['rowid'])
self.verbose(row, 'js', jssdhash)
#self.verbose(row, 'de_Js', de_jssdhash)
self.verbose(row, 'swf', swfsdhash)
def run(self, row):
if not row:
print 'Complete'
proceed = False
self.lastrow = row['rowid']
jssdhash = make_sdhash(row['js'])
# de_jssdhash = make_sdhash(row['de_js'])
swfsdhash = make_sdhash(row['swf'])
# actionscriptsdhash = make_sdhash(row['actionscript'])
# self.subs = (jssdhash, de_jssdhash, swfsdhash, actionscriptsdhash, row['rowid'])
self.subs = (jssdhash, swfsdhash, row['rowid'])
self.verbose(row, 'js', jssdhash)
# self.verbose(row, 'de_Js', de_jssdhash)
self.verbose(row, 'swf', swfsdhash)
logmsg(log, '%s: No pdf returned\n' % JSFLASH)
break
if not code:
logmsg(log, '%s: No code returned %s\n' % (JSFLASH, pdf))
continue
cnt += 1
if JSFLASH == 'js':
msg = 'JS CNT: %6d\tRID: %6d\tFile: %s\n' % (cnt, rid, pdf)
logmsg(log, msg)
try:
etree = tree_from_xml(xml)
de_js = clarify_js(code, etree)
de_js = beautify(de_js, jsopts)
de_js_sdhash = make_sdhash(de_js, log)
except Exception as e:
de_js = 'error: %s' % e
logmsg(log, 'Clarification error [%s]: %s\n' % (pdf, str(e)))
col = ('pdf_md5', 'js', 'de_js', 'de_js_sdhash')
val = (pdf, code, de_js, de_js_sdhash)
store(log, 'clarified', col, val)
elif JSFLASH == 'flash':
msg = 'FL CNT: %6d\tRID: %6d\tFile: %s\n' % (cnt, rid, pdf)
logmsg(log, msg)
try:
ascript = decompile_flash(code)
as_sdhash = make_sdhash(ascript, log)
def run(self):
while True:
pdf = self.qin.get()
if not pdf:
'''
This terminates the process by receiving a poison sentinel, None.
'''
self.qout.put(None)
self.qin.task_done()
return 0
'''
Reset the values on each pdf.
'''
err = []
urls = ''
t_hash = ''
t_str = ''
graph = ''
obf_js = ''
de_js = ''
obf_js_sdhash = ''
de_js_sdhash = ''
swf_sdhash = ''
swf = ''
fsize = ''
pdfsize = ''
bin_blob = ''
malformed = {}
'''
Arguments are validated when Jobber adds them to the queue based
on the Validators valid() return value. We can assume these will
succeed. However, this process must reach the task_done() call,
and so we try/catch everything
'''
try:
pdf_name = pdf.rstrip(os.path.sep).rpartition(os.path.sep)[2]
except Exception as e:
err.append('UNEXPECTED OS ERROR:\n%s' % traceback.format_exc())
pdf_name = pdf
write('H\t#%d\t(%d / %d)\t%s\n' % (self.pid, self.counter.value(), self.counter.ceil(), pdf_name))
'''
The parse_pdf call will return a value that evaluates to false if it
did not succeed. Error messages will appended to the err list.
'''
parsed_pdf = self.parse_pdf(pdf, err)
if parsed_pdf:
try:
fsize = self.get_file_size(pdf)
pdfsize = self.get_pdf_size(parsed_pdf, err)
graph = self.make_graph(parsed_pdf, err)
t_str = self.make_tree_string(parsed_pdf, err)
t_hash = self.make_tree_hash(graph, err)
obf_js = self.get_js(parsed_pdf, err)
de_js = self.get_deobf_js(obf_js, parsed_pdf, err)
obf_js_sdhash = make_sdhash(obf_js, err)
de_js_sdhash = make_sdhash(de_js, err)
urls = self.get_urls(obf_js, err)
urls += self.get_urls(de_js, err)
swf = self.get_swf(parsed_pdf, err)
swf_sdhash = make_sdhash(swf, err)
bin_blob = parsed_pdf.bin_blob
malformed = parsed_pdf.getmalformed()
self.get_errors(parsed_pdf, err)
except Exception as e:
err.append('UNCAUGHT PARSING EXCEPTION:\n%s' % traceback.format_exc())
err = 'Error: '.join(err)
malformed['skipkeys'] = False
try:
json_malformed = json.dumps(malformed)
except (TypeError, ValueError):
malformed['skipkeys'] = True
json_malformed = json.dumps(malformed, skipkeys=True)
self.qout.put({'fsize': fsize,
'pdf_md5': pdf_name,
'tree_md5': t_hash,
'tree': t_str,
'obf_js': obf_js,
'de_js': de_js,
'swf': swf,
'graph': graph,
'pdfsize': pdfsize,
'urls': urls,
'bin_blob': bin_blob,
'obf_js_sdhash': obf_js_sdhash,
'de_js_sdhash': de_js_sdhash,
'swf_sdhash': swf_sdhash,
'malformed': json_malformed,
'errors': err})
self.counter.inc()
self.qin.task_done()
logmsg(log, '%s: No pdf returned\n' % JSFLASH)
break
if not code:
logmsg(log, '%s: No code returned %s\n' % (JSFLASH, pdf))
continue
cnt += 1
if JSFLASH == 'js':
msg = 'JS CNT: %6d\tRID: %6d\tFile: %s\n' % (cnt, rid, pdf)
logmsg(log, msg)
try:
etree = tree_from_xml(xml)
de_js = clarify_js(code, etree)
de_js = beautify(de_js, jsopts)
de_js_sdhash = make_sdhash(de_js, log)
except Exception as e:
de_js = 'error: %s' % e
logmsg(log, 'Clarification error [%s]: %s\n' % (pdf, str(e)))
col = ('pdf_md5', 'js', 'de_js', 'de_js_sdhash')
val = (pdf, code, de_js, de_js_sdhash)
store(log, 'clarified', col, val)
elif JSFLASH == 'flash':
msg = 'FL CNT: %6d\tRID: %6d\tFile: %s\n' % (cnt, rid, pdf)
logmsg(log, msg)
try:
ascript = decompile_flash(code)
as_sdhash = make_sdhash(ascript, log)
def run(self):
while True:
pdf = self.qin.get()
if not pdf:
'''
This terminates the process by receiving a poison sentinel, None.
'''
self.qout.put(None)
self.qin.task_done()
return 0
'''
Reset the values on each pdf.
'''
err = []
urls = ''
t_hash = ''
t_str = ''
graph = ''
obf_js = ''
de_js = ''
obf_js_sdhash = ''
de_js_sdhash = ''
swf_sdhash = ''
swf = ''
fsize = ''
pdfsize = ''
bin_blob = ''
malformed = {}
'''
Arguments are validated when Jobber adds them to the queue based
on the Validators valid() return value. We can assume these will
succeed. However, this process must reach the task_done() call,
and so we try/catch everything
'''
try:
pdf_name = pdf.rstrip(os.path.sep).rpartition(os.path.sep)[2]
except Exception as e:
err.append('UNEXPECTED OS ERROR:\n%s' % traceback.format_exc())
pdf_name = pdf
write('H\t#%d\t(%d / %d)\t%s\n' % (self.pid, self.counter.value(),
self.counter.ceil(), pdf_name))
'''
The parse_pdf call will return a value that evaluates to false if it
did not succeed. Error messages will appended to the err list.
'''
parsed_pdf = self.parse_pdf(pdf, err)
if parsed_pdf:
try:
fsize = self.get_file_size(pdf)
pdfsize = self.get_pdf_size(parsed_pdf, err)
graph = self.make_graph(parsed_pdf, err)
t_str = self.make_tree_string(parsed_pdf, err)
t_hash = self.make_tree_hash(graph, err)
obf_js = self.get_js(parsed_pdf, err)
de_js = self.get_deobf_js(obf_js, parsed_pdf, err)
obf_js_sdhash = make_sdhash(obf_js, err)
de_js_sdhash = make_sdhash(de_js, err)
urls = self.get_urls(obf_js, err)
urls += self.get_urls(de_js, err)
swf = self.get_swf(parsed_pdf, err)
swf_sdhash = make_sdhash(swf, err)
bin_blob = parsed_pdf.bin_blob
malformed = parsed_pdf.getmalformed()
self.get_errors(parsed_pdf, err)
except Exception as e:
err.append('UNCAUGHT PARSING EXCEPTION:\n%s' %
traceback.format_exc())
err = 'Error: '.join(err)
malformed['skipkeys'] = False
try:
json_malformed = json.dumps(malformed)
except (TypeError, ValueError):
malformed['skipkeys'] = True
json_malformed = json.dumps(malformed, skipkeys=True)
self.qout.put({
'fsize': fsize,
'pdf_md5': pdf_name,
'tree_md5': t_hash,
'tree': t_str,
'obf_js': obf_js,
'de_js': de_js,
'swf': swf,
'graph': graph,
'pdfsize': pdfsize,
'urls': urls,
'bin_blob': bin_blob,
'obf_js_sdhash': obf_js_sdhash,
'de_js_sdhash': de_js_sdhash,
'swf_sdhash': swf_sdhash,
'malformed': json_malformed,
'errors': err
})
self.counter.inc()
self.qin.task_done()
