def test_iter(self):
cfg = CFG([Variable("S"),
Variable("A"),
Variable("B"),
Variable("C")],
[Terminal("a"), Terminal("b")], [
CFGRule(Variable("S"),
[Variable("A"), Variable("B")]),
CFGRule(Variable("S"), [Terminal("a")]),
CFGRule(Variable("A"), [Terminal("b")]),
CFGRule(Variable("A"), [Variable("C")])
], Variable("S"))
words = [x for x in cfg]
self.assertEqual(words, [[Terminal("a")]])
cfg = CFG([Variable("S"), Variable("A"),
Variable("C")],
[Terminal("a"), Terminal("b")], [
CFGRule(Variable("S"), [Variable("A")]),
CFGRule(Variable("S"), [Terminal("a")]),
CFGRule(Variable("A"), [Terminal("b")]),
CFGRule(Variable("A"), [Variable("C")])
], Variable("S"))
words = [x for x in cfg]
self.assertIn([Terminal("a")], words)
self.assertIn([Terminal("b")], words)
self.assertEqual(len(words), 2)
cfg = CFG([Variable("S"), Variable("A"),
Variable("B")],
[Terminal("a"), Terminal("b")], [
CFGRule(Variable("S"),
[Variable("S"), Variable("S")]),
CFGRule(Variable("S"), [Terminal("a")]),
CFGRule(Variable("A"), [Terminal("b")])
], Variable("S"))
it = iter(cfg)
word = next(it)
self.assertEqual(set(word), set([Terminal("a")]))
word = next(it)
self.assertEqual(set(word), set([Terminal("a")]))
word = next(it)
self.assertEqual(set(word), set([Terminal("a")]))
word = next(it)
self.assertEqual(set(word), set([Terminal("a")]))
word = next(it)
self.assertEqual(set(word), set([Terminal("a")]))
cfg = CFG([Variable("S"), Variable("A"),
Variable("B")],
[Terminal("a"), Terminal("b")], [
CFGRule(Variable("S"),
[Variable("A"), Variable("S")]),
CFGRule(Variable("S"), [Terminal("a")]),
CFGRule(Variable("A"), [Terminal("b")])
], Variable("S"))
it = iter(cfg)
temp = [next(it) for _ in range(100)]
self.assertIn(
[Terminal("b"),
Terminal("b"),
Terminal("b"),
Terminal("a")], temp)
def test_to_pda(self):
cfg = CFG([Variable("E"), Variable("I")], [
Terminal("a"),
Terminal("b"),
Terminal("0"),
Terminal("1"),
Terminal("+"),
Terminal("*"),
Terminal("("),
Terminal(")")
], [
CFGRule(Variable("I"), [Terminal("a")]),
CFGRule(Variable("I"), [Terminal("b")]),
CFGRule(Variable("I"),
[Variable("I"), Terminal("a")]),
CFGRule(Variable("I"),
[Variable("I"), Terminal("b")]),
CFGRule(Variable("I"),
[Variable("I"), Terminal("0")]),
CFGRule(Variable("I"),
[Variable("I"), Terminal("1")]),
CFGRule(Variable("E"), [Variable("I")]),
CFGRule(
Variable("E"),
[Variable("E"), Terminal("*"),
Variable("E")]),
CFGRule(
Variable("E"),
[Variable("E"), Terminal("+"),
Variable("E")]),
CFGRule(
Variable("E"),
[Terminal("("), Variable("E"),
Terminal(")")])
], Variable("E"))
pda = cfg.to_PDA()
self.assertIsInstance(pda, PDA)
self.assertIn(
PDATransitionFunction(PDAState("q"), "epsilon", Variable("I"),
PDAState("q"), [Terminal("a")]),
pda.transition_function)
self.assertIn(
PDATransitionFunction(PDAState("q"), "epsilon", Variable("I"),
PDAState("q"),
[Variable("I"), Terminal("0")]),
pda.transition_function)
self.assertIn(
PDATransitionFunction(PDAState("q"), "epsilon", Variable("E"),
PDAState("q"), [Variable("I")]),
pda.transition_function)
self.assertEqual(18, len(pda.transition_function))
self.assertTrue(pda.accepts_by_empty_stack([Terminal("a")], 100))
self.assertTrue(pda.accepts_by_empty_stack([Terminal("b")], 100))
self.assertFalse(
pda.accepts_by_empty_stack([Terminal(x) for x in "b0"], 100))
self.assertTrue(
pda.accepts_by_empty_stack([Terminal(x) for x in "b0"], 1000))
self.assertTrue(
pda.accepts_by_empty_stack([Terminal(x) for x in "b00"], 10000))
def test_emptiness(self):
cfg = CFG([Variable("S"), Variable("A"),
Variable("B")],
[Terminal("a"), Terminal("b")], [
CFGRule(Variable("S"),
[Variable("A"), Variable("B")]),
CFGRule(Variable("S"), [Terminal("a")]),
CFGRule(Variable("A"), [Terminal("b")])
], Variable("S"))
self.assertEqual(
set([Variable("S"),
Variable("A"),
Terminal("a"),
Terminal("b")]), cfg.get_reachable())
self.assertFalse(cfg.is_empty())
cfg = CFG([Variable("S"), Variable("A"),
Variable("B")],
[Terminal("a"), Terminal("b")], [
CFGRule(Variable("S"),
[Variable("A"), Variable("B")]),
CFGRule(Variable("S"), [Terminal("a")]),
CFGRule(Variable("A"), [Terminal("b")])
], Variable("B"))
self.assertEqual(
set([Variable("S"),
Variable("A"),
Terminal("a"),
Terminal("b")]), cfg.get_reachable())
self.assertTrue(cfg.is_empty())
def __init__( self ):
self.__gobject_init__()
self.alive = True
self.debugging = True
self.displays = [] # List of devices
self.current_display = -1 # Current device, -1 means no devices
self.append = True
self.modules = {}
self.signals = []
self.funcs = []
self.windows = {}
self.widget_template = {}
for window in windows:
self.windows[window] = False
self.widgets = []
self.current_window = ''
self.special_chars_set_signal = gobject.signal_new("special-chars-set", Generic, gobject.SIGNAL_RUN_LAST, gobject.TYPE_NONE, [])
self.layout_transition_finished_signal = gobject.signal_new("layout-transition-finished", Generic, gobject.SIGNAL_RUN_LAST, gobject.TYPE_NONE, [])
self.special_char_changed_signal = gobject.signal_new("special-char-changed", Generic, gobject.SIGNAL_RUN_LAST, gobject.TYPE_NONE, [gobject.TYPE_PYOBJECT])
self.layout_change_before_signal = gobject.signal_new("layout-change-before", Generic, gobject.SIGNAL_RUN_LAST, gobject.TYPE_NONE, [])
self.layout_change_after_signal = gobject.signal_new("layout-change-after", Generic, gobject.SIGNAL_RUN_LAST, gobject.TYPE_NONE, [])
self.display_disconnected_signal = gobject.signal_new('display-disconnected', Generic, gobject.SIGNAL_RUN_LAST, gobject.TYPE_NONE, [])
self.display_disconnected_before_signal = gobject.signal_new('display-disconnected-before', Generic, gobject.SIGNAL_RUN_LAST, gobject.TYPE_NONE, [])
self.display_connected_signal = gobject.signal_new('display-connected', Generic, gobject.SIGNAL_RUN_LAST, gobject.TYPE_NONE, [])
self.display_changed_signal = gobject.signal_new('display-changed', Generic, gobject.SIGNAL_RUN_LAST, gobject.TYPE_NONE, [])
self.book_changed_signal = gobject.signal_new('book-changed', Generic, gobject.SIGNAL_RUN_LAST, gobject.TYPE_NONE, [])
self.page_changed_signal = gobject.signal_new('page-changed', Generic, gobject.SIGNAL_RUN_LAST, gobject.TYPE_NONE, [])
self.version_ready_signal = gobject.signal_new('version-ready', Generic, gobject.SIGNAL_RUN_LAST, gobject.TYPE_NONE, [])
self.contrast_backlight_ready_signal = gobject.signal_new('contrast-backlight-ready', Generic, gobject.SIGNAL_RUN_LAST, gobject.TYPE_NONE, [])
self.packet_ready_signal = gobject.signal_new('packet-ready', Generic, gobject.SIGNAL_RUN_LAST, gobject.TYPE_NONE, [gobject.TYPE_PYOBJECT])
self.temp_packet_ready_signal = gobject.signal_new('temperature-packet-ready', Generic, gobject.SIGNAL_RUN_LAST, gobject.TYPE_NONE, [gobject.TYPE_PYOBJECT])
self.fan_packet_ready_signal = gobject.signal_new('fan-packet-ready', Generic, gobject.SIGNAL_RUN_LAST, gobject.TYPE_NONE, [gobject.TYPE_INT])
self.key_packet_ready_signal = gobject.signal_new('key-packet-ready', Generic, gobject.SIGNAL_RUN_LAST, gobject.TYPE_NONE, [gobject.TYPE_INT])
#self.devices_ready_signal = gobject.signal_new('devices-ready', App, gobject.SIGNAL_RUN_LAST, gobject.TYPE_NONE, ())
self.dow_ready_signal = gobject.signal_new('dow-ready', Generic, gobject.SIGNAL_RUN_LAST, gobject.TYPE_NONE, [gobject.TYPE_INT])
self.lcd_memory_ready_signal = gobject.signal_new('lcd-memory-ready', Generic, gobject.SIGNAL_RUN_LAST, gobject.TYPE_NONE, [gobject.TYPE_PYOBJECT])
self._8bytes_packet_ready_signal = gobject.signal_new('8byte-packet-ready', Generic, gobject.SIGNAL_RUN_LAST, gobject.TYPE_NONE, [])
self._8bytes_fill_start_signal = gobject.signal_new('8byte-fill-start', Generic, gobject.SIGNAL_RUN_LAST, gobject.TYPE_NONE, [])
self.fans_ready_signal = gobject.signal_new('fans-ready', Generic, gobject.SIGNAL_RUN_LAST, gobject.TYPE_NONE, [])
CFDmesg.__init__(self)
CFSensors.__init__(self)
CFMain.__init__(self)
CFPreferences.__init__(self)
CFLCD.__init__(self)
CFDisplay.__init__(self)
CFWidget.__init__(self)
CFLayout.__init__(self)
#self.config = cfg_get()
CFG.__init__(self)
self.config_setup()
def construct(self):
def extract_calls(cfg): # get all calls from cfg
calls = {}
visited = set()
work_list = [cfg.start]
while len(work_list) > 0:
op = work_list.pop()
visited.add(op)
# indirect calls (call %rcx, etc) not currently supported
if op.id == X86_INS_CALL and op.operands[0].type == X86_OP_IMM:
calls[op.address] = op.operands[0].imm
for succ in op.succs:
succ_op = cfg[succ]
if succ_op not in visited:
work_list.append(succ_op)
return calls
worklist = [self.start_addr]
while len(worklist) > 0:
func_addr = worklist.pop()
func_cfg = CFG(self.elf, func_addr)
func_calls = extract_calls(func_cfg)
self.cfgs[func_addr] = func_cfg
self.calls.update(func_calls)
worklist.extend(set(func_calls.values()) - set(self.cfgs))
def test_paper(self):
cfg = CFG([Variable("S"), Variable("C")], [
Terminal("a"),
Terminal("b"),
Terminal("c"),
Terminal("q"),
Terminal("am"),
Terminal("bm"),
Terminal("cm"),
Terminal("qm")
], [
CFGRule(
Variable("S"),
[Variable("C"), Terminal("q"),
Variable("C")]),
CFGRule(
Variable("C"),
[Terminal("a"), Variable("C"),
Terminal("am")]),
CFGRule(
Variable("C"),
[Terminal("b"), Variable("C"),
Terminal("bm")]),
CFGRule(
Variable("C"),
[Terminal("c"), Variable("C"),
Terminal("cm")]),
CFGRule(
Variable("C"),
[Terminal("q"), Variable("C"),
Terminal("qm")]),
CFGRule(Variable("C"),
[Variable("C"), Variable("C")]),
CFGRule(Variable("C"), [])
], Variable("S"))
regex = RegexTree(Node("(a,b)|(bm,c)|(cm,am,q)"))
fsm = regex.to_fsm()
fsm.close()
cfg_temp = cfg.intersect(fsm)
self.assertFalse(cfg_temp.is_empty())
regex = RegexTree(Node("(a,b)|(b,c)|(cm,am,q)"))
fsm = regex.to_fsm()
fsm.close()
cfg_temp = cfg.intersect(fsm)
self.assertTrue(cfg_temp.is_empty())
def test_intersect(self):
cfg = CFG([Variable("S"), Variable("A")],
[Terminal("a"), Terminal("b")], [
CFGRule(Variable("S"),
[Variable("A"), Variable("A")]),
CFGRule(Variable("S"), [Terminal("a")]),
CFGRule(Variable("A"), [Terminal("b")])
], Variable("S"))
regex = RegexTree(Node("a"))
fsm = regex.to_fsm()
fsm.close()
cfg_temp = cfg.intersect(fsm)
self.assertFalse(cfg_temp.is_empty())
regex = RegexTree(Node("b"))
fsm = regex.to_fsm()
fsm.close()
cfg_temp = cfg.intersect(fsm)
self.assertFalse(cfg_temp.is_empty())
regex = RegexTree(Node("b,b"))
fsm = regex.to_fsm()
fsm.close()
cfg_temp = cfg.intersect(fsm)
self.assertFalse(cfg_temp.is_empty())
regex = RegexTree(Node("b,a"))
fsm = regex.to_fsm()
fsm.close()
cfg_temp = cfg.intersect(fsm)
self.assertTrue(cfg_temp.is_empty())
def __init__ ( self, visitor, obj=None, config=None, type=TYPE_TEXT):
"""___init___ () - Inits the driver
"""
self.__gobject_init__()
self.visitor = visitor
if obj == None:
self.layouts = {}
self.widgets = {}
self.static_widgets = []
self.fake = False
self.ser = None
self.display_name = ''
self.serial_number = ''
else:
self.layouts = obj.layouts
self.fake = obj.fake
self.ser = obj.ser
self.display_name = obj.display_name
self.device_name = obj.device_name
self.path = obj.path
self.books = obj.books
self.current_command = obj.current_command
self.command_queue0 = obj.command_queue0
self.command_queue1 = obj.command_queue1
self.response_time_init = time.time()
self.TYPE = type
self.current_layout = None
self.app = visitor
#self.debug = visitor.debug
self.layoutG = self.GetLayout()
self.current_incr = 0
self.layout_id = None
self.transition_id = None
CFG.__init__(self, config)
Evaluator.__init__(self)
self.connect("layout-transition-finished", self.StartLayout)
self.AddFunction("transition", 1, self.my_transition)
def to_CFG(self, preprocess=False):
if preprocess:
return self.__preprocess_to_CFG().to_CFG(False)
self.__cache_variables = dict()
self.__cache_counter = 0
from CFG import CFG
start = Variable("S")
terminals = list(map(Terminal, self.input_symbols[:]))
productions = self.__init_productions_to_CFG(start)
counter = 0.0
for trans in self.transition_function:
counter += 1
self.__process_transition_to_CFG(trans, productions)
variables = list(self.__cache_variables.values())
variables.append(start)
return CFG(variables, terminals, productions, start)
def train(cfg_file, train_file, iter_num=20):
cfg = CFG(cfg_file=cfg_file)
pcfg = PCFG_EM(train_file=train_file, CFG=cfg)
(name, ext) = os.path.splitext(train_file)
state = pcfg.EM(iter_num=iter_num)
with open(name + '.pcfg', 'w') as f:
for (A, B, C) in cfg.binary_rules:
f.writelines(A + ' -> ' + B + ' ' + C + ' ' +
str(state.get((A, B, C))) + '\n')
for (A, w) in cfg.unary_rules:
f.writelines(A + ' -> ' + w + ' ' + str(state.get((A, w))) + '\n')
with open(name + '.gen', 'w') as f:
for i in range(2000):
f.writelines(pcfg.gen_sentence('S') + '\n')
def __init__(self):
cfg = CFG('../../cfg/farmersmarket.yaml')
self._db = DB(cfg.data['db']['type'],
cfg.data['db']['host'],
cfg.data['db']['user'],
cfg.data['db']['pass'],
cfg.data['db']['dbname'])
self._db.connect()
# Set log file
self._log = LOG('../../log/api.log')
self._log.logger.info('Opening connection to the database')
def test_repr(self):
cfg = CFG([Variable("I"), Variable("J")],
[Terminal("a"), Terminal("b")],
[CFGRule(Variable("I"),
[Terminal("a"), Variable("I")])], Variable("I"))
r = str(cfg)
self.assertIn(str(Variable("I")), r)
self.assertIn(str(Variable("J")), r)
self.assertIn(str(Terminal("a")), r)
self.assertIn(str(Terminal("b")), r)
self.assertIn("Variables", r)
self.assertIn("Terminals", r)
self.assertIn("Productions", r)
self.assertIn("Start", r)
self.assertEqual(r.count("->"), 1)
self.assertEqual(r.count("\n"), 4)
def __init__(self, argv):
cfg = CFG('../cfg/farmersmarket.yaml')
# Open connection database
self._db = DB(cfg.data['db']['type'], cfg.data['db']['host'],
cfg.data['db']['user'], cfg.data['db']['pass'],
cfg.data['db']['dbname'])
self._db.connect()
# Set CSV file path
self._file = argv[1]
# Set log file
self._log = LOG('../log/export-CSV2DB.log')
self._log.logger.info('Opening connection to the database')
def main(path=None):
if len(sys.argv) != 2:
print "Usage: python " + sys.argv[0] + " <config.json>"
print "Try with one of the JSON files from the ROP collection."
return -1
if path is None:
path = sys.argv[1]
config = Config.from_json(path)
out_dir = "output/"
if not os.path.exists(out_dir):
os.mkdir(out_dir)
try:
emu = Emulator.emu_config(config)
except MemoryError as me:
print me.message
return -1
# emu.add_function_hook(lambda e: e.emu_stop(), address=0x401030)
# from syscall_emu import virtualalloc_emu
# emu.add_function_hook(virtualalloc_emu, name='VirtualAlloc')
# from unicorn import UC_HOOK_MEM_VALID, UC_HOOK_CODE
# from types import MethodType
# def new_hook_mem(self, uc, type, address, size, value, user_data):
# print 'NEW HOOOOOOOK'
# def aa(self, uc, address, size, user_data):
# print 'ciao'
# emu.new_hook = MethodType(new_hook_mem, emu)
# emu.emu.hook_add(UC_HOOK_MEM_VALID, emu.new_hook)
# emu.add_unicorn_hook(UC_HOOK_MEM_VALID, new_hook_mem)
# emu.add_unicorn_hook(UC_HOOK_CODE, aa)
if path.endswith("CVE-2018-4990-POC.json"):
from unicorn.x86_const import UC_X86_REG_EDI
emu.add_pp_hook(0x64560028, 0x64145b62,
lambda e: e.emu.reg_write(UC_X86_REG_EDI, 1))
emu.emu_start()
emu.output.branch_handler.compute_eflags_setter()
spgs = [emu.output.sequence_spg]
seqs = [emu.output.gadget_sequence]
def explore_branches_from(first_branch, graphs):
while first_branch.bstat.get_status(
) != BranchStatus.STATUS.FULLY_EXPLORED:
for fb in first_branch.bstat.get_internal_branches():
explore_branches_from(fb, graphs)
if first_branch.bstat.get_status(
) == BranchStatus.STATUS.NOT_EXPLORED:
# force path
emu.restore_from(first_branch.flag_loading.checkpoint,
first_branch.flag_loading.get_eflag_bitmask())
emu.output.branch_handler.compute_eflags_setter()
graphs.append(emu.output.mm_esp_graph)
spgs.append(emu.output.sequence_spg)
seqs.append(emu.output.gadget_sequence)
def explore_all_branches():
graphs = [emu.output.mm_esp_graph]
if emu.output.branch_handler.possible_branch_gadget:
explore_branches_from(
emu.output.branch_handler.possible_branch_gadget.itervalues().
next(), graphs)
return graphs
cfgs = explore_all_branches()
combination = CFG.combine_CFGs(cfgs)
combination.render_graph()
combination.render_empty_graph()
with open(out_dir + 'MP-ip-trace.txt', 'w') as f:
brand_new = []
for seq in seqs:
brand_new.append([hex(a).strip('L') for a, _ in seq])
f.write(str(brand_new).replace("'", ""))
with open(out_dir + 'MP-sp-trace.txt', 'w') as f:
brand_new = []
for spg in spgs:
brand_new.append([(hex(a).rstrip('L'), hex(b).rstrip('L'))
for a, b in spg])
f.write(str(brand_new).replace("'", ""))
from IPBasedCFG import IPBasedMPCFG
ipCFG = IPBasedMPCFG()
ipCFG.add_nodes(seqs)
ipCFG.add_edges(seqs)
ipCFG.render()
print emu.output.called_syscall_sequence
# G = emu.output.mm_esp_graph
# emu.output.mm_esp_graph.render_graph()
#
# b = emu.output.branch_handler.possible_branch_gadget.itervalues().next()
# emu.restore_from(b.flag_loading.checkpoint, b.flag_loading.get_eflag_bitmask())
#
# emu.output.mm_esp_graph.render_graph()
# emu.output.mm_esp_graph.combine_with(G)
# emu.output.mm_esp_graph.render_graph()
# dis32 = Cs(CS_ARCH_X86, CS_MODE_32)
# dis32.detail = True
#
# def get_eflag_name(eflag):
# if eflag == X86_EFLAGS_UNDEFINED_OF:
# return "UNDEF_OF"
# elif eflag == X86_EFLAGS_UNDEFINED_SF:
# return "UNDEF_SF"
# elif eflag == X86_EFLAGS_UNDEFINED_ZF:
# return "UNDEF_ZF"
# elif eflag == X86_EFLAGS_MODIFY_AF:
# return "MOD_AF"
# elif eflag == X86_EFLAGS_UNDEFINED_PF:
# return "UNDEF_PF"
# elif eflag == X86_EFLAGS_MODIFY_CF:
# return "MOD_CF"
# elif eflag == X86_EFLAGS_MODIFY_SF:
# return "MOD_SF"
# elif eflag == X86_EFLAGS_MODIFY_ZF:
# return "MOD_ZF"
# elif eflag == X86_EFLAGS_UNDEFINED_AF:
# return "UNDEF_AF"
# elif eflag == X86_EFLAGS_MODIFY_PF:
# return "MOD_PF"
# elif eflag == X86_EFLAGS_UNDEFINED_CF:
# return "UNDEF_CF"
# elif eflag == X86_EFLAGS_MODIFY_OF:
# return "MOD_OF"
# elif eflag == X86_EFLAGS_RESET_OF:
# return "RESET_OF"
# elif eflag == X86_EFLAGS_RESET_CF:
# return "RESET_CF"
# elif eflag == X86_EFLAGS_RESET_DF:
# return "RESET_DF"
# elif eflag == X86_EFLAGS_RESET_IF:
# return "RESET_IF"
# elif eflag == X86_EFLAGS_TEST_OF:
# return "TEST_OF"
# elif eflag == X86_EFLAGS_TEST_SF:
# return "TEST_SF"
# elif eflag == X86_EFLAGS_TEST_ZF:
# return "TEST_ZF"
# elif eflag == X86_EFLAGS_TEST_PF:
# return "TEST_PF"
# elif eflag == X86_EFLAGS_TEST_CF:
# return "TEST_CF"
# elif eflag == X86_EFLAGS_RESET_SF:
# return "RESET_SF"
# elif eflag == X86_EFLAGS_RESET_AF:
# return "RESET_AF"
# elif eflag == X86_EFLAGS_RESET_TF:
# return "RESET_TF"
# elif eflag == X86_EFLAGS_RESET_NT:
# return "RESET_NT"
# elif eflag == X86_EFLAGS_PRIOR_OF:
# return "PRIOR_OF"
# elif eflag == X86_EFLAGS_PRIOR_SF:
# return "PRIOR_SF"
# elif eflag == X86_EFLAGS_PRIOR_ZF:
# return "PRIOR_ZF"
# elif eflag == X86_EFLAGS_PRIOR_AF:
# return "PRIOR_AF"
# elif eflag == X86_EFLAGS_PRIOR_PF:
# return "PRIOR_PF"
# elif eflag == X86_EFLAGS_PRIOR_CF:
# return "PRIOR_CF"
# elif eflag == X86_EFLAGS_PRIOR_TF:
# return "PRIOR_TF"
# elif eflag == X86_EFLAGS_PRIOR_IF:
# return "PRIOR_IF"
# elif eflag == X86_EFLAGS_PRIOR_DF:
# return "PRIOR_DF"
# elif eflag == X86_EFLAGS_TEST_NT:
# return "TEST_NT"
# elif eflag == X86_EFLAGS_TEST_DF:
# return "TEST_DF"
# elif eflag == X86_EFLAGS_RESET_PF:
# return "RESET_PF"
# elif eflag == X86_EFLAGS_PRIOR_NT:
# return "PRIOR_NT"
# elif eflag == X86_EFLAGS_MODIFY_TF:
# return "MOD_TF"
# elif eflag == X86_EFLAGS_MODIFY_IF:
# return "MOD_IF"
# elif eflag == X86_EFLAGS_MODIFY_DF:
# return "MOD_DF"
# elif eflag == X86_EFLAGS_MODIFY_NT:
# return "MOD_NT"
# elif eflag == X86_EFLAGS_MODIFY_RF:
# return "MOD_RF"
# elif eflag == X86_EFLAGS_SET_CF:
# return "SET_CF"
# elif eflag == X86_EFLAGS_SET_DF:
# return "SET_DF"
# elif eflag == X86_EFLAGS_SET_IF:
# return "SET_IF"
# else:
# return None
#
# def get_up_flags(eflags):
# updated_flags = []
# for i in range(0, 46):
# if eflags & (1 << i):
# updated_flags.append(get_eflag_name(1 << i))
# print("\tEFLAGS: %s" % (','.join(p for p in updated_flags)))
#
# # For every gadget that had used an eflag related instruction there are possible multiple traces if that gadget is
# # used in multiple branching instructions
# flag_insn = False
# infos = []
# eflags_g_uses = {}
# for fl, traces in emu.branch_handler.traces.items():
# for trace in traces:
# if fl.g_addr in eflags_g_uses:
# eflags_g_uses[fl.g_addr] += 1
# else:
# eflags_g_uses[fl.g_addr] = 1
# for g_addr in trace:
# if flag_insn:
# flag_insn = False
# break
# gadget_bytes = emu.gadget_map[g_addr].rop_bytes
# for insn in dis32.disasm(gadget_bytes, g_addr):
# if insn.eflags:
# infos.append((g_addr, insn.address, insn.eflags))
# fl.eflag_bitmask = insn.eflags
# flag_insn = True
# break
#
# print "BRANCH INFOS:\n"
#
# print "Flags set up:\n"
# for g_addr, insn_addr, eflags in infos:
# print emu.gadget_map[g_addr]
# get_up_flags(eflags)
# print
#
# print 'Eflags loading gadgets:\n'
# for g_addr, uses in eflags_g_uses.items():
# print 'Used {} times'.format(uses)
# print emu.gadget_map[g_addr]
print "Branching actions:"
for sp, g_addr in emu.output.branch_handler.possible_branch_gadget.keys():
print '\nGADGET:'
print 'sp value: 0x{:x}'.format(sp)
print emu.output.gadget_map[g_addr]
print 'STATUS DEBUG {}'.format(
emu.output.branch_handler.possible_branch_gadget[
sp, g_addr].bstat.get_status())
print 'INTERNAL BRANCH STATUSES: '
for branch in emu.output.branch_handler.possible_branch_gadget[
sp, g_addr].bstat.get_internal_branches():
print '0x{:x}, {}'.format(branch.bstat.sp,
branch.bstat.get_status())
tg_map = barf_classify(emu.output.gadget_map, True)
print tg_map
return 0
def train_DQN():
#init confirgations
cfg = CFG()
best_time_step = 0.
#DQN brain
dqn = DQNBrain(cfg)
if cfg.use_cuda:
dqn = dqn.cuda()
#game start
flappyBird = game.GameState()
#set optimizer
optimizer = torch.optim.RMSprop(dqn.parameters(), lr=cfg.lr)
ceriterion = nn.MSELoss()
#init replay memory by random action
for i in range(cfg.observations):
action = dqn.get_action_randomly()
o, r, terminal = flappyBird.frame_step(action)
o = preprocess(o)
dqn.store_transition(o, action, r, terminal)
for episode in range(cfg.max_episode):
total_value = 0
while True:
optimizer.zero_grad()
if random.random() <= cfg.epsilon:
action = dqn.get_action_randomly()
else:
action = dqn.get_action_optim()
o_next, r, terminal = flappyBird.frame_step(action)
total_value += cfg.gamma*total_value + r
o_next = preprocess(o_next)
#update replay memory
dqn.store_transition(o_next, action, r, terminal)
dqn.increase_step()
#train dqn brain model by one batch
#step 1: sample training data from replay memory
minibatch = random.sample(dqn.replayMemory, cfg.batch_size)
state_batch = np.array([data[0] for data in minibatch])
action_batch = np.array([data[1] for data in minibatch])
reward_batch = np.array([data[2] for data in minibatch])
next_state_batch = np.array([data[3] for data in minibatch])
state_batch_var = Variable(torch.from_numpy(state_batch))
next_state_batch_var = Variable(torch.from_numpy(next_state_batch))
if cfg.use_cuda:
state_batch_var = state_batch_var.cuda()
next_state_batch_var = next_state_batch_var.cuda()
#step 2: get label y
q_value = dqn.forward(state_batch_var)
q_value_next = dqn.forward(next_state_batch_var)
y_batch = reward_batch.astype(np.float32)
max_q, _ = torch.max(q_value_next, dim=1)
for i in range(cfg.batch_size):
if not minibatch[i][4]: #terminal
y_batch[i] = y_batch[i]*cfg.gamma + max_q.data[i]
y_batch = Variable(torch.from_numpy(y_batch))
action_batch_var = Variable(torch.from_numpy(action_batch))#predict action
if cfg.use_cuda:
y_batch = y_batch.cuda()
action_batch_var = action_batch_var.cuda()
q_value = torch.sum(torch.mul(action_batch_var, q_value), dim=1)#predict value
#step 3: bp to update model
loss = ceriterion(q_value, y_batch)
loss.backward()
optimizer.step()
#end episode when bird's dead
if terminal:
dqn.time_step = 0
break
#update epsilon
if dqn.epsilon > cfg.final_e:
delta = (cfg.init_e - cfg.final_e)/cfg.exploration
dqn.epsilon -= delta
#test dqn per 100 episode
if episode % 100 == 0:
ave_step = test_DQN(dqn, episode)
from Automaton import Automaton, Log
from CFG import CFG
if __name__ == "__main__":
Log.enable = False
Log.enable_print = False
# automat = Automaton.buildFromFile('data/automat_input.in')
# print(automat.getRegex())
cfg = CFG.buildFromFile('data/cfg_input.in')
print(cfg.getAllWords(4))
'''
a)
S A B
a b x y
S
A > xx<A>y | xxy
B > a<B>b | ab
'''
'''
b)
S B
a c b
S
S > aa<B>b | aa<S>b
B > b<B>ccc | bccc
'''
'''
def __init__(self, filename, func='main'): elf = ELF(filename) addr = elf.sym[func] self.cfg = CFG(elf, addr) self.paths = [SymState(self.cfg, elf)]
def test_creation(self):
cfg = CFG([], [], [], Variable(""))
self.assertIsInstance(cfg, CFG)
def __init__(self, grammarFile):
self.tree = []
cfg = CFG(grammarFile)
self.cfg = cfg.get_grammar()
dst, src = op.operands
op_1 = state.read_value(dst)
op_2 = state.read_value(src)
if isinstance(op_1, Immediate) and isinstance(op_2, Immediate):
val = Immediate(op_1.value & op_2.value)
else:
val = Unknown()
new_state.store_value(dst, val)
return new_state
if __name__ == '__main__':
import sys
if len(sys.argv) != 2:
print('Usage: {} <file>'.format(sys.argv[0]))
sys.exit()
e = ELF(sys.argv[1])
main_addr = e.symbols['main']
cfg = CFG(e, main_addr)
start = MachineState()
start.regs[X86_REG_RSP] = StackPointer(0)
vars = ConstantAnalysis(cfg, entry_state=start)
for op_addr in sorted(cfg.ops):
op = cfg.ops[op_addr]
print('{:120s} -- {}'.format(vars.before_states[op], op_str(op)))
def main():
parser = argparse.ArgumentParser(
description='MFS and CFG file manipulation utility.',
formatter_class=argparse.RawDescriptionHelpFormatter,
epilog="""
The default output is to stdout.
Either one of --mfs or --cfg must be specified to indicate on which \
type of file to work (MFS or CFG).
You can specify one of the mutually exclusive actions : \
--dump --zip, --extract, --add, --remove.
For the --extract, --add, --remove actions, if --mfs is specified, \
then --file-id is required, if --cfg is specified, then --file-path is required.
When adding a file to a CFG file, the --mode, --opt, --uid and --gid options can be added.
The --mode option needs to be a string in the form 'dAEIrwxrwxrwx' where \
unused bits can be either a space or a dash, like --mode ' rwx---rwx' for example.
The --opt option needs to be a string in the form '?!MF' where unused bits can be \
either a space or a dash.
When adding a directory, both the file path needs to end with a '/' character and the --mode needs to start with 'd'.
""")
parser.add_argument("-o",
"--output",
dest="output",
default='-',
help="Output file to write",
metavar="FILE")
parser.add_argument("-i",
"--file-id",
dest="file_id",
type=int,
help="ID of the file to manipulate in the MFS file",
metavar="ID")
parser.add_argument("-f",
"--file-path",
dest="file_path",
help="Path of the file to manipulate in the CFG file",
metavar="PATH")
parser.add_argument("--mode",
dest="mode",
default="---rwxrwxrwx",
help="Mode for file being added to CFG",
metavar="MODE")
parser.add_argument("--opt",
dest="opt",
default="----",
help="Deplyoment option for file being added to CFG",
metavar="OPT")
parser.add_argument("--uid",
dest="uid",
default=0,
type=int,
help="User ID for file being added to CFG",
metavar="UID")
parser.add_argument("--gid",
dest="gid",
default=0,
type=int,
help="Group ID for file being added to CFG",
metavar="GID")
parser.add_argument("--recursive",
dest="recursive",
action="store_true",
help="Recursive deletion for a file path in CFG")
parser.add_argument("--alignment",
dest="alignment",
type=int,
default=0,
help="Alignment type for CFG files. (default: 0).\n"
"0 : packed.\n"
"1 : align all files on chunk start.\n"
"2 : align end of files on end of chunk.")
parser.add_argument(
"--deoptimize",
dest="optimize",
action="store_false",
help="De-optimize chain sequences when adding a file to MFS.")
group = parser.add_mutually_exclusive_group(required=True)
group.add_argument("-m",
"--mfs",
dest="mfs",
type=argparse.FileType('rb'),
help="MFS file to read from",
metavar="FILE")
group.add_argument("-c",
"--cfg",
dest="cfg",
type=argparse.FileType('rb'),
help="CFG file to read from",
metavar="FILE")
group = parser.add_mutually_exclusive_group(required=True)
group.add_argument(
"-d",
"--dump",
dest='dump',
action="store_true",
help="Dump information about the MFS file, or the CFG file")
group.add_argument("-z",
"--zip",
dest='zip',
action="store_true",
help="Store the MFS contents to a ZIP file")
group.add_argument(
"-x",
"--extract",
dest='extract',
action="store_true",
help="Extract a file from the MFS file, or a file from the CFG file")
group.add_argument(
"-a",
"--add",
dest='add',
type=argparse.FileType('rb'),
help="Add a file to the MFS file or a file to the CFG file",
metavar="FILENAME")
group.add_argument(
"-r",
"--remove",
dest='remove',
action="store_true",
help="Remove a file from the MFS file, or a file from the CFG file")
args = parser.parse_args()
if (args.add or args.remove
or args.extract) and (args.cfg and args.file_path is None):
parser.error(
"--add/--remove/--extract on a --cfg file requires the --file-path option"
)
if (args.add or args.remove
or args.extract) and (args.mfs and args.file_id is None):
parser.error(
"--add/--remove/--extract on a --mfs file requires the --file-id option"
)
if args.mfs is not None:
data = args.mfs.read()
mfs = MFS(data)
if args.dump:
with argparse.FileType("wb")(args.output) as f:
f.write("%s" % mfs)
elif args.extract:
file = mfs.getSystemVolume().getFile(args.file_id)
if file:
with argparse.FileType("wb")(args.output) as f:
f.write(file.data)
else:
print "File ID %d does not exist in the MFS System Volume" % args.file_id
sys.exit(-1)
elif args.remove:
mfs.getSystemVolume().removeFile(args.file_id)
mfs.generate()
with argparse.FileType("wb")(args.output) as f:
f.write(mfs.data)
elif args.add:
file = mfs.getSystemVolume().getFile(args.file_id)
if file:
print "File ID %d already exists in the MFS System Volume" % args.file_id
sys.exit(-1)
data = args.add.read()
mfs.getSystemVolume().addFile(args.file_id, data, args.optimize)
mfs.generate()
with argparse.FileType("wb")(args.output) as f:
f.write(mfs.data)
elif args.zip:
z = zipfile.ZipFile(args.output, "w", zipfile.ZIP_STORED)
for id in xrange(mfs.getSystemVolume().numFiles):
file = mfs.getSystemVolume().getFile(id)
if file:
zi = zipfile.ZipInfo("file_%d.bin" % id)
zi.external_attr = (0644 << 16)
z.writestr(zi, file.data)
z.close()
else:
data = args.cfg.read()
cfg = CFG(data)
if args.dump:
with argparse.FileType("wb")(args.output) as f:
f.write("%s" % cfg)
cfg.generate(args.alignment)
#with argparse.FileType("wb")(args.output) as f: f.write(cfg.data)
assert cfg.data == data
elif args.zip:
z = zipfile.ZipFile(args.output, "w", zipfile.ZIP_STORED)
for file in cfg.files:
path = file.path
if file.isDirectory():
path += posixpath.sep
attr = (040755 << 16) | 0x30
else:
attr = (0644 << 16)
zi = zipfile.ZipInfo(path)
zi.external_attr = attr
z.writestr(zi, file.data)
z.close()
elif args.extract:
file = cfg.getFile(args.file_path)
if file is None:
print "File path '%s' does not exist in the CFG file" % args.file_path
sys.exit(-1)
with argparse.FileType("wb")(args.output) as f:
f.write(file.data)
elif args.remove:
res = cfg.removeFile(args.file_path, args.recursive)
if not res:
if cfg.getFile(args.file_path) is None:
print "File path '%s' does not exist in the CFG file" % args.file_path
else:
print "File path '%s' is a non-empty directory in the CFG file (use --recursive)" % args.file_path
sys.exit(-1)
cfg.generate(args.alignment)
with argparse.FileType("wb")(args.output) as f:
f.write(cfg.data)
elif args.add:
file = cfg.getFile(args.file_path)
if file:
print "File path '%s' already exists in the CFG file" % args.file_path
sys.exit(-1)
data = args.add.read()
mode = CFG.strToMode(args.mode)
opt = CFG.strToOpt(args.opt)
if args.file_path[-1] == '/':
assert mode & 0x1000 == 0x1000
else:
assert mode & 0x1000 == 0
if not cfg.addFile(args.file_path, data, mode, opt, args.uid,
args.gid):
print "Error adding file to path '%s' in the CFG file " \
"(parent doesn't exist or is not a directory?)" % args.file_path
sys.exit(-1)
cfg.generate(args.alignment)
with argparse.FileType("wb")(args.output) as f:
f.write(cfg.data)
