def GeneratePit(src, template): dom = None dom = pfp.parse(data_file=src, template_file=template) ppit("DateModel", dom) '''
def _test_parse_build( self, data, template, stdout=None, debug=False, predefines=False, verify=True, _stream=True, printf=True, ): if stdout is not None: fake_stdout = sys.stdout = six.StringIO() if _stream: data = six.StringIO(data) dom = pfp.parse(data, template, debug=debug, predefines=predefines, printf=printf) if stdout is not None: sys.stdout = sys.__stdout__ output = fake_stdout.getvalue() self.assertEqual(output, stdout) return dom
def GeneratePit(src, template): dom = None dom = pfp.parse(data_file=src, template_file=template) ppit("DateModel",dom) '''
def test_debug_prompt(self): return dom = pfp.parse( StringIO("aaaabbbbcccchello there\x00\x05abcdf"), """ Int3(); int a; int b; int c; string greeting; unsigned char length; char str[length]; """)
def test_file_size(self): input_ = six.StringIO("ABCDE") output_ = six.StringIO() sys.stdout = output_ dom = pfp.parse( input_, """ Printf("%d", FileSize()); """, ) sys.stdout = sys.__stdout__ self.assertEqual(output_.getvalue(), "5")
def ebootParse(file): template = """ uint align( uint v, uint a ) { return ( v + ( a - 1 ) ) & ~( a - 1 ); } void falign( uint a ) { FSeek( align( FTell(), a ) ); } void falignx( uint a ) { FSeek( align( FTell() + 1, a ) ); } typedef struct { string data; while( ReadByte() == 0 ) { falignx(4); } } str <optimize=false,read=read_str>; string read_str( str &o ) { return o.data; } FSeek( 0xe19cd8 ); str data[3000]; """ parsed_tlv = pfp.parse(template=template, data_file=file)
def test_debug_prompt(self): return dom = pfp.parse( StringIO("aaaabbbbcccchello there\x00\x05abcdf"), """ Int3(); int a; int b; int c; string greeting; unsigned char length; char str[length]; """, )
def test_changeset_with_bitfields(): template = """ BigEndian(); struct { char a:2; // 11 char b:2; // 00 char c:3; // 111 char d:1; // 0 uint e; } data; """ # 0xc3 = 0b11001110 data = "\xceeeee" dom = pfp.parse(template=template, data=data) orig_data = dom._pfp__build() assert orig_data == binary(data) dom.data.a = 0 changer = Changer(orig_data) with changer.change([dom.data.a]) as changed: assert changed == binary("\x0eeeee") # 0x0e = 0b00001110 assert changer.build() == binary(data) dom._pfp__snapshot() dom.data.a = 0 dom.data.d = 1 with changer.change([dom.data.a, dom.data.d]) as changed: assert changed == binary("\x0feeee") # 0x0f = 0b00001111 dom._pfp__snapshot() dom.data.b = 3 dom.data.c = 0 with changer.change([dom.data.b, dom.data.c]) as changed: assert changed == binary("\x31eeee") # 0x31 = 0b00110001 dom._pfp__snapshot() dom.data.e = 0x45454545 with changer.change([dom.data.e]) as changed: assert changed == binary("\x31EEEE") # 0x31 = 0b00110001 dom._pfp__restore_snapshot() assert changer.build() == binary("\x31eeee") # 0x31 = 0b00110001 dom._pfp__restore_snapshot() assert changer.build() == binary("\x0feeee") # 0x0f = 0b00001111 dom._pfp__restore_snapshot() assert changer.build() == binary(data)
def _test_parse_build(self, data, template, stdout=None, debug=False, predefines=False, verify=True): if stdout is not None: fake_stdout = sys.stdout = six.StringIO() dom = pfp.parse(six.StringIO(data), template, debug=debug, predefines=predefines) if stdout is not None: sys.stdout = sys.__stdout__ output = fake_stdout.getvalue() self.assertEqual(output, stdout) if verify: self.assertEqual(dom._pfp__build(), pfp.utils.binary(data)) return dom
def main(argv=None): """Main function for this script :argv: TODO :returns: TODO """ if argv is None: argv = sys.argv args = parse_args(argv) dom = pfp.parse( template_file=args.template, data=args.input, keep_successful=args.keep, ) print(dom._pfp__show(include_offset=args.show_offsets))
def _generate_dom(self, file_contents): """Generates the file's DOM Applies a template to a file in order to generate a DOM to use as an intelligent mutation seed. :param template: the 010 editor template :param file: the original file """ bytestream = io.BytesIO(file_contents) try: dom = pfp.parse( data = bytestream, template_file = self.template_file) return dom except Exception as e: print(e) return None
def test_fuzz_yield_fields(self): template = """ struct { struct { char a; char b; } nested; char c; } root; """ data = "abc" dom = pfp.parse(template=template, data=data) for at_once in [1, 2, 3]: for mutated, changed_fields in pfp.fuzz.mutate( dom, "basic", num=100, at_once=at_once, yield_changed=True ): self.assertEqual(len(changed_fields), at_once)
def test_fuzz_yield_fields_no_yield(self): template = """ struct { struct { char a; char b; } nested; char c; } root; """ data = "abc" dom = pfp.parse(template=template, data=data) for at_once in [0, 1, 2]: for mutated in pfp.fuzz.mutate( dom, "basic", num=100, at_once=at_once, yield_changed=False ): # make sure it does not return a tuple, as would be the case with # yield_changed = True self.assertFalse(isinstance(mutated, tuple))
def test_changeset(): template = """ struct { ushort a; ushort b; ushort c; ushort d; uint e; } data; """ data = "aabbccddeeee" dom = pfp.parse(template=template, data=data) orig_data = dom._pfp__build() assert orig_data == binary(data) dom.data.a = 0x4141 dom.data.b = 0x4242 dom.data.c = 0x4343 dom.data.d = 0x4444 dom.data.e = 0x45454545 changer = Changer(orig_data) changer.push_changes([dom.data.a]) assert changer.build() == bytearray(b"AAbbccddeeee") changer.pop_changes() assert changer.build() == bytearray(binary(data)) changer.push_changes([dom.data.a, dom.data.d]) assert changer.build() == bytearray(b"AAbbccDDeeee") changer.push_changes([dom.data.b, dom.data.c]) assert changer.build() == bytearray(b"AABBCCDDeeee") changer.push_changes([dom.data.e]) assert changer.build() == bytearray(b"AABBCCDDEEEE") changer.pop_changes() assert changer.build() == bytearray(b"AABBCCDDeeee") changer.pop_changes() assert changer.build() == bytearray(b"AAbbccDDeeee") changer.pop_changes() assert changer.build() == bytearray(binary(data))
def pfp_parse(): vim.command("silent! set noeol") vim.command("silent! set binary") curr_file = vim.current.buffer.name if curr_file.startswith("__PFP_HEX__"): curr_file = vim.eval("b:pfp_orig_file") vim.command("tabnew") vim.command("setlocal nolist") vim.command("setlocal nospell") vim.command("setlocal nonumber") vim.command("setlocal noswapfile") vim.command("setlocal nobuflisted") vim.command("setlocal bufhidden=hide") vim.command("setlocal buftype=nofile") vim.command("let b:pfp_orig_file = '" + curr_file + "'") vim.command("set filetype=pfp_hex") vim.command("set syntax=pfp_hex") count = 0 count_str = "" while True: try: name = "__PFP_HEX__\\ " + curr_file.replace(" ", "\\ ") + count_str vim.command("file " + name) break except: count += 1 count_str = "\\ " + str(count) curr_winnr = vim.current.window.number if not os.path.exists(curr_file): err("could not locate file {}".format(curr_file)) return template_path = pfp_choose_template() if template_path is None: return if not os.path.exists(template_path): err("could not locate template {}".format(template_path)) return pfp_hex_dump_file(curr_file) dom = pfp.parse( data_file=curr_file, template_file=template_path, int3=False ) total_width = 0 for window in vim.windows: total_width += window.width create_scratch( dom._pfp__show(include_offset=True), width=int( total_width / 2 ), # yes, explicitly make it an int b/c of python3 fit_to_contents=False, scratch_name=name.replace("__PFP_HEX__", "__PFP__DOM__"), ) vim.command("set syntax=pfp_dom") vim.command("set cursorline") vim.command("let b:winnr = {}".format(curr_winnr)) # go to the top vim.command("normal! gg") vim.command("set filetype=pfp_dom") vim.command("set syntax=pfp_dom")
try: binfile = os.environ["FILE"] except: try: binfile = sys.argv[2] except: print "Missing file to parse" sys.exit(1) else: print "Usage: 010-to-r2.py [template.bt] ([file])" print "> .!python 010-to-r2.py JPEGTemplate.bt" sys.exit(1) # XXX pfp.parse show noisy messages dom = pfp.parse ( data_file = binfile, template_file = template ) def filterFlagName(x): x = x.replace("[","_") x = x.replace("]","_") return x rows = dom._pfp__show(include_offset=True) structName = "" for line in rows.split("\n"): isStruct = line.find("struct") != -1 line = line.strip() line = line.split("=")[0] if line[0] == "}": continue line = "0x" + line
def parseDRStoLists(abs_file_path, abs_bt_path): print('------ IN ------> ', abs_file_path) # READ DRS FILE dom = pfp.parse(data_file=abs_file_path, template_file=abs_bt_path) # BASE LISTS list_VEC3_normals = [] list_VEC2_uvs = [] list_VEC3_positions = [] list_SCALAR_faceIndices = [] #LOOP OVER NODES for node in dom.Nodes: node_name = unpack_string(node.Name.Text._pfp__build()) print(node_name) # ATTRIBUTE NODE // POSITIONS - NORMALS - UVS - BBOX - MATERIALS if node_name == 'CDspMeshFile': print('CDspMeshFile!') #LOOP OVER SUBMESHES for submesh in node.MeshFile.meshes: # CHECK IF GOOD NODE try: print('A good submesh!') VertexCount = struct.unpack( 'i', submesh.VertexCount._pfp__build()) FaceCount = struct.unpack('i', submesh.FaceCount._pfp__build()) print('---------------') except: print('Not a good submesh!') VertexCount, FaceCount = None, None if VertexCount != None and FaceCount != None: # OFFSET FOR SUBMESHES offset = len(list_VEC3_positions) # LOOP OVER FACE STRUCTURE: for face in submesh.Faces: indices = struct.unpack('hhh', face._pfp__build()) for index in list(indices): list_SCALAR_faceIndices.append(index + offset) # LOOP OVER VERTEX STRUCTURE for vertex in submesh.Meshes[0].Vertex: #GET POS xyz = struct.unpack('fff', vertex.vertex._pfp__build()) list_VEC3_positions.append(xyz) # GET NORMALS nxyz = struct.unpack('fff', vertex.vertexNormal._pfp__build()) list_VEC3_normals.append(nxyz) # GET UVs uxy = struct.unpack('ff', vertex.vertexTexture._pfp__build()) list_VEC2_uvs.append(uxy) # GET JOINTS FROM NODE 'CSkSkeleton' # GET VERTEX WEIGHTS FROM NODE ## NODES LOOP END ########################################################## # Debug File Structure out_path = abs_file_path.replace(".drs", ".txt") print('----- OUT TEXT -----> ', out_path) with open(out_path, "w") as f: f.write(dom._pfp__show(include_offset=True)) f.close() return list_VEC3_positions, list_SCALAR_faceIndices, list_VEC3_normals, list_VEC2_uvs
def ebootParse(file): template = """ uint align( uint v, uint a ) { return ( v + ( a - 1 ) ) & ~( a - 1 ); } void falign( uint a ) { FSeek( align( FTell(), a ) ); } void falignx( uint a ) { FSeek( align( FTell() + 1, a ) ); } typedef struct { string data; while( ReadByte() == 0 ) { falignx(4); } } str <optimize=false,read=read_str>; string read_str( str &o ) { return o.data; } FSeek( 0xe19cd8 ); str data[3000]; """ parsed_tlv = pfp.parse(template=template, data_file=file) template = """ uint align( uint v, uint a ) { return ( v + ( a - 1 ) ) & ~( a - 1 ); } void falign( uint a ) { FSeek( align( FTell(), a ) ); } void falignx( uint a ) { FSeek( align( FTell() + 1, a ) ); } typedef struct { string data; while( ReadByte() == 0 ) { falignx(4); } } str <optimize=false,read=read_str>; string read_str( str &o ) { return o.data; } FSeek( 0xe19cd8 ); str data[3000]; """ domD = pfp.parse(template=template, data_file="patch/USRDIR/EBOOT_DECR.BIN") print(domD._pfp__show(include_offset=True))
template = sys.argv[1] try: binfile = os.environ["FILE"] except: try: binfile = sys.argv[2] except: print "Missing file to parse" sys.exit(1) else: print "Usage: 010-to-rizin.py [template.bt] ([file])" print "> .!python 010-to-rizin.py JPEGTemplate.bt" sys.exit(1) # XXX pfp.parse show noisy messages dom = pfp.parse(data_file=binfile, template_file=template) def filterFlagName(x): x = x.replace("[", "_") x = x.replace("]", "_") return x rows = dom._pfp__show(include_offset=True) structName = "" for line in rows.split("\n"): isStruct = line.find("struct") != -1 line = line.strip() line = line.split("=")[0] if line[0] == "}":
#!/usr/bin/env python2 # -*- coding: utf-8 -*- import pfp, pfp.fuzz # import IPython # import sys # if __name__ == "__main__": # #file = sys.argv[1] # #print file file = "input/date" dom = pfp.parse(data_file=file, template_file="ELF.bt") # print dom._pfp__show() class IntegersOnly(pfp.fuzz.StratGroup): name = "ints_only" class IntStrat(pfp.fuzz.FieldStrat): klass = pfp.fields.IntBase choices = [xrange(10)] def filter_fields(self, fields): return filter(lambda x: isinstance(x, pfp.fields.IntBase), fields) counter = 0
class DataTypes(QMainWindow): def __init__(self, main_window, company_name, software_name): QMainWindow.__init__(self) self.setWindowTitle("Data Types") self.setWindowFlags(Qt.CustomizeWindowHint | Qt.WindowTitleHint | Qt.WindowCloseButtonHint) self.main_window = main_window self.parsed_struct = None self.main_struct = None self.parsed_template = None self.template_path = None # Set up UI loader = QtUiTools.QUiLoader() this_dir = os.path.dirname(__file__) self.ui = loader.load(os.path.join(this_dir, 'data_types.ui'), None) self.setCentralWidget(self.ui) QMetaObject.connectSlotsByName(self) # Size constraints self.setMinimumSize(self.ui.minimumSize()) self.setMaximumSize(self.ui.maximumSize()) # Read settings self.settings = QSettings(company_name, software_name) self.restoreGeometry(self.settings.value("DataTypes/geometry")) self.view = None # Internal flags. self.date_changed_internally = False self.time_changed_internally = False def set_view(self, view): self.view = view def get_format_string(self): if self.ui.littleEndianCheckBox.isChecked(): format_string = '<' else: format_string = '>' if self.ui.eightBitRadioButton.isChecked(): format_string += 'b' printf_string = '%d' elif self.ui.sixteenBitRadioButton.isChecked(): format_string += 'h' printf_string = '%d' elif self.ui.thirtyTwoBitRadioButton.isChecked(): format_string += 'i' printf_string = '%d' elif self.ui.sixtyFourBitRadioButton.isChecked(): format_string += 'q' printf_string = '%d' if not self.ui.signedCheckBox.isChecked(): format_string = format_string.upper() if self.ui.singleRadioButton.isChecked(): format_string += 'f' printf_string = '%e' elif self.ui.doubleRadioButton.isChecked(): format_string += 'd' printf_string = '%e' return format_string, printf_string def set_hexEdit_bytes(self, bytes): text = ''.join(["%02X " % ord(x) for x in bytes]).strip() # self.ui.hexEdit.setText(text) self.ui.hexEdit.setPlainText(text) # If there is text in the hex data field, the change # button should be activated. if len(text) > 0: self.ui.changeButton.setEnabled(True) else: self.ui.changeButton.setEnabled(False) def date_and_time_to_bytes(self): qdate = self.ui.calendarWidget.selectedDate() qtime = self.ui.timeEdit.time() dt = datetime.datetime(qdate.year(), qdate.month(), qdate.day(), qtime.hour(), qtime.minute(), qtime.second()) number = int(time.mktime(dt.timetuple())) bytes = struct.pack("I", number) return bytes def set_bytes(self, bytes_or_view, is_hexEdit_changer=False): current_tab = self.ui.tabWidget.currentWidget() if current_tab == self.ui.tab_numbers: #Get the format string. format_string, printf_string = self.get_format_string() # Compute how many bytes are needed. size_needed = struct.calcsize(format_string) # Extract the correct number of bytes if the # input is a memoryview. if isinstance(bytes_or_view, memoryview): bytes_or_view = bytes_or_view[:size_needed].tobytes() # Try to parse a number. self.ui.numberEdit.setEnabled(True) if printf_string == '%d': self.ui.signedCheckBox.setEnabled(True) else: self.ui.signedCheckBox.setEnabled(False) self.ui.littleEndianCheckBox.setEnabled(True) number = None try: assert (size_needed == len(bytes_or_view)) number = struct.unpack(format_string, bytes_or_view)[0] except: self.ui.numberEdit.setText("n/a") self.ui.numberEdit.setEnabled(False) self.ui.signedCheckBox.setEnabled(False) self.ui.littleEndianCheckBox.setEnabled(False) if number is not None: self.ui.numberEdit.setText(printf_string % number) number_bytes = struct.pack(format_string, number) if not is_hexEdit_changer: self.set_hexEdit_bytes(number_bytes) elif current_tab == self.ui.tab_dates: size_needed = 4 if isinstance(bytes_or_view, memoryview): bytes_or_view = bytes_or_view[:size_needed].tobytes() # Try to parse a number. number = None try: assert (size_needed == len(bytes_or_view)) number = struct.unpack('I', bytes_or_view)[0] # Success. Enable calendar. self.ui.calendarWidget.setEnabled(True) self.ui.timeEdit.setEnabled(True) except: self.ui.numberEdit.setText("n/a") self.ui.numberEdit.setEnabled(False) self.ui.calendarWidget.setEnabled(False) self.ui.timeEdit.setEnabled(False) if number is not None: localtime = time.localtime(number) qdate = QDate(localtime.tm_year, localtime.tm_mon, localtime.tm_mday) qtime = QTime(localtime.tm_hour, localtime.tm_min, localtime.tm_sec) self.date_changed_internally = True self.time_changed_internally = True self.ui.calendarWidget.setSelectedDate(qdate) self.ui.timeEdit.setTime(qtime) if not is_hexEdit_changer: self.set_hexEdit_bytes(bytes_or_view) elif current_tab == self.ui.tab_construct: #parse data with self.main_struct.parse() and str() it to the text box if isinstance(bytes_or_view, memoryview): bytes_or_view = bytes_or_view.tobytes() try: if self.main_struct is not None: #Its useless to check the size, since there can be dynamic size self.parsed_struct = self.main_struct.parse(bytes_or_view) self._dump_container_to_list(self.parsed_struct) if not is_hexEdit_changer: self.set_hexEdit_bytes(bytes_or_view) except Exception, err: self.ui.lstConstruct.clear() raise sys.exc_info()[1], None, sys.exc_info()[2] elif current_tab == self.ui.tab_010_templates: #parse data with self.main_struct.parse() and str() it to the text box if isinstance(bytes_or_view, memoryview): bytes_or_view = bytes_or_view.tobytes() try: if self.template_path is not None: #Its useless to check the size, since there can be dynamic size self.parsed_template = pfp.parse( data=bytes_or_view, template_file=self.template_path) self._dump_parsed_010_object_to_list( OrderedDict(self.parsed_template._pfp__children_map)) if not is_hexEdit_changer: self.set_hexEdit_bytes(bytes_or_view) except Exception, err: self.ui.lst010Template.clear() raise sys.exc_info()[1], None, sys.exc_info()[2]
import os import sys import pfp datafileName = sys.argv[1].split("\\")[-1].split(".")[0] os.environ["CYGWIN"]="nodosfilewarning" mdl = pfp.parse(data_file=sys.argv[1], template_file="mdl.bt", cpp_path=".//cpp//bin//i686-nacl-cpp.exe") types = mdl._pfp__types._types_map def process(chunk): if not os.path.exists("./%s" % (datafileName)): os.mkdir("./%s" % (datafileName)) if chunk.tag == 'Mesh ': print 'extract : Mesh' i = 0 for submesh in chunk.meshs._pfp__children: if isinstance(submesh,types["MESH"]): i = i + 1 # submesh f = open("./%s/mesh-%d.obj" % (datafileName,i),'wb') for block in submesh.blocks: f.write("v %f %f %f\r\n" % (block.v1._pfp__value,block.v2._pfp__value,block.v3._pfp__value)) for block in submesh.blocks: f.write("vt %f %f\r\n" % (block.vt2._pfp__value,block.vt3._pfp__value)) for block in submesh.blocks: f.write("vn %f %f %f\r\n" % (block.vn1._pfp__value,block.vn2._pfp__value,block.vn3._pfp__value)) f.write("g submesh_%d\r\n" % i) for face in submesh.faces: indice1 = face.i1._pfp__value + 1 indice2 = face.i2._pfp__value + 1 indice3 = face.i3._pfp__value + 1
def main(): parser = argparse.ArgumentParser() parser.add_argument('--file', type=str, help='specify the input file', action='store') parser.add_argument('--dir', type=str, help='specify the input directory', action='store') parser.add_argument('-t', '--template_file', type=str, help='the template file to parse the files', action='store') args = parser.parse_args() assert os.path.isfile( args.template_file), "%s does not exist" % args.template_file ftype = os.path.splitext(os.path.basename(args.template_file))[0] failed = False if args.file: assert os.path.isfile(args.file), " %s does not exist" % args.file try: dom = pfp.parse(data_file=args.file, template_file=args.template_file) except (pfp.errors.PrematureEOF, pfp.errors.PfpError): print("[-] Err: %s cannot be parsed, ignoring it..." % args.file) failed = True exit(-1) dom_stream = dom._pfp__show(include_offset=True) print(type(dom_stream), len(dom_stream)) ctype_dic = dict() for item in dom_stream.split('\n'): if 'ctype' in item: ctype_name = '(chunk:type:cname,' + item.split('[')[-1].split( ']')[0] + ')' loc = int(item.split('ctype')[0].strip(), 16) ctype_dic[ctype_name] = loc print(ctype_dic) fsize = os.path.getsize(args.file) fdir, fn = os.path.split(args.file) outdir = fdir + '_parsed_structs' ofn = os.path.splitext(fn)[0] + '_' + str( fsize) + '.' + ftype.lower() + '_struct' outfile = os.path.join(outdir, ofn) if not os.path.exists(outdir): print('[+] Creating %s to store struct files.' % outdir) os.mkdir(outdir) # ofn = 'output_content' # ofn = args.file + '_struct' # fn = ofn + '_' + str(fsize) # ext = '.txt' # outfile = fn + ext with open(outfile, 'w') as outf: outf.write(dom_stream) print(" [+] Structure parsing completed.") if args.dir: assert os.path.isdir(args.dir), " %s does not exist" % args.dir data_dir, _ = os.path.split(os.path.join(args.dir, 'parsed_structs')) outdir = mk_struct_dir(data_dir) for root, dirs, files in os.walk(args.dir): if root == outdir: continue for fn in files: fpath = os.path.join(root, fn) print(' [+] Parsing ' + fpath + '...') if fn.endswith(ftype.upper()) or fn.endswith(ftype.lower()): fpath = os.path.join(root, fn) try: dom = pfp.parse(data_file=fpath, template_file=args.template_file) except (pfp.errors.PrematureEOF, pfp.errors.PfpError): print( "\t [-] Err: %s cannot be parsed, ignoring it..." % args.file) continue else: dom_stream = dom._pfp__show(include_offset=True) fsize = os.path.getsize(fpath) ofn = os.path.splitext(fn)[0] + '_' + str( fsize) + '.' + ftype.lower() + '_struct' outfile = os.path.join(outdir, ofn) with open(outfile, 'w') as outf: outf.write(dom_stream) else: print('[-] %s is not a %s file, ignoring it...' % (fpath, ftype)) failed = True continue print(" [+] Structure parsing completed", end="") if failed: print(", with some failure.") else: print(".")