def add_miscellaneous_fatures_section(self):
"""SITE
"""
serial_num = 0
for site in self.struct.iter_sites():
num_fragments = len(site.fragment_dict_list)
site_pdb = None
key_index = 0
for frag_dict in site.fragment_dict_list:
if site_pdb is None or key_index==4:
serial_num += 1
key_index = 0
site_pdb = PDB.SITE()
self.pdb_file.append(site_pdb)
site_pdb["serNum"] = serial_num
site_pdb["siteID"] = site.site_id
site_pdb["numRes"] = num_fragments
chain_id = "chainID%d" % (key_index)
res_name = "resName%d" % (key_index)
res_seq = "seq%d" % (key_index)
icode = "icode%d" % (key_index)
site_pdb[chain_id] = frag_dict["chain_id"]
site_pdb[res_name] = frag_dict["res_name"]
try:
site_pdb[res_seq], site_pdb[icode] = Structure.fragment_id_split(frag_dict["frag_id"])
except KeyError:
pass
def add_atom_records(self):
"""With a default model set, output all the ATOM and associated
records for the model.
"""
## atom records for standard groups
for chain in self.struct.iter_chains():
res = None
for res in chain.iter_standard_residues():
for atm in res.iter_all_atoms():
self.add_ATOM("ATOM", atm)
## chain termination record
if res:
ter_rec = PDB.TER()
self.pdb_file.append(ter_rec)
res_seq, icode = Structure.fragment_id_split(res.fragment_id)
ter_rec["serial"] = self.next_serial_number()
ter_rec["resName"] = res.res_name
ter_rec["chainID"] = res.chain_id
ter_rec["resSeq"] = res_seq
ter_rec["iCode"] = icode
## HETATM records for non-standard groups
for chain in self.struct.iter_chains():
for frag in chain.iter_non_standard_residues():
for atm in frag.iter_all_atoms():
self.add_ATOM("HETATM", atm)
def loadStructure(self, structureFiles, inputFileType):
"""
Load a structureFile into the system
"""
if type(structureFiles).__name__ == 'str':
temp = structureFiles
structureFiles = list()
structureFiles.append(temp)
if inputFileType == "Gromacs":
from GromacsTopology import *
for structure in structureFiles:
# Create GromacsStructure/readStructureFile
gs = GromacsStructure(grofile=structure)
gs.readStructureFile(structure)
# Create generic Structure/convertFromGromacsStructure
s = Structure()
s.convertFromGromacsStructure(gs)
self.structureMolecules.extend(s.molecules)
self.structures.append(s)
if inputFileType == "Amber":
# Import Amber stuff?
for structure in structureFiles:
# Create AmberStructure/readStructureFile
gs = AmberStructure(grofile=structure)
gs.readStructureFile(structure)
# Create generic Structure/convertFromGromacsStructure
s = Structure()
s.convertFromAmberStructure(gs)
self.structureMolecules.extend(s.molecules)
self.structures.append(s)
if inputFileType == "Desmond":
# Import Desmond
for structure in structureFiles:
# Create DesmondStructure/readStructureFile
gs = DesmondStructure(grofile=structure)
gs.readStructureFile(structure)
# Create generic Structure/convertFromGromacsStructure
s = Structure()
s.convertFromDesmondStructure(gs)
self.structureMolecules.extend(s.molecules)
self.structures.append(s)
def __init__(self, methanalysis, this):
self.memory = {}
self.method = methanalysis.get_method()
self.name = self.method.get_name()
self.lparams = []
self.variables = Instruction.Variable()
if self.name == '<init>':
self.name = self.method.get_class_name()[1:-1].split('/')[-1]
self.basic_blocks = methanalysis.basic_blocks.bb
code = self.method.get_code()
access = self.method.get_access()
self.access = [
flag for flag in Util.ACCESS_FLAGS_METHODS if flag & access
]
desc = self.method.get_descriptor()
self.type = Util.get_type(desc.split(')')[-1])
self.paramsType = Util.get_params_type(desc)
#Util.log('Searching loops in method %s' % self.name, 'debug')
"""
Findloop.loopCounter = 0
cfg = Findloop.CFG()
lsg = Findloop.LSG()
self.cfg = cfg
self.lsg = lsg
Findloop.AddEdges(cfg, self)
nbLoops = Findloop.FindLoops(cfg, lsg)
if nbLoops > 1:
Util.log('==================', 'debug')
Util.log('==== DUMP CFG ====', 'debug')
cfg.dump()
Util.log('==================', 'debug')
Findloop.ShowBlocks(lsg, 0)
Util.log('==== DUMP LSG ====', 'debug')
lsg.dump()
Util.log('==================', 'debug')
"""
lsg = None
blocks = []
exceptions = methanalysis.exceptions.exceptions
print "METHOD :", self.name
self.graph = Structure.ConstructCFG(lsg, self.basic_blocks, exceptions,
blocks)
print
#androguard.bytecode.method2png('graphs/%s#%s.png' % (self.method.get_class_name().split('/')[-1][:-1], self.name), methanalysis)
if code is None:
Util.log(
'CODE NONE : %s %s' %
(self.name, self.method.get_class_name()), 'debug')
else:
start = code.registers_size.get_value() - code.ins_size.get_value()
# 0x8 == Static case : this is not passed as a parameter
if 0x8 in self.access:
self._add_parameters(start)
else:
self.memory[start] = Register(this, start)
#print 'THIS ( %s ) : %d' % (self.name, start)
self._add_parameters(start + 1)
self.ins = []
self.cur = 0
def add_ATOM(self, rec_type, atm):
"""Adds ATOM/SIGATM/ANISOU/SIGUIJ/TER/HETATM
"""
self.atom_count += 1
if rec_type == "ATOM":
atom_rec = PDB.ATOM()
elif rec_type == "HETATM":
atom_rec = PDB.HETATM()
self.pdb_file.append(atom_rec)
serial = self.new_atom_serial(atm)
res_seq, icode = Structure.fragment_id_split(atm.fragment_id)
atom_rec["serial"] = serial
atom_rec["chainID"] = atm.chain_id
atom_rec["resName"] = atm.res_name
atom_rec["resSeq"] = res_seq
atom_rec["iCode"] = icode
atom_rec["name"] = atm.name
atom_rec["element"] = atm.element
atom_rec["altLoc"] = atm.alt_loc
if atm.position is not None:
if atm.position[0] is not None:
atom_rec["x"] = atm.position[0]
if atm.position[1] is not None:
atom_rec["y"] = atm.position[1]
if atm.position[2] is not None:
atom_rec["z"] = atm.position[2]
if atm.occupancy is not None:
atom_rec["occupancy"] = atm.occupancy
if atm.temp_factor is not None:
atom_rec["tempFactor"] = atm.temp_factor
if atm.column6768 is not None:
atom_rec["column6768"] = atm.column6768
if atm.charge is not None:
atom_rec["charge"] = atm.charge
def atom_common(arec1, arec2):
if arec1.has_key("serial"):
arec2["serial"] = arec1["serial"]
if arec1.has_key("chainID"):
arec2["chainID"] = arec1["chainID"]
if arec1.has_key("resName"):
arec2["resName"] = arec1["resName"]
if arec1.has_key("resSeq"):
arec2["resSeq"] = arec1["resSeq"]
if arec1.has_key("iCode"):
arec2["iCode"] = arec1["iCode"]
if arec1.has_key("name"):
arec2["name"] = arec1["name"]
if arec1.has_key("altLoc"):
arec2["altLoc"] = arec1["altLoc"]
if arec1.has_key("element"):
arec2["element"] = arec1["element"]
if arec1.has_key("charge"):
arec2["charge"] = arec1["charge"]
if atm.sig_position is not None:
sigatm_rec = PDB.SIGATM()
self.pdb_file.append(sigatm_rec)
atom_common(atom_rec, sigatm_rec)
if atm.sig_position[0] is not None:
sigatm_rec["sigX"] = atm.sig_position[0]
if atm.sig_position[1] is not None:
sigatm_rec["sigY"] = atm.sig_position[1]
if atm.sig_position[2] is not None:
sigatm_rec["sigZ"] = atm.sig_position[2]
if atm.sig_temp_factor is not None:
sigatm_rec["sigTempFactor"] = atm.sig_temp_factor
if atm.sig_occupancy is not None:
sigatm_rec["sigOccupancy"] = atm.sig_occupancy
if atm.U is not None:
anisou_rec = PDB.ANISOU()
self.pdb_file.append(anisou_rec)
atom_common(atom_rec, anisou_rec)
if atm.U[0,0] is not None:
anisou_rec["u[0][0]"] = int(round(atm.U[0,0] * 10000.0))
if atm.U[1,1] is not None:
anisou_rec["u[1][1]"] = int(round(atm.U[1,1] * 10000.0))
if atm.U[2,2] is not None:
anisou_rec["u[2][2]"] = int(round(atm.U[2,2] * 10000.0))
if atm.U[0,1] is not None:
anisou_rec["u[0][1]"] = int(round(atm.U[0,1] * 10000.0))
if atm.U[0,2] is not None:
anisou_rec["u[0][2]"] = int(round(atm.U[0,2] * 10000.0))
if atm.U[1,2] is not None:
anisou_rec["u[1][2]"] = int(round(atm.U[1,2] * 10000.0))
if atm.sig_U is not None:
siguij_rec = PDB.SIGUIJ()
self.pdb_file.append(siguij_rec)
atom_common(atom_rec, siguij_rec)
if atm.sig_U[0,0] is not None:
siguij_rec["u[0][0]"] = int(round(atm.sig_U[0,0] * 10000.0))
if atm.sig_U[1,1] is not None:
siguij_rec["u[1][1]"] = int(round(atm.sig_U[1,1] * 10000.0))
if atm.sig_U[2,2] is not None:
siguij_rec["u[2][2]"] = int(round(atm.sig_U[2,2] * 10000.0))
if atm.sig_U[0,1] is not None:
siguij_rec["u[0][1]"] = int(round(atm.sig_U[0,1] * 10000.0))
if atm.sig_U[0,2] is not None:
siguij_rec["u[0][2]"] = int(round(atm.sig_U[0,2] * 10000.0))
if atm.sig_U[1,2] is not None:
siguij_rec["u[1][2]"] = int(round(atm.sig_U[1,2] * 10000.0))
class MachOFile(BinaryAccessor):
def __init__(self, data):
self.data = data
self.valid = False
self.callbacks = []
self.symbols_by_name = {}
self.symbols_by_addr = {}
self.plt = {}
if not self.is_macho():
return
try:
self.tree = Structure(self.data)
self.header = self.tree.struct("Mach-O header", "header")
self.header.uint32_le("magic")
if (self.header.magic == 0xfeedface) or (self.header.magic == 0xfeedfacf):
self.header.uint32_le("cputype")
self.header.uint32_le("cpusubtype")
self.header.uint32_le("filetype")
self.header.uint32_le("cmds")
self.header.uint32_le("cmdsize")
self.header.uint32_le("flags")
if self.header.magic == 0xfeedfacf:
self.header.uint32_le("reserved")
self.bits = 64
else:
self.bits = 32
self.big_endian = False
elif (self.header.magic == 0xcefaedfe) or (self.header.magic == 0xcffaedfe):
self.header.uint32_be("cputype")
self.header.uint32_be("cpusubtype")
self.header.uint32_be("filetype")
self.header.uint32_be("cmds")
self.header.uint32_be("cmdsize")
self.header.uint32_be("flags")
if self.header.magic == 0xcffaedfe:
self.header.uint32_be("reserved")
self.bits = 64
else:
self.bits = 32
self.big_endian = True
self.symbol_table = None
self.dynamic_symbol_table = None
# Parse loader commands
self.commands = self.tree.array(self.header.cmds, "commands")
self.segments = []
self.sections = []
offset = self.header.getSize()
for i in xrange(0, self.header.cmds):
cmd = self.commands[i]
cmd.seek(offset)
if self.big_endian:
cmd.uint32_be("cmd")
cmd.uint32_be("size")
else:
cmd.uint32_le("cmd")
cmd.uint32_le("size")
if cmd.cmd == 1: # SEGMENT
cmd.bytes(16, "name")
if self.big_endian:
cmd.uint32_be("vmaddr")
cmd.uint32_be("vmsize")
cmd.uint32_be("fileoff")
cmd.uint32_be("filesize")
cmd.uint32_be("maxprot")
cmd.uint32_be("initprot")
cmd.uint32_be("nsects")
cmd.uint32_be("flags")
else:
cmd.uint32_le("vmaddr")
cmd.uint32_le("vmsize")
cmd.uint32_le("fileoff")
cmd.uint32_le("filesize")
cmd.uint32_le("maxprot")
cmd.uint32_le("initprot")
cmd.uint32_le("nsects")
cmd.uint32_le("flags")
if cmd.initprot != 0: # Ignore __PAGE_ZERO or anything like it
self.segments.append(cmd)
cmd.array(cmd.nsects, "sections")
for i in xrange(0, cmd.nsects):
section = cmd.sections[i]
section.bytes(16, "name")
section.bytes(16, "segment")
if self.big_endian:
section.uint32_be("addr")
section.uint32_be("size")
section.uint32_be("offset")
section.uint32_be("align")
section.uint32_be("reloff")
section.uint32_be("nreloc")
section.uint32_be("flags")
section.uint32_be("reserved1")
section.uint32_be("reserved2")
else:
section.uint32_le("addr")
section.uint32_le("size")
section.uint32_le("offset")
section.uint32_le("align")
section.uint32_le("reloff")
section.uint32_le("nreloc")
section.uint32_le("flags")
section.uint32_le("reserved1")
section.uint32_le("reserved2")
self.sections.append(section)
for i in xrange(0, cmd.nsects):
section = cmd.sections[i]
section.array(section.nreloc, "relocs")
for j in xrange(0, section.nreloc):
reloc = section.relocs[j]
reloc.seek(section.reloff + (j * 8))
if self.big_endian:
reloc.uint32_be("addr")
reloc.uint32_be("value")
else:
reloc.uint32_le("addr")
reloc.uint32_le("value")
elif cmd.cmd == 25: # SEGMENT_64
cmd.bytes(16, "name")
if self.big_endian:
cmd.uint64_be("vmaddr")
cmd.uint64_be("vmsize")
cmd.uint64_be("fileoff")
cmd.uint64_be("filesize")
cmd.uint32_be("maxprot")
cmd.uint32_be("initprot")
cmd.uint32_be("nsects")
cmd.uint32_be("flags")
else:
cmd.uint64_le("vmaddr")
cmd.uint64_le("vmsize")
cmd.uint64_le("fileoff")
cmd.uint64_le("filesize")
cmd.uint32_le("maxprot")
cmd.uint32_le("initprot")
cmd.uint32_le("nsects")
cmd.uint32_le("flags")
if cmd.initprot != 0: # Ignore __PAGE_ZERO or anything like it
self.segments.append(cmd)
cmd.array(cmd.nsects, "sections")
for i in xrange(0, cmd.nsects):
section = cmd.sections[i]
section.bytes(16, "name")
section.bytes(16, "segment")
if self.big_endian:
section.uint64_be("addr")
section.uint64_be("size")
section.uint32_be("offset")
section.uint32_be("align")
section.uint32_be("reloff")
section.uint32_be("nreloc")
section.uint32_be("flags")
section.uint32_be("reserved1")
section.uint32_be("reserved2")
section.uint32_be("reserved3")
else:
section.uint64_le("addr")
section.uint64_le("size")
section.uint32_le("offset")
section.uint32_le("align")
section.uint32_le("reloff")
section.uint32_le("nreloc")
section.uint32_le("flags")
section.uint32_le("reserved1")
section.uint32_le("reserved2")
section.uint32_le("reserved3")
self.sections.append(section)
for i in xrange(0, cmd.nsects):
section = cmd.sections[i]
section.array(section.nreloc, "relocs")
for j in xrange(0, section.nreloc):
reloc = section.relocs[j]
reloc.seek(section.reloff + (j * 8))
if self.big_endian:
reloc.uint32_be("addr")
reloc.uint32_be("value")
else:
reloc.uint32_le("addr")
reloc.uint32_le("value")
elif cmd.cmd == 5: # UNIX_THREAD
if self.header.cputype == 7: # x86
cmd.uint32_le("flavor")
cmd.uint32_le("count")
for reg in ["eax", "ebx", "ecx", "edx", "edi", "esi", "ebp", "esp", "ss", "eflags",
"eip", "cs", "ds", "es", "fs", "gs"]:
cmd.uint32_le(reg)
self.entry_addr = cmd.eip
elif self.header.cputype == 0x01000007: # x86_64
cmd.uint32_le("flavor")
cmd.uint32_le("count")
for reg in ["rax", "rbx", "rcx", "rdx", "rdi", "rsi", "rbp", "rsp", "r8", "r9",
"r10", "r11", "r12", "r13", "r14", "r15", "rip", "rflags", "cs", "fs", "gs"]:
cmd.uint64_le(reg)
self.entry_addr = cmd.rip
elif self.header.cputype == 18: # PPC32
cmd.uint32_be("flavor")
cmd.uint32_be("count")
for reg in ["srr0", "srr1"] + ["r%d" % i for i in xrange(0, 32)] + ["cr", "xer",
"lr", "ctr", "mq", "vrsave"]:
cmd.uint32_be(reg)
self.entry_addr = cmd.srr0
elif self.header.cputype == 0x01000012: # PPC64
cmd.uint32_be("flavor")
cmd.uint32_be("count")
for reg in ["srr0", "srr1"] + ["r%d" % i for i in xrange(0, 32)] + ["cr", "xer",
"lr", "ctr", "mq", "vrsave"]:
cmd.uint64_be(reg)
self.entry_addr = cmd.srr0
elif self.header.cputype == 12: # ARM
cmd.uint32_le("flavor")
cmd.uint32_le("count")
for reg in ["r%d" % i for i in xrange(0, 13)] + ["sp", "lr", "pc", "cpsr"]:
cmd.uint32_le(reg)
self.entry_addr = cmd.pc
elif cmd.cmd == 2: # SYMTAB
if self.big_endian:
cmd.uint32_be("symoff")
cmd.uint32_be("nsyms")
cmd.uint32_be("stroff")
cmd.uint32_be("strsize")
else:
cmd.uint32_le("symoff")
cmd.uint32_le("nsyms")
cmd.uint32_le("stroff")
cmd.uint32_le("strsize")
self.symbol_table = self.tree.array(cmd.nsyms, "symtab")
strings = self.data.read(cmd.stroff, cmd.strsize)
sym_offset = cmd.symoff
for j in xrange(0, cmd.nsyms):
entry = self.symbol_table[j]
entry.seek(sym_offset)
if self.big_endian:
entry.uint32_be("strx")
entry.uint8("type")
entry.uint8("sect")
entry.uint16_be("desc")
if self.bits == 32:
entry.uint32_be("value")
else:
entry.uint64_be("value")
else:
entry.uint32_le("strx")
entry.uint8("type")
entry.uint8("sect")
entry.uint16_le("desc")
if self.bits == 32:
entry.uint32_le("value")
else:
entry.uint64_le("value")
str_end = strings.find("\x00", entry.strx)
entry.name = strings[entry.strx:str_end]
if self.bits == 32:
sym_offset += 12
else:
sym_offset += 16
elif cmd.cmd == 11: # DYSYMTAB
if self.big_endian:
cmd.uint32_be("ilocalsym")
cmd.uint32_be("nlocalsym")
cmd.uint32_be("iextdefsym")
cmd.uint32_be("nextdefsym")
cmd.uint32_be("iundefsym")
cmd.uint32_be("nundefsym")
cmd.uint32_be("tocoff")
cmd.uint32_be("ntoc")
cmd.uint32_be("modtaboff")
cmd.uint32_be("nmodtab")
cmd.uint32_be("extrefsymoff")
cmd.uint32_be("nextrefsyms")
cmd.uint32_be("indirectsymoff")
cmd.uint32_be("nindirectsyms")
cmd.uint32_be("extreloff")
cmd.uint32_be("nextrel")
cmd.uint32_be("locreloff")
cmd.uint32_be("nlocrel")
else:
cmd.uint32_le("ilocalsym")
cmd.uint32_le("nlocalsym")
cmd.uint32_le("iextdefsym")
cmd.uint32_le("nextdefsym")
cmd.uint32_le("iundefsym")
cmd.uint32_le("nundefsym")
cmd.uint32_le("tocoff")
cmd.uint32_le("ntoc")
cmd.uint32_le("modtaboff")
cmd.uint32_le("nmodtab")
cmd.uint32_le("extrefsymoff")
cmd.uint32_le("nextrefsyms")
cmd.uint32_le("indirectsymoff")
cmd.uint32_le("nindirectsyms")
cmd.uint32_le("extreloff")
cmd.uint32_le("nextrel")
cmd.uint32_le("locreloff")
cmd.uint32_le("nlocrel")
elif (cmd.cmd & 0x7fffffff) == 0x22: # DYLD_INFO
self.dynamic_symbol_table = cmd
if self.big_endian:
cmd.uint32_be("rebaseoff")
cmd.uint32_be("rebasesize")
cmd.uint32_be("bindoff")
cmd.uint32_be("bindsize")
cmd.uint32_be("weakbindoff")
cmd.uint32_be("weakbindsize")
cmd.uint32_be("lazybindoff")
cmd.uint32_be("lazybindsize")
cmd.uint32_be("exportoff")
cmd.uint32_be("exportsize")
else:
cmd.uint32_le("rebaseoff")
cmd.uint32_le("rebasesize")
cmd.uint32_le("bindoff")
cmd.uint32_le("bindsize")
cmd.uint32_le("weakbindoff")
cmd.uint32_le("weakbindsize")
cmd.uint32_le("lazybindoff")
cmd.uint32_le("lazybindsize")
cmd.uint32_le("exportoff")
cmd.uint32_le("exportsize")
offset += cmd.size
# Add symbols from symbol table
if self.symbol_table:
for i in xrange(0, len(self.symbol_table)):
symbol = self.symbol_table[i]
# Only use symbols that are within a section
if ((symbol.type & 0xe) == 0xe) and (symbol.sect <= len(self.sections)):
self.create_symbol(symbol.value, symbol.name)
# If there is a DYLD_INFO section, parse it and add PLT entries
if self.dynamic_symbol_table:
self.parse_dynamic_tables([[self.dynamic_symbol_table.bindoff, self.dynamic_symbol_table.bindsize],
[self.dynamic_symbol_table.lazybindoff, self.dynamic_symbol_table.lazybindsize]])
self.tree.complete()
self.valid = True
except:
self.valid = False
raise
if self.valid:
self.data.add_callback(self)
def read_leb128(self, data, ofs):
value = 0
shift = 0
while ofs < len(data):
cur = ord(data[ofs])
ofs += 1
value |= (cur & 0x7f) << shift
shift += 7
if (cur & 0x80) == 0:
break
return value, ofs
def parse_dynamic_tables(self, tables):
# Interpret DYLD_INFO instructions (not documented by Apple)
# http://networkpx.blogspot.com/2009/09/about-lcdyldinfoonly-command.html
ordinal = 0
segment = 0
offset = 0
sym_type = 0
name = ""
for table in tables:
offset = table[0]
size = table[1]
opcodes = self.data.read(offset, size)
i = 0
while i < len(opcodes):
opcode = ord(opcodes[i])
i += 1
if (opcode >> 4) == 0:
continue
elif (opcode >> 4) == 1:
ordinal = opcode & 0xf
elif (opcode >> 4) == 2:
ordinal, i = self.read_leb128(opcodes, i)
elif (opcode >> 4) == 3:
ordinal = -(opcode & 0xf)
elif (opcode >> 4) == 4:
name = ""
while i < len(opcodes):
ch = opcodes[i]
i += 1
if ch == '\x00':
break
name += ch
elif (opcode >> 4) == 5:
sym_type = opcode & 0xf
elif (opcode >> 4) == 6:
addend, i = self.read_leb128(opcodes, i)
elif (opcode >> 4) == 7:
segment = opcode & 0xf
offset, i = self.read_leb128(opcodes, i)
elif (opcode >> 4) == 8:
rel, i = self.read_leb128(opcodes, i)
offset += rel
elif (opcode >> 4) >= 9:
if (sym_type == 1) and (segment <= len(self.segments)):
# Add pointer type entries to the PLT
addr = self.segments[segment - 1].vmaddr + offset
self.plt[addr] = name
self.create_symbol(addr, self.decorate_plt_name(name))
if self.bits == 32:
offset += 4
else:
offset += 8
if (opcode >> 4) == 10:
rel, i = self.read_leb128(opcodes, i)
offset += rel
elif (opcode >> 4) == 11:
offset += (opcode & 0xf) * 4
elif (opcode >> 4) == 12:
count, i = self.read_leb128(opcodes, i)
skip, i = self.read_leb128(opcodes, i)
def read(self, ofs, len):
result = ""
while len > 0:
cur = None
for i in self.segments:
if ((ofs >= i.vmaddr) and (ofs < (i.vmaddr + i.vmsize))) and (i.vmsize != 0):
cur = i
if cur == None:
break
prog_ofs = ofs - cur.vmaddr
mem_len = cur.vmsize - prog_ofs
file_len = cur.filesize - prog_ofs
if mem_len > len:
mem_len = len
if file_len > len:
file_len = len
if file_len <= 0:
result += "\x00" * mem_len
len -= mem_len
ofs += mem_len
continue
result += self.data.read(cur.fileoff + prog_ofs, file_len)
len -= file_len
ofs += file_len
return result
def next_valid_addr(self, ofs):
result = -1
for i in self.segments:
if (i.vmaddr >= ofs) and (i.vmsize != 0) and ((result == -1) or (i.vmaddr < result)):
result = i.vmaddr
return result
def get_modification(self, ofs, len):
result = []
while len > 0:
cur = None
for i in self.segments:
if ((ofs >= i.vmaddr) and (ofs < (i.vmaddr + i.vmsize))) and (i.vmsize != 0):
cur = i
if cur == None:
break
prog_ofs = ofs - cur.vmaddr
mem_len = cur.vmsize - prog_ofs
file_len = cur.filesize - prog_ofs
if mem_len > len:
mem_len = len
if file_len > len:
file_len = len
if file_len <= 0:
result += [DATA_ORIGINAL] * mem_len
len -= mem_len
ofs += mem_len
continue
result += self.data.get_modification(cur.fileoff + prog_ofs, file_len)
len -= file_len
ofs += file_len
return result
def write(self, ofs, data):
result = 0
while len(data) > 0:
cur = None
for i in self.segments:
if ((ofs >= i.vmaddr) and (ofs < (i.vmaddr + i.vmsize))) and (i.vmsize != 0):
cur = i
if cur == None:
break
prog_ofs = ofs - cur.vmaddr
mem_len = cur.vmsize - prog_ofs
file_len = cur.filesize - prog_ofs
if mem_len > len:
mem_len = len
if file_len > len:
file_len = len
if file_len <= 0:
break
result += self.data.write(cur.fileoff + prog_ofs, data[0:file_len])
data = data[file_len:]
ofs += file_len
return result
def insert(self, ofs, data):
return 0
def remove(self, ofs, size):
return 0
def notify_data_write(self, data, ofs, contents):
# Find sections that hold data backed by updated regions of the file
for i in self.segments:
if ((ofs + len(contents)) > i.fileoff) and (ofs < (i.fileoff + i.filesize)) and (i.vmsize != 0):
# This section has been updated, compute which region has been changed
from_start = ofs - i.fileoff
data_ofs = 0
length = len(contents)
if from_start < 0:
length += from_start
data_ofs -= from_start
from_start = 0
if (from_start + length) > i.filesize:
length = i.filesize - from_start
# Notify callbacks
if length > 0:
for cb in self.callbacks:
if hasattr(cb, "notify_data_write"):
cb.notify_data_write(self, i.vmaddr + from_start,
contents[data_ofs:(data_ofs + length)])
def save(self, filename):
self.data.save(filename)
def start(self):
result = None
for i in self.segments:
if ((result == None) or (i.vmaddr < result)) and (i.vmsize != 0):
result = i.vmaddr
return result
def entry(self):
if not hasattr(self, "entry_addr"):
return self.start()
return self.entry_addr
def __len__(self):
max = None
for i in self.segments:
if ((max == None) or ((i.vmaddr + i.vmsize) > max)) and (i.vmsize != 0):
max = i.vmaddr + i.vmsize
return max - self.start()
def is_macho(self):
if self.data.read(0, 4) == "\xfe\xed\xfa\xce":
return True
if self.data.read(0, 4) == "\xfe\xed\xfa\xcf":
return True
if self.data.read(0, 4) == "\xce\xfa\xed\xfe":
return True
if self.data.read(0, 4) == "\xcf\xfa\xed\xfe":
return True
return False
def architecture(self):
if self.header.cputype == 7:
return "x86"
if self.header.cputype == 0x01000007:
return "x86_64"
if self.header.cputype == 12:
return "arm"
if self.header.cputype == 18:
return "ppc"
if self.header.cputype == 0x01000012:
return "ppc"
return None
def decorate_plt_name(self, name):
return name + "@PLT"
def create_symbol(self, addr, name):
self.symbols_by_name[name] = addr
self.symbols_by_addr[addr] = name
def delete_symbol(self, addr, name):
if name in self.symbols_by_name:
del(self.symbols_by_name[name])
if addr in self.symbols_by_addr:
del(self.symbols_by_addr[addr])
def add_callback(self, cb):
self.callbacks.append(cb)
def remove_callback(self, cb):
self.callbacks.remove(cb)
def is_modified(self):
return self.data.is_modified()
def find(self, regex, addr):
while (addr < self.end()) and (addr != -1):
data = self.read(addr, 0xfffffffff)
match = regex.search(data)
if match != None:
return match.start() + addr
addr += len(data)
addr = self.next_valid_addr(addr)
return -1
def has_undo_actions(self):
return self.data.has_undo_actions()
def commit_undo(self, before_loc, after_loc):
self.data.commit_undo(before_loc, after_loc)
def undo(self):
self.data.undo()
def redo(self):
self.data.redo()
shape[start - 1:end],
mode='fill',
highlight_region=highlight_region,
bpprob=subbpprob,
bpprob_mode='both',
title=title,
bpwarning=False,
period=100,
first_base_pos=start,
peroid_color='#ff5722')
#cmd = manual_period(cmd, end-start+1, start, 100)
print(cmd, file=OUT)
start = end + 1
i += 1
OUT.close()
####################
### Frame-Shift Element
####################
subseq = sequence[13476 - 1:13503]
subshape = shape[13476 - 1:13503]
subdot = Structure.predict_structure(subseq)
cmd = Visual.Plot_RNAStructure_Shape(subseq,
subdot,
subshape,
mode='fill',
first_base_pos=13476)
print(cmd)
def remove_gap_columns(id2seq, refSeq=None, refStr=None, minGapRatio=1.0):
"""
Remove alignment columns with too many gaps
minGapRatio -- Remove columns with gap ratio greater than minGapRatio [0-1]
Return {
'id2seq': {id1:aligned_seq1, id2:aligned_seq2, ...},
'refSeq': refSeq, # if refSeq != None
'refStr': refStr # if refStr != None
}
"""
keys = list(id2seq.keys())
alignment_list = []
for key in keys:
alignment_list.append(id2seq[key].replace('~', '-').replace(
':', '-').replace('.', '-').upper().replace('U', 'T'))
align_columns = collect_columns(alignment_list)
Len = len(id2seq[keys[0]])
if refStr is None:
refBpmap = {}
else:
refBpmap = Structure.dot2bpmap(refStr)
columns_to_remove = []
for i in range(Len):
if i in columns_to_remove:
continue
#print( i, align_columns[i].count('-'), len(align_columns[i]) )
if align_columns[i].count('-') / len(align_columns[i]) >= minGapRatio:
columns_to_remove.append(i)
if i + 1 in refBpmap:
columns_to_remove.append(refBpmap[i + 1] - 1)
columns_to_remove.sort()
#print(columns_to_remove)
clean_id2seq = {}
for key in id2seq:
clean_id2seq[key] = ""
cur_seq = id2seq[key]
for i in range(Len):
if not General.bi_search(i, columns_to_remove):
clean_id2seq[key] += cur_seq[i]
return_value = {'id2seq': clean_id2seq}
if refSeq:
_refSeq = ""
for i in range(Len):
if not General.bi_search(i, columns_to_remove):
_refSeq += refSeq[i]
return_value['refSeq'] = _refSeq
if refStr:
_refStr = ""
for i in range(Len):
if not General.bi_search(i, columns_to_remove):
_refStr += refStr[i]
_Len = len(_refStr)
#_refStr = Structure.ct2dot(Structure.dot2ct(_refStr), _Len)
return_value['refStr'] = _refStr
return return_value
from matplotlib import pyplot as plt
from Structure import *
from JsonRead import *
from Material import *
# structure_js = readFile('hipoStructure.json')
structure_js = readFile('testStructure.json')
# structure_js = readFile('structure00.json')
material_models_js = readFile('material_models.json')
load_material_models(material_models_js)
structure = Structure(structure_js)
total_section_deformation = np.array([[0], [0]], dtype=np.float64)
total_section_force = np.array([[0], [0]], dtype=np.float64)
initial_section_k = np.array([[4.01601550e+06, -3.63797881e-11],
[-3.63797881e-11, 5.34565486e+04]])
# initial_section_k = np.array([[4.55056806e+06, -2.18278728e-11], [-2.18278728e-11, 9.76568675e+04]])
# initial_section_k = np.array([[5.612190356482407e+06, 0], [0, 0.119102335060676e+06]])
def conditionCheck(mat, value):
max_abs_val = abs(max(mat.min(), mat.max(), key=abs))
if max_abs_val > value:
return True
else:
return False
# We will instantiate a number of Structure() objects to construct the ensemble
ensemble = []
for i in range(nclusters+1):
print
print '#### STRUCTURE %d ####'%i
# no information from QM --> lam = 0.0
# QM + exp --> lam = 1.0
## s = Structure('gens-pdb-kcenters-dih-1.8/Gen%d.pdb'%i, args.lam*energies[i], expdata_filename, use_log_normal_distances=False)
model_distances = loadtxt('mean_distances/mean_distances_state%d.dat'%i)*10.0 # convert to A
# model_chemicalshift = loadtxt('chemical_shift/cs/cs_%d.txt'%i)
# s = Structure('Gens/Gens%d.pdb'%i, args.lam*energies[i], expdata_filename_noe, expdata_filename_J, expdata_filename_cs, use_log_normal_distances=False, dloggamma=np.log(1.01), gamma_min=0.2, gamma_max=5.0)
s = Structure('Gens/Gens%d.pdb'%i, args.lam*energies[i], expdata_filename_noe=expdata_filename_noe, expdata_filename_cs=expdata_filename_cs, expdata_filename_PF=expdata_filename_PF, use_log_normal_distances=False, dloggamma=np.log(1.01), gamma_min=0.2, gamma_max=5.0) #GYH
# s = Structure('Gens/Gens%d.pdb'%i, args.lam*energies[i], expdata_filename_noe=expdata_filename_noe, use_log_normal_distances=False, dloggamma=np.log(1.01), gamma_min=0.2, gamma_max=5.0)
# NOTE: Upon instantiation, each Structure() object computes the distances from the given PDB.
# However, our clusters represent averaged conformation states, and so we
# need to replace these values with our own r^-6-averaged, precomputed ones
# replace PDB distances with r^-6-averaged distances
print 'len(s.distance_restraints)', len(s.distance_restraints)
for j in range(len(s.distance_restraints)):
print s.distance_restraints[j].i, s.distance_restraints[j].j, model_distances[j]
s.distance_restraints[j].model_distance = model_distances[j]
# replace chemicalshift with precomputed ones
print 'len(s.chemicalshift_restraints)', len(s.chemicalshift_restraints)
for j in range(len(s.chemicalshift_restraints)):
def name_service(self):
"""Runs the name service on all atoms needing to be named. This is a
complicated function which corrects most commonly found errors and
omissions from PDB files.
"""
if len(self.name_service_list) == 0:
return
## returns the next available chain_id in self.struct
## XXX: it's possible to run out of chain IDs!
def next_chain_id(suggest_chain_id):
if suggest_chain_id != "":
chain = self.struct.get_chain(suggest_chain_id)
if not chain:
return suggest_chain_id
## TODO: Add the following alphanumeric string to Constants.py, 2010-09-21
for chain_id in "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789":
chain = self.struct.get_chain(chain_id)
if not chain:
return chain_id
raise StructureBuilderError("name_service exhausted new chain_ids")
## NAME SERVICE FOR POLYMER ATOMS
## What if we are given a list of atoms with res_name, frag_id, and
## model_id where the frag_id are sequential? They can be sequential
## several ways using insertion codes, but large breaks often denote
## chain breaks.
## We need to handle the special case of a list of polymer residues
## which do not have chain_ids. This requires a first pass over the
## atom list using different rules than what we use for sorting out
## non-polymers.
current_polymer_type = None
current_polymer_model_id = None
current_polymer_chain_id = None
current_polymer_frag_id = None
current_polymer_res_name = None
current_polymer_name_dict = None
polymer_model_dict = {}
current_frag = None
current_frag_list = None
for atm in self.name_service_list[:]:
## determine the polymer type of the atom
if Library.library_is_amino_acid(atm.res_name):
polymer_type = "protein"
elif Library.library_is_nucleic_acid(atm.res_name):
polymer_type = "dna"
else:
## if the atom is not a polymer, we definitely have a break
## in this chain
current_polymer_type = None
current_polymer_model_id = None
current_polymer_chain_id = None
current_polymer_frag_id = None
current_polymer_res_name = None
current_polymer_name_dict = None
current_frag = None
current_frag_list = None
continue
fragment_id = Structure.FragmentID(atm.fragment_id)
## now we deal with conditions which can terminate the current
## polymer chain
if polymer_type!=current_polymer_type or \
atm.model_id!=current_polymer_model_id or \
atm.chain_id!=current_polymer_chain_id or \
fragment_id<current_polymer_frag_id:
current_polymer_type = polymer_type
current_polymer_model_id = atm.model_id
current_polymer_chain_id = atm.chain_id
current_polymer_frag_id = Structure.FragmentID(atm.fragment_id)
current_polymer_res_name = atm.res_name
current_polymer_name_dict = {atm.name: True}
## create new fragment
current_frag = [atm]
current_frag_list = [current_frag]
## create new fragment list (chain)
try:
model = polymer_model_dict[atm.model_id]
except KeyError:
model = [current_frag_list]
polymer_model_dict[atm.model_id] = model
else:
model.append(current_frag_list)
## we have now dealt with the atom, so it can be removed from
## the name service list
self.name_service_list.remove(atm)
continue
## if we get here, then we know this atom is destine for the
## current chain, and the algorithm needs to place the atom
## in the current fragment, or create a new fragment for it
## to go into; the conditions for it going into the current
## fragment are: it has it have the same res_name, and its
## atom name cannot conflict with the names of atoms already in
## in the fragment
if atm.res_name != current_polymer_res_name or current_polymer_name_dict.has_key(
atm.name):
current_polymer_res_name = atm.res_name
current_polymer_name_dict = {atm.name: True}
## create new fragment and add it to the current fragment list
current_frag = [atm]
current_frag_list.append(current_frag)
## we have now dealt with the atom, so it can be removed
## from the name service list
self.name_service_list.remove(atm)
continue
## okay, put it in the current fragment
current_frag.append(atm)
self.name_service_list.remove(atm)
## now assign chain_ids and add the atoms to the structure
model_ids = polymer_model_dict.keys()
model_ids.sort()
model_list = [polymer_model_dict[model_id] for model_id in model_ids]
num_chains = 0
for frag_list in polymer_model_dict.itervalues():
num_chains = max(num_chains, len(frag_list))
for chain_index in xrange(num_chains):
## get next available chain_id
chain_id = next_chain_id("")
## assign the chain_id to all the atoms in the chain
## TODO: check fragment_id too, 2010-09-22
for model in model_list:
frag_list = model[chain_index]
for frag in frag_list:
for atm in frag:
atm.chain_id = chain_id
self.struct.add_atom(atm, True)
## free the memory used by the polymer naming service
del polymer_model_dict
del model_list
## NAME SERVICE FOR NON-POLYMER ATOMS
## cr = (chain_id, res_name)
##
## cr_dict[cr_key] = model_dict
##
## model_dict[model] = frag_list
##
## frag_list = [ frag1, frag2, frag3, ...]
##
## frag = [atm1, atm2, atm3, ...]
cr_dict = {}
cr_key_list = []
frag_id = None
frag = None
name_dict = {}
## split atoms into fragments
for atm in self.name_service_list:
atm_id = (atm.name, atm.alt_loc)
atm_frag_id = (atm.model_id, atm.chain_id, atm.fragment_id,
atm.res_name)
## if the atom fragment id matches the current fragment id
## and doesn't conflict with any other atom name in the fragment
## then add it to the fragment
if atm_frag_id == frag_id and not name_dict.has_key(atm_id):
frag.append(atm)
name_dict[atm_id] = True
else:
cr_key = (atm.chain_id, atm.res_name)
### debug
if frag:
msg = "name_service: fragment detected in cr=%s" % (
str(cr_key))
ConsoleOutput.debug(msg)
for a in frag:
ConsoleOutput.debug(" " + str(a))
### /debug
try:
model_dict = cr_dict[cr_key]
except KeyError:
model_dict = cr_dict[cr_key] = {}
cr_key_list.append(cr_key)
try:
frag_list = model_dict[atm.model_id]
except KeyError:
frag_list = model_dict[atm.model_id] = []
name_dict = {atm_id: True}
frag_id = atm_frag_id
frag = [atm]
frag_list.append(frag)
## free self.name_service_list and other vars to save some memory
del self.name_service_list
new_chain_id = None
fragment_id_num = None
for cr_key in cr_key_list:
### debug
msg = "name_service: chain_id / res_name keys\n"
msg += " cr_key: chain_id='%s' res_name='%s'" % (cr_key[0],
cr_key[1])
ConsoleOutput.debug(msg)
### /debug
## get the next chain ID, use the cfr group's
## loaded chain_id if possible
chain_id = next_chain_id(cr_key[0])
## if we are not out of chain IDs, use the new chain ID and
## reset the fragment_id
if chain_id != None:
new_chain_id = chain_id
fragment_id_num = 0
elif new_chain_id == None or fragment_id_num == None:
ConsoleOutput.fatal(
"name_service: unable to assign any chain ids")
## get model dictionary
model_dict = cr_dict[cr_key]
## inspect the model dictionary to determine the number of
## fragments in each model -- they should be the same
## and have a 1:1 correspondence; if not, match up the
## fragments as much as possible
max_frags = -1
for (model, frag_list) in model_dict.iteritems():
frag_list_len = len(frag_list)
if max_frags == -1:
max_frags = frag_list_len
continue
if max_frags != frag_list_len:
strx = "name_service: model fragments not identical"
ConsoleOutput.debug(strx)
ConsoleOutput.warning(strx)
max_frags = max(max_frags, frag_list_len)
## now iterate through the fragment lists in parallel and assign
## the new chain_id and fragment_id
for i in xrange(max_frags):
fragment_id_num += 1
for frag_list in model_dict.itervalues():
try:
frag = frag_list[i]
except IndexError:
continue
## assign new chain_id and fragment_id, than place the
## atom in the structure
for atm in frag:
atm.chain_id = new_chain_id
atm.fragment_id = str(fragment_id_num)
self.struct.add_atom(atm, True)
## logging
ConsoleOutput.warning(
"name_service(): added chain_id=%s, res_name=%s, num_residues=%d"
% (new_chain_id, cr_key[1], fragment_id_num))
lmu = np.linalg.norm(mu[n, :])
# print(lmu)
# Plot the structure in 2D
plt.plot(x[:, 0], x[:, 1])
for n in range(N):
plt.arrow(x[n, 0], x[n, 1], mu[n, 0], mu[n, 1])
# Make x and y direction equivalent on screen
plt.axis('equal')
plt.show()
# Create Hamiltonian
H = np.zeros((N, N))
for n in range(N):
for m in range(n + 1, N):
dx = x[n, :] - x[m, :]
dd = np.linalg.norm(dx)
d3 = dd * dd * dd
d5 = d3 * dd * dd
J = np.inner(mu[n, :], mu[m, :]) / d3 + np.inner(
mu[n, :], dx) * np.inner(dx, mu[m, :]) / d5
H[n, m] = J * A
H[m, n] = J * A
# Plot structure
Structure.visual(x, mu, N, 1)
# Make spectrum
Spectra.absorption(H, mu, N, 10)
# Visualize state
#visual_exciton(x,mu,c,index,N,scale)
from Structure import * from MyViewer import * structure = Structure() r = 0.2e-1 a = 0.01 e = 2.1e11 c1 = Constraint(False, False, False) c2 = Constraint(False, False, True) c3 = Constraint(True, False, True) c4 = Constraint(True, False, False) c5 = Constraint(False, True, False) f1 = Force(1e4, -20.0e3, -100.0e2) f2 = Force(0, 0, -100e1) length = 10. height = 10 width = 10. n0 = structure.add_node(0., 0., height) n1a = structure.add_node(length * (3 / 5.), -width, height * (3 / 5.)) n2a = structure.add_node(length, 0., height * (3 / 5.)) n3a = structure.add_node(length * (3 / 5.), width, height * (3 / 5.)) n1b = structure.add_node(-length * (3 / 5.), -width, height * (3 / 5.)) n2b = structure.add_node(-length, 0., height * (3 / 5.)) n3b = structure.add_node(-length * (3 / 5.), width, height * (3 / 5.)) n1 = structure.add_node(0, width * 1.5, 0.) n2 = structure.add_node(0, -width * 1.5, 0.) n3 = structure.add_node(length * 1.2, width * 0.7, 0.) n4 = structure.add_node(-length * 1.2, -width * 0.7, 0.) n5 = structure.add_node(-length * 1.2, width * 0.7, 0.)
def takePlayerInput(self, board=Structure.Board()):
self.playerInput = input()
if not int(self.playerInput) in board.listOfAvailableIndexes:
self.takePlayerInput()
print(f"{self.name} chose Position - {self.playerInput}")
return self.playerInput
expdata_filename_J = 'albo.Jcoupling'
energies_filename = 'albocycline_QMenergies.dat'
energies = loadtxt(energies_filename)*627.509 # convert from hartrees to kcal/mol
energies = energies/0.5959 # convert to reduced free energies F = f/kT
energies -= energies.min()
ensemble = []
for i in range(100):
print
print '#### STRUCTURE %d ####'%i
# no information from QM --> lam = 0.0
# QM + exp --> lam = 1.0
# model_distances = loadtxt('NOE/average_state%d.txt'%i)*10.0 # convert to A
s=Structure('pdbs_guangfeng/%d.pdb'%i, args.lam*energies[i], expdata_filename_noe, expdata_filename_J, use_log_normal_distances=False, dloggamma=np.log(1.01), gamma_min=0.2, gamma_max=5.0 )
# replace PDB distances with r^-6-averaged distances
# print 'len(s.distance_restraints)', len(s.distance_restraints)
# for j in range(len(s.distance_restraints)):
# print s.distance_restraints[j].i, s.distance_restraints[j].j, model_distances[j]
# s.distance_restraints[j].model_distance = model_distances[j]
# update the distance sse's!
s.compute_sse_distances()
# add the structure to the ensemble
ensemble.append( s )
def __init__(self, data):
self.data = data
self.valid = False
self.callbacks = []
self.symbols_by_name = {}
self.symbols_by_addr = {}
self.plt = {}
if not self.is_macho():
return
try:
self.tree = Structure(self.data)
self.header = self.tree.struct("Mach-O header", "header")
self.header.uint32_le("magic")
if (self.header.magic == 0xfeedface) or (self.header.magic == 0xfeedfacf):
self.header.uint32_le("cputype")
self.header.uint32_le("cpusubtype")
self.header.uint32_le("filetype")
self.header.uint32_le("cmds")
self.header.uint32_le("cmdsize")
self.header.uint32_le("flags")
if self.header.magic == 0xfeedfacf:
self.header.uint32_le("reserved")
self.bits = 64
else:
self.bits = 32
self.big_endian = False
elif (self.header.magic == 0xcefaedfe) or (self.header.magic == 0xcffaedfe):
self.header.uint32_be("cputype")
self.header.uint32_be("cpusubtype")
self.header.uint32_be("filetype")
self.header.uint32_be("cmds")
self.header.uint32_be("cmdsize")
self.header.uint32_be("flags")
if self.header.magic == 0xcffaedfe:
self.header.uint32_be("reserved")
self.bits = 64
else:
self.bits = 32
self.big_endian = True
self.symbol_table = None
self.dynamic_symbol_table = None
# Parse loader commands
self.commands = self.tree.array(self.header.cmds, "commands")
self.segments = []
self.sections = []
offset = self.header.getSize()
for i in xrange(0, self.header.cmds):
cmd = self.commands[i]
cmd.seek(offset)
if self.big_endian:
cmd.uint32_be("cmd")
cmd.uint32_be("size")
else:
cmd.uint32_le("cmd")
cmd.uint32_le("size")
if cmd.cmd == 1: # SEGMENT
cmd.bytes(16, "name")
if self.big_endian:
cmd.uint32_be("vmaddr")
cmd.uint32_be("vmsize")
cmd.uint32_be("fileoff")
cmd.uint32_be("filesize")
cmd.uint32_be("maxprot")
cmd.uint32_be("initprot")
cmd.uint32_be("nsects")
cmd.uint32_be("flags")
else:
cmd.uint32_le("vmaddr")
cmd.uint32_le("vmsize")
cmd.uint32_le("fileoff")
cmd.uint32_le("filesize")
cmd.uint32_le("maxprot")
cmd.uint32_le("initprot")
cmd.uint32_le("nsects")
cmd.uint32_le("flags")
if cmd.initprot != 0: # Ignore __PAGE_ZERO or anything like it
self.segments.append(cmd)
cmd.array(cmd.nsects, "sections")
for i in xrange(0, cmd.nsects):
section = cmd.sections[i]
section.bytes(16, "name")
section.bytes(16, "segment")
if self.big_endian:
section.uint32_be("addr")
section.uint32_be("size")
section.uint32_be("offset")
section.uint32_be("align")
section.uint32_be("reloff")
section.uint32_be("nreloc")
section.uint32_be("flags")
section.uint32_be("reserved1")
section.uint32_be("reserved2")
else:
section.uint32_le("addr")
section.uint32_le("size")
section.uint32_le("offset")
section.uint32_le("align")
section.uint32_le("reloff")
section.uint32_le("nreloc")
section.uint32_le("flags")
section.uint32_le("reserved1")
section.uint32_le("reserved2")
self.sections.append(section)
for i in xrange(0, cmd.nsects):
section = cmd.sections[i]
section.array(section.nreloc, "relocs")
for j in xrange(0, section.nreloc):
reloc = section.relocs[j]
reloc.seek(section.reloff + (j * 8))
if self.big_endian:
reloc.uint32_be("addr")
reloc.uint32_be("value")
else:
reloc.uint32_le("addr")
reloc.uint32_le("value")
elif cmd.cmd == 25: # SEGMENT_64
cmd.bytes(16, "name")
if self.big_endian:
cmd.uint64_be("vmaddr")
cmd.uint64_be("vmsize")
cmd.uint64_be("fileoff")
cmd.uint64_be("filesize")
cmd.uint32_be("maxprot")
cmd.uint32_be("initprot")
cmd.uint32_be("nsects")
cmd.uint32_be("flags")
else:
cmd.uint64_le("vmaddr")
cmd.uint64_le("vmsize")
cmd.uint64_le("fileoff")
cmd.uint64_le("filesize")
cmd.uint32_le("maxprot")
cmd.uint32_le("initprot")
cmd.uint32_le("nsects")
cmd.uint32_le("flags")
if cmd.initprot != 0: # Ignore __PAGE_ZERO or anything like it
self.segments.append(cmd)
cmd.array(cmd.nsects, "sections")
for i in xrange(0, cmd.nsects):
section = cmd.sections[i]
section.bytes(16, "name")
section.bytes(16, "segment")
if self.big_endian:
section.uint64_be("addr")
section.uint64_be("size")
section.uint32_be("offset")
section.uint32_be("align")
section.uint32_be("reloff")
section.uint32_be("nreloc")
section.uint32_be("flags")
section.uint32_be("reserved1")
section.uint32_be("reserved2")
section.uint32_be("reserved3")
else:
section.uint64_le("addr")
section.uint64_le("size")
section.uint32_le("offset")
section.uint32_le("align")
section.uint32_le("reloff")
section.uint32_le("nreloc")
section.uint32_le("flags")
section.uint32_le("reserved1")
section.uint32_le("reserved2")
section.uint32_le("reserved3")
self.sections.append(section)
for i in xrange(0, cmd.nsects):
section = cmd.sections[i]
section.array(section.nreloc, "relocs")
for j in xrange(0, section.nreloc):
reloc = section.relocs[j]
reloc.seek(section.reloff + (j * 8))
if self.big_endian:
reloc.uint32_be("addr")
reloc.uint32_be("value")
else:
reloc.uint32_le("addr")
reloc.uint32_le("value")
elif cmd.cmd == 5: # UNIX_THREAD
if self.header.cputype == 7: # x86
cmd.uint32_le("flavor")
cmd.uint32_le("count")
for reg in ["eax", "ebx", "ecx", "edx", "edi", "esi", "ebp", "esp", "ss", "eflags",
"eip", "cs", "ds", "es", "fs", "gs"]:
cmd.uint32_le(reg)
self.entry_addr = cmd.eip
elif self.header.cputype == 0x01000007: # x86_64
cmd.uint32_le("flavor")
cmd.uint32_le("count")
for reg in ["rax", "rbx", "rcx", "rdx", "rdi", "rsi", "rbp", "rsp", "r8", "r9",
"r10", "r11", "r12", "r13", "r14", "r15", "rip", "rflags", "cs", "fs", "gs"]:
cmd.uint64_le(reg)
self.entry_addr = cmd.rip
elif self.header.cputype == 18: # PPC32
cmd.uint32_be("flavor")
cmd.uint32_be("count")
for reg in ["srr0", "srr1"] + ["r%d" % i for i in xrange(0, 32)] + ["cr", "xer",
"lr", "ctr", "mq", "vrsave"]:
cmd.uint32_be(reg)
self.entry_addr = cmd.srr0
elif self.header.cputype == 0x01000012: # PPC64
cmd.uint32_be("flavor")
cmd.uint32_be("count")
for reg in ["srr0", "srr1"] + ["r%d" % i for i in xrange(0, 32)] + ["cr", "xer",
"lr", "ctr", "mq", "vrsave"]:
cmd.uint64_be(reg)
self.entry_addr = cmd.srr0
elif self.header.cputype == 12: # ARM
cmd.uint32_le("flavor")
cmd.uint32_le("count")
for reg in ["r%d" % i for i in xrange(0, 13)] + ["sp", "lr", "pc", "cpsr"]:
cmd.uint32_le(reg)
self.entry_addr = cmd.pc
elif cmd.cmd == 2: # SYMTAB
if self.big_endian:
cmd.uint32_be("symoff")
cmd.uint32_be("nsyms")
cmd.uint32_be("stroff")
cmd.uint32_be("strsize")
else:
cmd.uint32_le("symoff")
cmd.uint32_le("nsyms")
cmd.uint32_le("stroff")
cmd.uint32_le("strsize")
self.symbol_table = self.tree.array(cmd.nsyms, "symtab")
strings = self.data.read(cmd.stroff, cmd.strsize)
sym_offset = cmd.symoff
for j in xrange(0, cmd.nsyms):
entry = self.symbol_table[j]
entry.seek(sym_offset)
if self.big_endian:
entry.uint32_be("strx")
entry.uint8("type")
entry.uint8("sect")
entry.uint16_be("desc")
if self.bits == 32:
entry.uint32_be("value")
else:
entry.uint64_be("value")
else:
entry.uint32_le("strx")
entry.uint8("type")
entry.uint8("sect")
entry.uint16_le("desc")
if self.bits == 32:
entry.uint32_le("value")
else:
entry.uint64_le("value")
str_end = strings.find("\x00", entry.strx)
entry.name = strings[entry.strx:str_end]
if self.bits == 32:
sym_offset += 12
else:
sym_offset += 16
elif cmd.cmd == 11: # DYSYMTAB
if self.big_endian:
cmd.uint32_be("ilocalsym")
cmd.uint32_be("nlocalsym")
cmd.uint32_be("iextdefsym")
cmd.uint32_be("nextdefsym")
cmd.uint32_be("iundefsym")
cmd.uint32_be("nundefsym")
cmd.uint32_be("tocoff")
cmd.uint32_be("ntoc")
cmd.uint32_be("modtaboff")
cmd.uint32_be("nmodtab")
cmd.uint32_be("extrefsymoff")
cmd.uint32_be("nextrefsyms")
cmd.uint32_be("indirectsymoff")
cmd.uint32_be("nindirectsyms")
cmd.uint32_be("extreloff")
cmd.uint32_be("nextrel")
cmd.uint32_be("locreloff")
cmd.uint32_be("nlocrel")
else:
cmd.uint32_le("ilocalsym")
cmd.uint32_le("nlocalsym")
cmd.uint32_le("iextdefsym")
cmd.uint32_le("nextdefsym")
cmd.uint32_le("iundefsym")
cmd.uint32_le("nundefsym")
cmd.uint32_le("tocoff")
cmd.uint32_le("ntoc")
cmd.uint32_le("modtaboff")
cmd.uint32_le("nmodtab")
cmd.uint32_le("extrefsymoff")
cmd.uint32_le("nextrefsyms")
cmd.uint32_le("indirectsymoff")
cmd.uint32_le("nindirectsyms")
cmd.uint32_le("extreloff")
cmd.uint32_le("nextrel")
cmd.uint32_le("locreloff")
cmd.uint32_le("nlocrel")
elif (cmd.cmd & 0x7fffffff) == 0x22: # DYLD_INFO
self.dynamic_symbol_table = cmd
if self.big_endian:
cmd.uint32_be("rebaseoff")
cmd.uint32_be("rebasesize")
cmd.uint32_be("bindoff")
cmd.uint32_be("bindsize")
cmd.uint32_be("weakbindoff")
cmd.uint32_be("weakbindsize")
cmd.uint32_be("lazybindoff")
cmd.uint32_be("lazybindsize")
cmd.uint32_be("exportoff")
cmd.uint32_be("exportsize")
else:
cmd.uint32_le("rebaseoff")
cmd.uint32_le("rebasesize")
cmd.uint32_le("bindoff")
cmd.uint32_le("bindsize")
cmd.uint32_le("weakbindoff")
cmd.uint32_le("weakbindsize")
cmd.uint32_le("lazybindoff")
cmd.uint32_le("lazybindsize")
cmd.uint32_le("exportoff")
cmd.uint32_le("exportsize")
offset += cmd.size
# Add symbols from symbol table
if self.symbol_table:
for i in xrange(0, len(self.symbol_table)):
symbol = self.symbol_table[i]
# Only use symbols that are within a section
if ((symbol.type & 0xe) == 0xe) and (symbol.sect <= len(self.sections)):
self.create_symbol(symbol.value, symbol.name)
# If there is a DYLD_INFO section, parse it and add PLT entries
if self.dynamic_symbol_table:
self.parse_dynamic_tables([[self.dynamic_symbol_table.bindoff, self.dynamic_symbol_table.bindsize],
[self.dynamic_symbol_table.lazybindoff, self.dynamic_symbol_table.lazybindsize]])
self.tree.complete()
self.valid = True
except:
self.valid = False
raise
if self.valid:
self.data.add_callback(self)
class MachOFile(BinaryAccessor):
def __init__(self, data):
self.data = data
self.valid = False
self.callbacks = []
self.symbols_by_name = {}
self.symbols_by_addr = {}
self.plt = {}
if not self.is_macho():
return
try:
self.tree = Structure(self.data)
self.header = self.tree.struct("Mach-O header", "header")
self.header.uint32_le("magic")
if (self.header.magic == 0xfeedface) or (self.header.magic == 0xfeedfacf):
self.header.uint32_le("cputype")
self.header.uint32_le("cpusubtype")
self.header.uint32_le("filetype")
self.header.uint32_le("cmds")
self.header.uint32_le("cmdsize")
self.header.uint32_le("flags")
if self.header.magic == 0xfeedfacf:
self.header.uint32_le("reserved")
self.bits = 64
else:
self.bits = 32
self.big_endian = False
elif (self.header.magic == 0xcefaedfe) or (self.header.magic == 0xcffaedfe):
self.header.uint32_be("cputype")
self.header.uint32_be("cpusubtype")
self.header.uint32_be("filetype")
self.header.uint32_be("cmds")
self.header.uint32_be("cmdsize")
self.header.uint32_be("flags")
if self.header.magic == 0xcffaedfe:
self.header.uint32_be("reserved")
self.bits = 64
else:
self.bits = 32
self.big_endian = True
self.symbol_table = None
self.dynamic_symbol_table = None
# Parse loader commands
self.commands = self.tree.array(self.header.cmds, "commands")
self.segments = []
self.sections = []
offset = self.header.getSize()
for i in xrange(0, self.header.cmds):
cmd = self.commands[i]
cmd.seek(offset)
if self.big_endian:
cmd.uint32_be("cmd")
cmd.uint32_be("size")
else:
cmd.uint32_le("cmd")
cmd.uint32_le("size")
if cmd.cmd == 1: # SEGMENT
cmd.bytes(16, "name")
if self.big_endian:
cmd.uint32_be("vmaddr")
cmd.uint32_be("vmsize")
cmd.uint32_be("fileoff")
cmd.uint32_be("filesize")
cmd.uint32_be("maxprot")
cmd.uint32_be("initprot")
cmd.uint32_be("nsects")
cmd.uint32_be("flags")
else:
cmd.uint32_le("vmaddr")
cmd.uint32_le("vmsize")
cmd.uint32_le("fileoff")
cmd.uint32_le("filesize")
cmd.uint32_le("maxprot")
cmd.uint32_le("initprot")
cmd.uint32_le("nsects")
cmd.uint32_le("flags")
if cmd.initprot != 0: # Ignore __PAGE_ZERO or anything like it
self.segments.append(cmd)
cmd.array(cmd.nsects, "sections")
for i in xrange(0, cmd.nsects):
section = cmd.sections[i]
section.bytes(16, "name")
section.bytes(16, "segment")
if self.big_endian:
section.uint32_be("addr")
section.uint32_be("size")
section.uint32_be("offset")
section.uint32_be("align")
section.uint32_be("reloff")
section.uint32_be("nreloc")
section.uint32_be("flags")
section.uint32_be("reserved1")
section.uint32_be("reserved2")
else:
section.uint32_le("addr")
section.uint32_le("size")
section.uint32_le("offset")
section.uint32_le("align")
section.uint32_le("reloff")
section.uint32_le("nreloc")
section.uint32_le("flags")
section.uint32_le("reserved1")
section.uint32_le("reserved2")
self.sections.append(section)
for i in xrange(0, cmd.nsects):
section = cmd.sections[i]
section.array(section.nreloc, "relocs")
for j in xrange(0, section.nreloc):
reloc = section.relocs[j]
reloc.seek(section.reloff + (j * 8))
if self.big_endian:
reloc.uint32_be("addr")
reloc.uint32_be("value")
else:
reloc.uint32_le("addr")
reloc.uint32_le("value")
elif cmd.cmd == 25: # SEGMENT_64
cmd.bytes(16, "name")
if self.big_endian:
cmd.uint64_be("vmaddr")
cmd.uint64_be("vmsize")
cmd.uint64_be("fileoff")
cmd.uint64_be("filesize")
cmd.uint32_be("maxprot")
cmd.uint32_be("initprot")
cmd.uint32_be("nsects")
cmd.uint32_be("flags")
else:
cmd.uint64_le("vmaddr")
cmd.uint64_le("vmsize")
cmd.uint64_le("fileoff")
cmd.uint64_le("filesize")
cmd.uint32_le("maxprot")
cmd.uint32_le("initprot")
cmd.uint32_le("nsects")
cmd.uint32_le("flags")
if cmd.initprot != 0: # Ignore __PAGE_ZERO or anything like it
self.segments.append(cmd)
cmd.array(cmd.nsects, "sections")
for i in xrange(0, cmd.nsects):
section = cmd.sections[i]
section.bytes(16, "name")
section.bytes(16, "segment")
if self.big_endian:
section.uint64_be("addr")
section.uint64_be("size")
section.uint32_be("offset")
section.uint32_be("align")
section.uint32_be("reloff")
section.uint32_be("nreloc")
section.uint32_be("flags")
section.uint32_be("reserved1")
section.uint32_be("reserved2")
section.uint32_be("reserved3")
else:
section.uint64_le("addr")
section.uint64_le("size")
section.uint32_le("offset")
section.uint32_le("align")
section.uint32_le("reloff")
section.uint32_le("nreloc")
section.uint32_le("flags")
section.uint32_le("reserved1")
section.uint32_le("reserved2")
section.uint32_le("reserved3")
self.sections.append(section)
for i in xrange(0, cmd.nsects):
section = cmd.sections[i]
section.array(section.nreloc, "relocs")
for j in xrange(0, section.nreloc):
reloc = section.relocs[j]
reloc.seek(section.reloff + (j * 8))
if self.big_endian:
reloc.uint32_be("addr")
reloc.uint32_be("value")
else:
reloc.uint32_le("addr")
reloc.uint32_le("value")
elif cmd.cmd == 5: # UNIX_THREAD
if self.header.cputype == 7: # x86
cmd.uint32_le("flavor")
cmd.uint32_le("count")
for reg in ["eax", "ebx", "ecx", "edx", "edi", "esi", "ebp", "esp", "ss", "eflags",
"eip", "cs", "ds", "es", "fs", "gs"]:
cmd.uint32_le(reg)
self.entry_addr = cmd.eip
elif self.header.cputype == 0x01000007: # x86_64
cmd.uint32_le("flavor")
cmd.uint32_le("count")
for reg in ["rax", "rbx", "rcx", "rdx", "rdi", "rsi", "rbp", "rsp", "r8", "r9",
"r10", "r11", "r12", "r13", "r14", "r15", "rip", "rflags", "cs", "fs", "gs"]:
cmd.uint64_le(reg)
self.entry_addr = cmd.rip
elif self.header.cputype == 18: # PPC32
cmd.uint32_be("flavor")
cmd.uint32_be("count")
for reg in ["srr0", "srr1"] + ["r%d" % i for i in xrange(0, 32)] + ["cr", "xer",
"lr", "ctr", "mq", "vrsave"]:
cmd.uint32_be(reg)
self.entry_addr = cmd.srr0
elif self.header.cputype == 0x01000012: # PPC64
cmd.uint32_be("flavor")
cmd.uint32_be("count")
for reg in ["srr0", "srr1"] + ["r%d" % i for i in xrange(0, 32)] + ["cr", "xer",
"lr", "ctr", "mq", "vrsave"]:
cmd.uint64_be(reg)
self.entry_addr = cmd.srr0
elif self.header.cputype == 12: # ARM
cmd.uint32_le("flavor")
cmd.uint32_le("count")
for reg in ["r%d" % i for i in xrange(0, 13)] + ["sp", "lr", "pc", "cpsr"]:
cmd.uint32_le(reg)
self.entry_addr = cmd.pc
elif cmd.cmd == 2: # SYMTAB
if self.big_endian:
cmd.uint32_be("symoff")
cmd.uint32_be("nsyms")
cmd.uint32_be("stroff")
cmd.uint32_be("strsize")
else:
cmd.uint32_le("symoff")
cmd.uint32_le("nsyms")
cmd.uint32_le("stroff")
cmd.uint32_le("strsize")
self.symbol_table = self.tree.array(cmd.nsyms, "symtab")
strings = self.data.read(cmd.stroff, cmd.strsize)
sym_offset = cmd.symoff
for j in xrange(0, cmd.nsyms):
entry = self.symbol_table[j]
entry.seek(sym_offset)
if self.big_endian:
entry.uint32_be("strx")
entry.uint8("type")
entry.uint8("sect")
entry.uint16_be("desc")
if self.bits == 32:
entry.uint32_be("value")
else:
entry.uint64_be("value")
else:
entry.uint32_le("strx")
entry.uint8("type")
entry.uint8("sect")
entry.uint16_le("desc")
if self.bits == 32:
entry.uint32_le("value")
else:
entry.uint64_le("value")
str_end = strings.find("\x00", entry.strx)
entry.name = strings[entry.strx:str_end]
if self.bits == 32:
sym_offset += 12
else:
sym_offset += 16
elif cmd.cmd == 11: # DYSYMTAB
if self.big_endian:
cmd.uint32_be("ilocalsym")
cmd.uint32_be("nlocalsym")
cmd.uint32_be("iextdefsym")
cmd.uint32_be("nextdefsym")
cmd.uint32_be("iundefsym")
cmd.uint32_be("nundefsym")
cmd.uint32_be("tocoff")
cmd.uint32_be("ntoc")
cmd.uint32_be("modtaboff")
cmd.uint32_be("nmodtab")
cmd.uint32_be("extrefsymoff")
cmd.uint32_be("nextrefsyms")
cmd.uint32_be("indirectsymoff")
cmd.uint32_be("nindirectsyms")
cmd.uint32_be("extreloff")
cmd.uint32_be("nextrel")
cmd.uint32_be("locreloff")
cmd.uint32_be("nlocrel")
else:
cmd.uint32_le("ilocalsym")
cmd.uint32_le("nlocalsym")
cmd.uint32_le("iextdefsym")
cmd.uint32_le("nextdefsym")
cmd.uint32_le("iundefsym")
cmd.uint32_le("nundefsym")
cmd.uint32_le("tocoff")
cmd.uint32_le("ntoc")
cmd.uint32_le("modtaboff")
cmd.uint32_le("nmodtab")
cmd.uint32_le("extrefsymoff")
cmd.uint32_le("nextrefsyms")
cmd.uint32_le("indirectsymoff")
cmd.uint32_le("nindirectsyms")
cmd.uint32_le("extreloff")
cmd.uint32_le("nextrel")
cmd.uint32_le("locreloff")
cmd.uint32_le("nlocrel")
elif (cmd.cmd & 0x7fffffff) == 0x22: # DYLD_INFO
self.dynamic_symbol_table = cmd
if self.big_endian:
cmd.uint32_be("rebaseoff")
cmd.uint32_be("rebasesize")
cmd.uint32_be("bindoff")
cmd.uint32_be("bindsize")
cmd.uint32_be("weakbindoff")
cmd.uint32_be("weakbindsize")
cmd.uint32_be("lazybindoff")
cmd.uint32_be("lazybindsize")
cmd.uint32_be("exportoff")
cmd.uint32_be("exportsize")
else:
cmd.uint32_le("rebaseoff")
cmd.uint32_le("rebasesize")
cmd.uint32_le("bindoff")
cmd.uint32_le("bindsize")
cmd.uint32_le("weakbindoff")
cmd.uint32_le("weakbindsize")
cmd.uint32_le("lazybindoff")
cmd.uint32_le("lazybindsize")
cmd.uint32_le("exportoff")
cmd.uint32_le("exportsize")
offset += cmd.size
# Add symbols from symbol table
if self.symbol_table:
for i in xrange(0, len(self.symbol_table)):
symbol = self.symbol_table[i]
# Only use symbols that are within a section
if ((symbol.type & 0xe) == 0xe) and (symbol.sect <= len(self.sections)):
self.create_symbol(symbol.value, symbol.name)
# If there is a DYLD_INFO section, parse it and add PLT entries
if self.dynamic_symbol_table:
self.parse_dynamic_tables([[self.dynamic_symbol_table.bindoff, self.dynamic_symbol_table.bindsize],
[self.dynamic_symbol_table.lazybindoff, self.dynamic_symbol_table.lazybindsize]])
self.tree.complete()
self.valid = True
except:
self.valid = False
raise
if self.valid:
self.data.add_callback(self)
def read_leb128(self, data, ofs):
value = 0
shift = 0
while ofs < len(data):
cur = ord(data[ofs])
ofs += 1
value |= (cur & 0x7f) << shift
shift += 7
if (cur & 0x80) == 0:
break
return value, ofs
def parse_dynamic_tables(self, tables):
# Interpret DYLD_INFO instructions (not documented by Apple)
# http://networkpx.blogspot.com/2009/09/about-lcdyldinfoonly-command.html
ordinal = 0
segment = 0
offset = 0
sym_type = 0
name = ""
for table in tables:
offset = table[0]
size = table[1]
opcodes = self.data.read(offset, size)
i = 0
while i < len(opcodes):
opcode = ord(opcodes[i])
i += 1
if (opcode >> 4) == 0:
continue
elif (opcode >> 4) == 1:
ordinal = opcode & 0xf
elif (opcode >> 4) == 2:
ordinal, i = self.read_leb128(opcodes, i)
elif (opcode >> 4) == 3:
ordinal = -(opcode & 0xf)
elif (opcode >> 4) == 4:
name = ""
while i < len(opcodes):
ch = opcodes[i]
i += 1
if ch == '\x00':
break
name += ch
elif (opcode >> 4) == 5:
sym_type = opcode & 0xf
elif (opcode >> 4) == 6:
addend, i = self.read_leb128(opcodes, i)
elif (opcode >> 4) == 7:
segment = opcode & 0xf
offset, i = self.read_leb128(opcodes, i)
elif (opcode >> 4) == 8:
rel, i = self.read_leb128(opcodes, i)
offset += rel
elif (opcode >> 4) >= 9:
if (sym_type == 1) and (segment <= len(self.segments)):
# Add pointer type entries to the PLT
addr = self.segments[segment - 1].vmaddr + offset
self.plt[addr] = name
self.create_symbol(addr, self.decorate_plt_name(name))
if self.bits == 32:
offset += 4
else:
offset += 8
if (opcode >> 4) == 10:
rel, i = self.read_leb128(opcodes, i)
offset += rel
elif (opcode >> 4) == 11:
offset += (opcode & 0xf) * 4
elif (opcode >> 4) == 12:
count, i = self.read_leb128(opcodes, i)
skip, i = self.read_leb128(opcodes, i)
def read(self, ofs, len):
result = ""
while len > 0:
cur = None
for i in self.segments:
if ((ofs >= i.vmaddr) and (ofs < (i.vmaddr + i.vmsize))) and (i.vmsize != 0):
cur = i
if cur == None:
break
prog_ofs = ofs - cur.vmaddr
mem_len = cur.vmsize - prog_ofs
file_len = cur.filesize - prog_ofs
if mem_len > len:
mem_len = len
if file_len > len:
file_len = len
if file_len <= 0:
result += "\x00" * mem_len
len -= mem_len
ofs += mem_len
continue
result += self.data.read(cur.fileoff + prog_ofs, file_len)
len -= file_len
ofs += file_len
return result
def next_valid_addr(self, ofs):
result = -1
for i in self.segments:
if (i.vmaddr >= ofs) and (i.vmsize != 0) and ((result == -1) or (i.vmaddr < result)):
result = i.vmaddr
return result
def get_modification(self, ofs, len):
result = []
while len > 0:
cur = None
for i in self.segments:
if ((ofs >= i.vmaddr) and (ofs < (i.vmaddr + i.vmsize))) and (i.vmsize != 0):
cur = i
if cur == None:
break
prog_ofs = ofs - cur.vmaddr
mem_len = cur.vmsize - prog_ofs
file_len = cur.filesize - prog_ofs
if mem_len > len:
mem_len = len
if file_len > len:
file_len = len
if file_len <= 0:
result += [DATA_ORIGINAL] * mem_len
len -= mem_len
ofs += mem_len
continue
result += self.data.get_modification(cur.fileoff + prog_ofs, file_len)
len -= file_len
ofs += file_len
return result
def write(self, ofs, data):
result = 0
while len(data) > 0:
cur = None
for i in self.segments:
if ((ofs >= i.vmaddr) and (ofs < (i.vmaddr + i.vmsize))) and (i.vmsize != 0):
cur = i
if cur == None:
break
prog_ofs = ofs - cur.vmaddr
mem_len = cur.vmsize - prog_ofs
file_len = cur.filesize - prog_ofs
if mem_len > len:
mem_len = len
if file_len > len:
file_len = len
if file_len <= 0:
break
result += self.data.write(cur.fileoff + prog_ofs, data[0:file_len])
data = data[file_len:]
ofs += file_len
return result
def insert(self, ofs, data):
return 0
def remove(self, ofs, size):
return 0
def notify_data_write(self, data, ofs, contents):
# Find sections that hold data backed by updated regions of the file
for i in self.segments:
if ((ofs + len(contents)) > i.fileoff) and (ofs < (i.fileoff + i.filesize)) and (i.vmsize != 0):
# This section has been updated, compute which region has been changed
from_start = ofs - i.fileoff
data_ofs = 0
length = len(contents)
if from_start < 0:
length += from_start
data_ofs -= from_start
from_start = 0
if (from_start + length) > i.filesize:
length = i.filesize - from_start
# Notify callbacks
if length > 0:
for cb in self.callbacks:
if hasattr(cb, "notify_data_write"):
cb.notify_data_write(self, i.vmaddr + from_start,
contents[data_ofs:(data_ofs + length)])
def save(self, filename):
self.data.save(filename)
def start(self):
result = None
for i in self.segments:
if ((result == None) or (i.vmaddr < result)) and (i.vmsize != 0):
result = i.vmaddr
return result
def entry(self):
if not hasattr(self, "entry_addr"):
return self.start()
return self.entry_addr
def __len__(self):
max = None
for i in self.segments:
if ((max == None) or ((i.vmaddr + i.vmsize) > max)) and (i.vmsize != 0):
max = i.vmaddr + i.vmsize
return max - self.start()
def is_macho(self):
if self.data.read(0, 4) == "\xfe\xed\xfa\xce":
return True
if self.data.read(0, 4) == "\xfe\xed\xfa\xcf":
return True
if self.data.read(0, 4) == "\xce\xfa\xed\xfe":
return True
if self.data.read(0, 4) == "\xcf\xfa\xed\xfe":
return True
return False
def architecture(self):
if self.header.cputype == 7:
return "x86"
if self.header.cputype == 0x01000007:
return "x86_64"
if self.header.cputype == 12:
return "arm"
if self.header.cputype == 18:
return "ppc"
if self.header.cputype == 0x01000012:
return "ppc"
return None
def decorate_plt_name(self, name):
return name + "@PLT"
def create_symbol(self, addr, name):
self.symbols_by_name[name] = addr
self.symbols_by_addr[addr] = name
def delete_symbol(self, addr, name):
if name in self.symbols_by_name:
del(self.symbols_by_name[name])
if addr in self.symbols_by_addr:
del(self.symbols_by_addr[addr])
def add_callback(self, cb):
self.callbacks.append(cb)
def remove_callback(self, cb):
self.callbacks.remove(cb)
def is_modified(self):
return self.data.is_modified()
def find(self, regex, addr):
while (addr < self.end()) and (addr != -1):
data = self.read(addr, 0xfffffffff)
match = regex.search(data)
if match != None:
return match.start() + addr
addr += len(data)
addr = self.next_valid_addr(addr)
return -1
def has_undo_actions(self):
return self.data.has_undo_actions()
def commit_undo(self, before_loc, after_loc):
self.data.commit_undo(before_loc, after_loc)
def undo(self):
self.data.undo()
def redo(self):
self.data.redo()
import General, Structure, Visual, SARS2
wuhan_id = 'NC_045512.2'
shape = General.load_shape('/Share2/home/zhangqf7/lipan/SARS2/icSHAPE/2020-06-01-process/virus-w50.shape')[wuhan_id]
sequence = General.load_fasta('/Share2/home/zhangqf7/lipan/SARS2/sequence/SARS2.fa')[wuhan_id]
Len = len(sequence)
out = '/Share2/home/zhangqf7/lipan/SARS2/predict_structure/SARS2/full_length'
start = 0
finish = False
winsize = 5000
overlap = 1000
while not finish:
if start+winsize>Len:
start = Len - winsize
finish = True
shapefn = f"{start+1}-{start+winsize}.shape"
seqfn = f"{start+1}-{start+winsize}.seq"
Structure.__build_SHAPE_constraint(shape[start:start+winsize], join(out, shapefn))
Structure.__build_single_seq_fasta(sequence[start:start+winsize], join(out, seqfn))
start += winsize - overlap
def __init__(self, data):
self.data = data
self.valid = False
self.callbacks = []
self.symbols_by_name = {}
self.symbols_by_addr = {}
if not self.is_pe():
return
try:
self.tree = Structure(self.data)
self.mz = self.tree.struct("MZ header", "mz")
self.mz.uint16("magic")
self.mz.uint16("lastsize")
self.mz.uint16("nblocks")
self.mz.uint16("nreloc")
self.mz.uint16("hdrsize")
self.mz.uint16("minalloc")
self.mz.uint16("maxalloc")
self.mz.uint16("ss")
self.mz.uint16("sp")
self.mz.uint16("checksum")
self.mz.uint16("ip")
self.mz.uint16("cs")
self.mz.uint16("relocpos")
self.mz.uint16("noverlay")
self.mz.bytes(8, "reserved1")
self.mz.uint16("oem_id")
self.mz.uint16("oem_info")
self.mz.bytes(20, "reserved2")
self.mz.uint32("pe_offset")
self.header = self.tree.struct("PE header", "header")
self.header.seek(self.mz.pe_offset)
self.header.uint32("magic")
self.header.uint16("machine")
self.header.uint16("section_count")
self.header.uint32("timestamp")
self.header.uint32("coff_symbol_table")
self.header.uint32("coff_symbol_count")
self.header.uint16("optional_header_size")
self.header.uint16("characteristics")
self.header.struct("Optional header", "opt")
self.header.opt.uint16("magic")
self.header.opt.uint8("major_linker_version")
self.header.opt.uint8("minor_linker_version")
self.header.opt.uint32("size_of_code")
self.header.opt.uint32("size_of_init_data")
self.header.opt.uint32("size_of_uninit_data")
self.header.opt.uint32("address_of_entry")
self.header.opt.uint32("base_of_code")
if self.header.opt.magic == 0x10B: # 32-bit
self.bits = 32
self.header.opt.uint32("base_of_data")
self.header.opt.uint32("image_base")
self.header.opt.uint32("section_align")
self.header.opt.uint32("file_align")
self.header.opt.uint16("major_os_version")
self.header.opt.uint16("minor_os_version")
self.header.opt.uint16("major_image_version")
self.header.opt.uint16("minor_image_version")
self.header.opt.uint16("major_subsystem_version")
self.header.opt.uint16("minor_subsystem_version")
self.header.opt.uint32("win32_version")
self.header.opt.uint32("size_of_image")
self.header.opt.uint32("size_of_headers")
self.header.opt.uint32("checksum")
self.header.opt.uint16("subsystem")
self.header.opt.uint16("dll_characteristics")
self.header.opt.uint32("size_of_stack_reserve")
self.header.opt.uint32("size_of_stack_commit")
self.header.opt.uint32("size_of_heap_reserve")
self.header.opt.uint32("size_of_heap_commit")
self.header.opt.uint32("loader_flags")
self.header.opt.uint32("data_dir_count")
elif self.header.opt.magic == 0x20B: # 64-bit
self.bits = 64
self.header.opt.uint64("image_base")
self.header.opt.uint32("section_align")
self.header.opt.uint32("file_align")
self.header.opt.uint16("major_os_version")
self.header.opt.uint16("minor_os_version")
self.header.opt.uint16("major_image_version")
self.header.opt.uint16("minor_image_version")
self.header.opt.uint16("major_subsystem_version")
self.header.opt.uint16("minor_subsystem_version")
self.header.opt.uint32("win32_version")
self.header.opt.uint32("size_of_image")
self.header.opt.uint32("size_of_headers")
self.header.opt.uint32("checksum")
self.header.opt.uint16("subsystem")
self.header.opt.uint16("dll_characteristics")
self.header.opt.uint64("size_of_stack_reserve")
self.header.opt.uint64("size_of_stack_commit")
self.header.opt.uint64("size_of_heap_reserve")
self.header.opt.uint64("size_of_heap_commit")
self.header.opt.uint32("loader_flags")
self.header.opt.uint32("data_dir_count")
else:
self.valid = False
return
self.image_base = self.header.opt.image_base
self.data_dirs = self.header.array(self.header.opt.data_dir_count, "data_dirs")
for i in xrange(0, self.header.opt.data_dir_count):
self.data_dirs[i].uint32("virtual_address")
self.data_dirs[i].uint32("size")
self.sections = []
header_section_obj = PEFile.SectionInfo()
header_section_obj.virtual_size = self.header.opt.size_of_headers
header_section_obj.virtual_address = 0
header_section_obj.size_of_raw_data = self.header.opt.size_of_headers
header_section_obj.pointer_to_raw_data = 0
header_section_obj.characteristics = 0
self.sections.append(header_section_obj)
self.tree.array(self.header.section_count, "sections")
for i in xrange(0, self.header.section_count):
section = self.tree.sections[i]
section.seek(self.mz.pe_offset + self.header.optional_header_size + 24 + (i * 40))
section.bytes(8, "name")
section.uint32("virtual_size")
section.uint32("virtual_address")
section.uint32("size_of_raw_data")
section.uint32("pointer_to_raw_data")
section.uint32("pointer_to_relocs")
section.uint32("pointer_to_line_numbers")
section.uint16("reloc_count")
section.uint16("line_number_count")
section.uint32("characteristics")
section_obj = PEFile.SectionInfo()
section_obj.virtual_size = section.virtual_size
section_obj.virtual_address = section.virtual_address & ~(self.header.opt.section_align - 1)
section_obj.size_of_raw_data = section.size_of_raw_data
section_obj.pointer_to_raw_data = section.pointer_to_raw_data & ~(self.header.opt.file_align - 1)
section_obj.characteristics = section.characteristics
self.sections.append(section_obj)
self.symbols_by_name["_start"] = self.entry()
self.symbols_by_addr[self.entry()] = "_start"
if self.header.opt.data_dir_count >= 2:
self.imports = self.tree.array(0, "imports")
for i in xrange(0, self.data_dirs[1].size / 20):
if self.read(self.image_base + self.data_dirs[1].virtual_address + (i * 20), 4) == "\0\0\0\0":
break
if self.read(self.image_base + self.data_dirs[1].virtual_address + (i * 20) + 16, 4) == "\0\0\0\0":
break
self.imports.append()
dll = self.imports[i]
dll.seek(
self.virtual_address_to_file_offset(self.image_base + self.data_dirs[1].virtual_address)
+ (i * 20)
)
dll.uint32("lookup")
dll.uint32("timestamp")
dll.uint32("forward_chain")
dll.uint32("name")
dll.uint32("iat")
for dll in self.imports:
name = self.read_string(self.image_base + dll.name).split(".")
if len(name) > 1:
name = ".".join(name[0:-1])
else:
name = name[0]
entry_ofs = self.image_base + dll.lookup
iat_ofs = self.image_base + dll.iat
while True:
if self.bits == 32:
entry = self.read_uint32(entry_ofs)
is_ordinal = (entry & 0x80000000) != 0
entry &= 0x7FFFFFFF
else:
entry = self.read_uint64(entry_ofs)
is_ordinal = (entry & 0x8000000000000000) != 0
entry &= 0x7FFFFFFFFFFFFFFF
if (not is_ordinal) and (entry == 0):
break
if is_ordinal:
func = name + "!Ordinal%d" % (entry & 0xFFFF)
else:
func = name + "!" + self.read_string(self.image_base + entry + 2)
self.symbols_by_name[func] = iat_ofs
self.symbols_by_addr[iat_ofs] = func
entry_ofs += self.bits / 8
iat_ofs += self.bits / 8
if (self.header.opt.data_dir_count >= 1) and (self.data_dirs[0].size >= 40):
self.exports = self.tree.struct("Export directory", "exports")
self.exports.seek(
self.virtual_address_to_file_offset(self.image_base + self.data_dirs[0].virtual_address)
)
self.exports.uint32("characteristics")
self.exports.uint32("timestamp")
self.exports.uint16("major_version")
self.exports.uint16("minor_version")
self.exports.uint32("dll_name")
self.exports.uint32("base")
self.exports.uint32("function_count")
self.exports.uint32("name_count")
self.exports.uint32("address_of_functions")
self.exports.uint32("address_of_names")
self.exports.uint32("address_of_name_ordinals")
self.exports.array(self.exports.function_count, "functions")
for i in xrange(0, self.exports.function_count):
self.exports.functions[i].seek(
self.virtual_address_to_file_offset(self.image_base + self.exports.address_of_functions)
+ (i * 4)
)
self.exports.functions[i].uint32("address")
self.exports.array(self.exports.name_count, "names")
for i in xrange(0, self.exports.name_count):
self.exports.names[i].seek(
self.virtual_address_to_file_offset(self.image_base + self.exports.address_of_names) + (i * 4)
)
self.exports.names[i].uint32("address_of_name")
self.exports.array(self.exports.name_count, "name_ordinals")
for i in xrange(0, self.exports.name_count):
self.exports.name_ordinals[i].seek(
self.virtual_address_to_file_offset(self.image_base + self.exports.address_of_name_ordinals)
+ (i * 2)
)
self.exports.name_ordinals[i].uint16("ordinal")
for i in xrange(0, self.exports.name_count):
function_index = self.exports.name_ordinals[i].ordinal - self.exports.base
address = self.image_base + self.exports.functions[function_index].address
name = self.read_string(self.image_base + self.exports.names[i].address_of_name)
self.symbols_by_addr[address] = name
self.symbols_by_name[name] = address
self.tree.complete()
self.valid = True
except:
self.valid = False
if self.valid:
self.data.add_callback(self)
from Structure import * from MyViewer import * from math import pi, sqrt, pow structure = Structure() lb = 15.0 r = 457.2 / 2000.0 t = 10.0 / 1000.0 a = pi * (pow(r, 2) - pow(r - t, 2)) e = 2.1e11 c1 = Constraint(False, False, False) c2 = Constraint(True, True, False) f = Force(0.0, -20.0e3, -100.0e3) n1 = structure.add_node(0.0, 0.0, lb * sqrt(2.0 / 3.0)) n2 = structure.add_node(0.0, lb / sqrt(3.0), 0) n3 = structure.add_node(-lb / 2.0, -lb / sqrt(12.0), 0.0) n4 = structure.add_node(lb / 2.0, -lb / sqrt(12.0), 0.0) n1.set_force(f) n2.set_constraint(c1) n3.set_constraint(c1) n4.set_constraint(c2) structure.add_element(TrussElementLinear(e, a, n1, n2)) structure.add_element(TrussElementLinear(e, a, n1, n3)) structure.add_element(TrussElementLinear(e, a, n1, n4)) structure.add_element(TrussElementLinear(e, a, n2, n3)) structure.add_element(TrussElementLinear(e, a, n3, n4)) structure.add_element(TrussElementLinear(e, a, n4, n2)) u = structure.linear_solve() structure.print_structure() structure.print_results()
class ElfFile(BinaryAccessor):
def __init__(self, data):
self.data = data
self.valid = False
self.callbacks = []
self.symbols_by_name = {}
self.symbols_by_addr = {}
if not self.is_elf():
return
try:
self.tree = Structure(self.data)
self.header = self.tree.struct("ELF header", "header")
self.header.struct("ELF identification", "ident")
self.header.ident.uint32("magic")
self.header.ident.uint8("file_class")
self.header.ident.uint8("encoding")
self.header.ident.uint8("version")
self.header.ident.uint8("abi")
self.header.ident.uint8("abi_version")
self.header.ident.bytes(7, "pad")
self.header.uint16("type")
self.header.uint16("arch")
self.header.uint32("version")
self.symbol_table_section = None
self.dynamic_symbol_table_section = None
if self.header.ident.file_class == 1: # 32-bit
self.header.uint32("entry")
self.header.uint32("program_header_offset")
self.header.uint32("section_header_offset")
self.header.uint32("flags")
self.header.uint16("header_size")
self.header.uint16("program_header_size")
self.header.uint16("program_header_count")
self.header.uint16("section_header_size")
self.header.uint16("section_header_count")
self.header.uint16("string_table")
try:
self.sections = self.tree.array(self.header.section_header_count, "sections")
for i in range(0, self.header.section_header_count):
section = self.sections[i]
section.seek(self.header.section_header_offset + (i * 40))
section.uint32("name")
section.uint32("type")
section.uint32("flags")
section.uint32("addr")
section.uint32("offset")
section.uint32("size")
section.uint32("link")
section.uint32("info")
section.uint32("align")
section.uint32("entry_size")
if section.type == 2:
self.symbol_table_section = section
elif section.type == 11:
self.dynamic_symbol_table_section = section
except:
# Section headers are not required to load an ELF, skip errors
self.sections = self.tree.array(0, "sections")
pass
self.program_headers = self.tree.array(self.header.program_header_count, "programHeaders")
for i in range(0, self.header.program_header_count):
header = self.program_headers[i]
header.seek(self.header.program_header_offset + (i * 32))
header.uint32("type")
header.uint32("offset")
header.uint32("virtual_addr")
header.uint32("physical_addr")
header.uint32("file_size")
header.uint32("memory_size")
header.uint32("flags")
header.uint32("align")
# Parse symbol tables
self.symbols_by_name["_start"] = self.entry()
self.symbols_by_addr[self.entry()] = "_start"
try:
if self.symbol_table_section:
self.symbol_table = self.tree.array(self.symbol_table_section.size / 16, "Symbols", "symbols")
self.parse_symbol_table_32(self.symbol_table, self.symbol_table_section, self.sections[self.symbol_table_section.link])
if self.dynamic_symbol_table_section:
self.dynamic_symbol_table = self.tree.array(self.dynamic_symbol_table_section.size / 16, "Symbols", "symbols")
self.parse_symbol_table_32(self.dynamic_symbol_table, self.dynamic_symbol_table_section, self.sections[self.dynamic_symbol_table_section.link])
except:
# Skip errors in symbol table
pass
# Parse relocation tables
self.plt = {}
for section in self.sections:
if section.type == 9:
self.parse_reloc_32(section)
elif section.type == 4:
self.parse_reloca_32(section)
elif self.header.ident.file_class == 2: # 64-bit
self.header.uint64("entry")
self.header.uint64("program_header_offset")
self.header.uint64("section_header_offset")
self.header.uint32("flags")
self.header.uint16("header_size")
self.header.uint16("program_header_size")
self.header.uint16("program_header_count")
self.header.uint16("section_header_size")
self.header.uint16("section_header_count")
self.header.uint16("string_table")
try:
self.sections = self.tree.array(self.header.section_header_count, "sections")
for i in range(0, self.header.section_header_count):
section = self.sections[i]
section.seek(self.header.section_header_offset + (i * 64))
section.uint32("name")
section.uint32("type")
section.uint64("flags")
section.uint64("addr")
section.uint64("offset")
section.uint64("size")
section.uint32("link")
section.uint32("info")
section.uint64("align")
section.uint64("entry_size")
if section.type == 2:
self.symbol_table_section = section
elif section.type == 11:
self.dynamic_symbol_table_section = section
except:
# Section headers are not required to load an ELF, skip errors
self.sections = self.tree.array(0, "sections")
pass
self.program_headers = self.tree.array(self.header.program_header_count, "program_headers")
for i in range(0, self.header.program_header_count):
header = self.program_headers[i]
header.seek(self.header.program_header_offset + (i * 56))
header.uint32("type")
header.uint32("flags")
header.uint64("offset")
header.uint64("virtual_addr")
header.uint64("physical_addr")
header.uint64("file_size")
header.uint64("memory_size")
header.uint64("align")
# Parse symbol tables
self.symbols_by_name["_start"] = self.entry()
self.symbols_by_addr[self.entry()] = "_start"
try:
if self.symbol_table_section:
self.symbol_table = self.tree.array(self.symbol_table_section.size / 24, "Symbols", "symbols")
self.parse_symbol_table_64(self.symbol_table, self.symbol_table_section, self.sections[self.symbol_table_section.link])
if self.dynamic_symbol_table_section:
self.dynamic_symbol_table = self.tree.array(self.dynamic_symbol_table_section.size / 24, "Symbols", "symbols")
self.parse_symbol_table_64(self.dynamic_symbol_table, self.dynamic_symbol_table_section, self.sections[self.dynamic_symbol_table_section.link])
except:
# Skip errors in symbol table
pass
# Parse relocation tables
self.plt = {}
for section in self.sections:
if section.type == 9:
self.parse_reloc_64(section)
elif section.type == 4:
self.parse_reloca_64(section)
self.tree.complete()
self.valid = True
except:
self.valid = False
if self.valid:
self.data.add_callback(self)
def read_string_table(self, strings, offset):
end = strings.find("\x00", offset)
return strings[offset:end]
def parse_symbol_table_32(self, table, section, string_table):
strings = self.data.read(string_table.offset, string_table.size)
for i in range(0, section.size / 16):
table[i].seek(section.offset + (i * 16))
table[i].uint32("name_offset")
table[i].uint32("value")
table[i].uint32("size")
table[i].uint8("info")
table[i].uint8("other")
table[i].uint16("section")
table[i].name = self.read_string_table(strings, table[i].name_offset)
if len(table[i].name) > 0:
self.symbols_by_name[table[i].name] = table[i].value
self.symbols_by_addr[table[i].value] = table[i].name
def parse_symbol_table_64(self, table, section, string_table):
strings = self.data.read(string_table.offset, string_table.size)
for i in range(0, section.size / 24):
table[i].seek(section.offset + (i * 24))
table[i].uint32("name_offset")
table[i].uint8("info")
table[i].uint8("other")
table[i].uint16("section")
table[i].uint64("value")
table[i].uint64("size")
table[i].name = self.read_string_table(strings, table[i].name_offset)
if len(table[i].name) > 0:
self.symbols_by_name[table[i].name] = table[i].value
self.symbols_by_addr[table[i].value] = table[i].name
def parse_reloc_32(self, section):
for i in range(0, section.size / 8):
ofs = self.data.read_uint32(section.offset + (i * 8))
info = self.data.read_uint32(section.offset + (i * 8) + 4)
sym = info >> 8
reloc_type = info & 0xff
if reloc_type == 7: # R_386_JUMP_SLOT
self.plt[ofs] = self.dynamic_symbol_table[sym].name
self.symbols_by_name[self.decorate_plt_name(self.dynamic_symbol_table[sym].name)] = ofs
self.symbols_by_addr[ofs] = self.decorate_plt_name(self.dynamic_symbol_table[sym].name)
def parse_reloca_32(self, section):
for i in range(0, section.size / 12):
ofs = self.data.read_uint32(section.offset + (i * 12))
info = self.data.read_uint32(section.offset + (i * 12) + 4)
sym = info >> 8
reloc_type = info & 0xff
if reloc_type == 7: # R_386_JUMP_SLOT
self.plt[ofs] = self.dynamic_symbol_table[sym].name
self.symbols_by_name[self.decorate_plt_name(self.dynamic_symbol_table[sym].name)] = ofs
self.symbols_by_addr[ofs] = self.decorate_plt_name(self.dynamic_symbol_table[sym].name)
def parse_reloc_64(self, section):
for i in range(0, section.size / 16):
ofs = self.data.read_uint64(section.offset + (i * 16))
info = self.data.read_uint64(section.offset + (i * 16) + 8)
sym = info >> 32
reloc_type = info & 0xff
if reloc_type == 7: # R_X86_64_JUMP_SLOT
self.plt[ofs] = self.dynamic_symbol_table[sym].name
self.symbols_by_name[self.decorate_plt_name(self.dynamic_symbol_table[sym].name)] = ofs
self.symbols_by_addr[ofs] = self.decorate_plt_name(self.dynamic_symbol_table[sym].name)
def parse_reloca_64(self, section):
for i in range(0, section.size / 24):
ofs = self.data.read_uint64(section.offset + (i * 24))
info = self.data.read_uint64(section.offset + (i * 24) + 8)
sym = info >> 32
reloc_type = info & 0xff
if reloc_type == 7: # R_X86_64_JUMP_SLOT
self.plt[ofs] = self.dynamic_symbol_table[sym].name
self.symbols_by_name[self.decorate_plt_name(self.dynamic_symbol_table[sym].name)] = ofs
self.symbols_by_addr[ofs] = self.decorate_plt_name(self.dynamic_symbol_table[sym].name)
def read(self, ofs, len):
result = ""
while len > 0:
cur = None
for i in self.program_headers:
if ((ofs >= i.virtual_addr) and (ofs < (i.virtual_addr + i.memory_size))) and (i.memory_size != 0):
cur = i
if cur == None:
break
prog_ofs = ofs - cur.virtual_addr
mem_len = cur.memory_size - prog_ofs
file_len = cur.file_size - prog_ofs
if mem_len > len:
mem_len = len
if file_len > len:
file_len = len
if file_len <= 0:
result += "\x00" * mem_len
len -= mem_len
ofs += mem_len
continue
result += self.data.read(cur.offset + prog_ofs, file_len)
len -= file_len
ofs += file_len
return result
def next_valid_addr(self, ofs):
result = -1
for i in self.program_headers:
if (i.virtual_addr >= ofs) and (i.memory_size != 0) and ((result == -1) or (i.virtual_addr < result)):
result = i.virtual_addr
return result
def get_modification(self, ofs, len):
result = []
while len > 0:
cur = None
for i in self.program_headers:
if ((ofs >= i.virtual_addr) and (ofs < (i.virtual_addr + i.memory_size))) and (i.memory_size != 0):
cur = i
if cur == None:
break
prog_ofs = ofs - cur.virtual_addr
mem_len = cur.memory_size - prog_ofs
file_len = cur.file_size - prog_ofs
if mem_len > len:
mem_len = len
if file_len > len:
file_len = len
if file_len <= 0:
result += [DATA_ORIGINAL] * mem_len
len -= mem_len
ofs += mem_len
continue
result += self.data.get_modification(cur.offset + prog_ofs, file_len)
len -= file_len
ofs += file_len
return result
def write(self, ofs, data):
result = 0
while len(data) > 0:
cur = None
for i in self.program_headers:
if ((ofs >= i.virtual_addr) and (ofs < (i.virtual_addr + i.memory_size))) and (i.memory_size != 0):
cur = i
if cur == None:
break
prog_ofs = ofs - cur.virtual_addr
mem_len = cur.memory_size - prog_ofs
file_len = cur.file_size - prog_ofs
if mem_len > len:
mem_len = len
if file_len > len:
file_len = len
if file_len <= 0:
break
result += self.data.write(cur.offset + prog_ofs, data[0:file_len])
data = data[file_len:]
ofs += file_len
return result
def insert(self, ofs, data):
return 0
def remove(self, ofs, size):
return 0
def notify_data_write(self, data, ofs, contents):
# Find sections that hold data backed by updated regions of the file
for i in self.program_headers:
if ((ofs + len(contents)) > i.offset) and (ofs < (i.offset + i.file_size)) and (i.memory_size != 0):
# This section has been updated, compute which region has been changed
from_start = ofs - i.offset
data_ofs = 0
length = len(contents)
if from_start < 0:
length += from_start
data_ofs -= from_start
from_start = 0
if (from_start + length) > i.file_size:
length = i.file_size - from_start
# Notify callbacks
if length > 0:
for cb in self.callbacks:
if hasattr(cb, "notify_data_write"):
cb.notify_data_write(self, i.virtual_addr + from_start,
contents[data_ofs:(data_ofs + length)])
def save(self, filename):
self.data.save(filename)
def start(self):
result = None
for i in self.program_headers:
if ((result == None) or (i.virtual_addr < result)) and (i.memory_size != 0):
result = i.virtual_addr
return result
def entry(self):
return self.header.entry
def __len__(self):
max = None
for i in self.program_headers:
if ((max == None) or ((i.virtual_addr + i.memory_size) > max)) and (i.memory_size != 0):
max = i.virtual_addr + i.memory_size
return max - self.start()
def is_elf(self):
return self.data.read(0, 4) == "\x7fELF"
def architecture(self):
if self.header.arch == 2:
return "sparc"
if self.header.arch == 3:
return "x86"
if self.header.arch == 4:
return "m68000"
if self.header.arch == 8:
return "mips"
if self.header.arch == 15:
return "pa_risc"
if self.header.arch == 18:
return "sparc_32plus"
if self.header.arch == 20:
return "ppc"
if self.header.arch == 40:
return "arm"
if self.header.arch == 41:
return "alpha"
if self.header.arch == 43:
return "sparc_v9"
if self.header.arch == 62:
return "x86_64"
return None
def decorate_plt_name(self, name):
return name + "@PLT"
def create_symbol(self, addr, name):
self.symbols_by_name[name] = addr
self.symbols_by_addr[addr] = name
def delete_symbol(self, addr, name):
if name in self.symbols_by_name:
del(self.symbols_by_name[name])
if addr in self.symbols_by_addr:
del(self.symbols_by_addr[addr])
def add_callback(self, cb):
self.callbacks.append(cb)
def remove_callback(self, cb):
self.callbacks.remove(cb)
def is_modified(self):
return self.data.is_modified()
def find(self, regex, addr):
while (addr < self.end()) and (addr != -1):
data = self.read(addr, 0xfffffffff)
match = regex.search(data)
if match != None:
return match.start() + addr
addr += len(data)
addr = self.next_valid_addr(addr)
return -1
def has_undo_actions(self):
return self.data.has_undo_actions()
def commit_undo(self, before_loc, after_loc):
self.data.commit_undo(before_loc, after_loc)
def undo(self):
self.data.undo()
def redo(self):
self.data.redo()
# We will instantiate a number of Structure() objects to construct the ensemble
ensemble = []
for i in range(nclusters):
print
print '#### STRUCTURE %d ####'%i
# no information from QM --> lam = 0.0
# QM + exp --> lam = 1.0
## s = Structure('gens-pdb-kcenters-dih-1.8/Gen%d.pdb'%i, args.lam*energies[i], expdata_filename, use_log_normal_distances=False)
model_distances = loadtxt('NOE/rminus6_whole_state%d.txt'%i)*10.0 # convert to A
# model_chemicalshift_H = loadtxt('chemical_shift/cs/H/cs_%d.txt'%i)
# model_chemicalshift_Ha = loadtxt('chemical_shift/cs/Ha/cs_%d.txt'%i)
# s = Structure('Gens/Gens%d.pdb'%i, args.lam*energies[i], expdata_filename_noe, expdata_filename_J, expdata_filename_cs, use_log_normal_distances=False, dloggamma=np.log(1.01), gamma_min=0.2, gamma_max=5.0)
s = Structure('pdb/state_%s.pdb'%str(i).zfill(2), args.lam*energies[i], expdata_filename_noe=expdata_filename_noe, use_log_normal_distances=False, dloggamma=np.log(1.01), gamma_min=0.2, gamma_max=5.0, dalpha=0.1, alpha_min=0.0, alpha_max=0.1)
# s = Structure('Gens/Gens%d.pdb'%i, args.lam*energies[i], expdata_filename_cs_H=expdata_filename_cs_H, expdata_filename_cs_Ha=expdata_filename_cs_Ha, use_log_normal_distances=False, dloggamma=np.log(1.01), gamma_min=0.2, gamma_max=5.0, dalpha=0.1, alpha_min=0.0, alpha_max=0.1)
# s = Structure('Gens/Gens%d.pdb'%i, args.lam*energies[i], expdata_filename_noe=expdata_filename_noe, use_log_normal_distances=False, dloggamma=np.log(1.01), gamma_min=0.2, gamma_max=5.0)
# NOTE: Upon instantiation, each Structure() object computes the distances from the given PDB.
# However, our clusters represent averaged conformation states, and so we
# need to replace these values with our own r^-6-averaged, precomputed ones
# replace PDB distances with r^-6-averaged distances
# print 'len(s.distance_restraints)', len(s.distance_restraints)
for j in range(len(s.distance_restraints)):
# print s.distance_restraints[j].i, s.distance_restraints[j].j, model_distances[j]
s.distance_restraints[j].model_distance = model_distances[j]
# replace chemicalshift with precomputed ones
# print 'len(s.chemicalshift_H_restraints)', len(s.chemicalshift_H_restraints)
# We will instantiate a number of Structure() objects to construct the ensemble
ensemble = []
for i in range(nclusters+1):
print
print '#### STRUCTURE %d ####'%i
# no information from QM --> lam = 0.0
# QM + exp --> lam = 1.0
## s = Structure('gens-pdb-kcenters-dih-1.8/Gen%d.pdb'%i, args.lam*energies[i], expdata_filename, use_log_normal_distances=False)
model_distances = loadtxt('NOE/rminus6_whole_state%d.txt'%i)*10.0 # convert to A
model_chemicalshift = loadtxt('chemical_shift/cs/cs_%d.txt'%i)
# s = Structure('Gens/Gens%d.pdb'%i, args.lam*energies[i], expdata_filename_noe, expdata_filename_J, expdata_filename_cs, use_log_normal_distances=False, dloggamma=np.log(1.01), gamma_min=0.2, gamma_max=5.0)
s = Structure('Gens/Gens%d.pdb'%i, args.lam*energies[i], expdata_filename_noe=expdata_filename_noe,expdata_filename_cs=expdata_filename_cs, use_log_normal_distances=False, dloggamma=np.log(1.01), gamma_min=0.2, gamma_max=5.0)
# s = Structure('Gens/Gens%d.pdb'%i, args.lam*energies[i], expdata_filename_noe=expdata_filename_noe, use_log_normal_distances=False, dloggamma=np.log(1.01), gamma_min=0.2, gamma_max=5.0)
# NOTE: Upon instantiation, each Structure() object computes the distances from the given PDB.
# However, our clusters represent averaged conformation states, and so we
# need to replace these values with our own r^-6-averaged, precomputed ones
# replace PDB distances with r^-6-averaged distances
print 'len(s.distance_restraints)', len(s.distance_restraints)
for j in range(len(s.distance_restraints)):
print s.distance_restraints[j].i, s.distance_restraints[j].j, model_distances[j]
s.distance_restraints[j].model_distance = model_distances[j]
# replace chemicalshift with precomputed ones
print 'len(s.chemicalshift_restraints)', len(s.chemicalshift_restraints)
for j in range(len(s.chemicalshift_restraints)):
class PEFile(BinaryAccessor):
class SectionInfo:
def __init__(self):
self.virtual_size = None
self.virtual_address = None
self.size_of_raw_data = None
self.pointer_to_raw_data = None
self.characteristics = None
def __init__(self, data):
self.data = data
self.valid = False
self.callbacks = []
self.symbols_by_name = {}
self.symbols_by_addr = {}
if not self.is_pe():
return
try:
self.tree = Structure(self.data)
self.mz = self.tree.struct("MZ header", "mz")
self.mz.uint16("magic")
self.mz.uint16("lastsize")
self.mz.uint16("nblocks")
self.mz.uint16("nreloc")
self.mz.uint16("hdrsize")
self.mz.uint16("minalloc")
self.mz.uint16("maxalloc")
self.mz.uint16("ss")
self.mz.uint16("sp")
self.mz.uint16("checksum")
self.mz.uint16("ip")
self.mz.uint16("cs")
self.mz.uint16("relocpos")
self.mz.uint16("noverlay")
self.mz.bytes(8, "reserved1")
self.mz.uint16("oem_id")
self.mz.uint16("oem_info")
self.mz.bytes(20, "reserved2")
self.mz.uint32("pe_offset")
self.header = self.tree.struct("PE header", "header")
self.header.seek(self.mz.pe_offset)
self.header.uint32("magic")
self.header.uint16("machine")
self.header.uint16("section_count")
self.header.uint32("timestamp")
self.header.uint32("coff_symbol_table")
self.header.uint32("coff_symbol_count")
self.header.uint16("optional_header_size")
self.header.uint16("characteristics")
self.header.struct("Optional header", "opt")
self.header.opt.uint16("magic")
self.header.opt.uint8("major_linker_version")
self.header.opt.uint8("minor_linker_version")
self.header.opt.uint32("size_of_code")
self.header.opt.uint32("size_of_init_data")
self.header.opt.uint32("size_of_uninit_data")
self.header.opt.uint32("address_of_entry")
self.header.opt.uint32("base_of_code")
if self.header.opt.magic == 0x10B: # 32-bit
self.bits = 32
self.header.opt.uint32("base_of_data")
self.header.opt.uint32("image_base")
self.header.opt.uint32("section_align")
self.header.opt.uint32("file_align")
self.header.opt.uint16("major_os_version")
self.header.opt.uint16("minor_os_version")
self.header.opt.uint16("major_image_version")
self.header.opt.uint16("minor_image_version")
self.header.opt.uint16("major_subsystem_version")
self.header.opt.uint16("minor_subsystem_version")
self.header.opt.uint32("win32_version")
self.header.opt.uint32("size_of_image")
self.header.opt.uint32("size_of_headers")
self.header.opt.uint32("checksum")
self.header.opt.uint16("subsystem")
self.header.opt.uint16("dll_characteristics")
self.header.opt.uint32("size_of_stack_reserve")
self.header.opt.uint32("size_of_stack_commit")
self.header.opt.uint32("size_of_heap_reserve")
self.header.opt.uint32("size_of_heap_commit")
self.header.opt.uint32("loader_flags")
self.header.opt.uint32("data_dir_count")
elif self.header.opt.magic == 0x20B: # 64-bit
self.bits = 64
self.header.opt.uint64("image_base")
self.header.opt.uint32("section_align")
self.header.opt.uint32("file_align")
self.header.opt.uint16("major_os_version")
self.header.opt.uint16("minor_os_version")
self.header.opt.uint16("major_image_version")
self.header.opt.uint16("minor_image_version")
self.header.opt.uint16("major_subsystem_version")
self.header.opt.uint16("minor_subsystem_version")
self.header.opt.uint32("win32_version")
self.header.opt.uint32("size_of_image")
self.header.opt.uint32("size_of_headers")
self.header.opt.uint32("checksum")
self.header.opt.uint16("subsystem")
self.header.opt.uint16("dll_characteristics")
self.header.opt.uint64("size_of_stack_reserve")
self.header.opt.uint64("size_of_stack_commit")
self.header.opt.uint64("size_of_heap_reserve")
self.header.opt.uint64("size_of_heap_commit")
self.header.opt.uint32("loader_flags")
self.header.opt.uint32("data_dir_count")
else:
self.valid = False
return
self.image_base = self.header.opt.image_base
self.data_dirs = self.header.array(self.header.opt.data_dir_count, "data_dirs")
for i in xrange(0, self.header.opt.data_dir_count):
self.data_dirs[i].uint32("virtual_address")
self.data_dirs[i].uint32("size")
self.sections = []
header_section_obj = PEFile.SectionInfo()
header_section_obj.virtual_size = self.header.opt.size_of_headers
header_section_obj.virtual_address = 0
header_section_obj.size_of_raw_data = self.header.opt.size_of_headers
header_section_obj.pointer_to_raw_data = 0
header_section_obj.characteristics = 0
self.sections.append(header_section_obj)
self.tree.array(self.header.section_count, "sections")
for i in xrange(0, self.header.section_count):
section = self.tree.sections[i]
section.seek(self.mz.pe_offset + self.header.optional_header_size + 24 + (i * 40))
section.bytes(8, "name")
section.uint32("virtual_size")
section.uint32("virtual_address")
section.uint32("size_of_raw_data")
section.uint32("pointer_to_raw_data")
section.uint32("pointer_to_relocs")
section.uint32("pointer_to_line_numbers")
section.uint16("reloc_count")
section.uint16("line_number_count")
section.uint32("characteristics")
section_obj = PEFile.SectionInfo()
section_obj.virtual_size = section.virtual_size
section_obj.virtual_address = section.virtual_address & ~(self.header.opt.section_align - 1)
section_obj.size_of_raw_data = section.size_of_raw_data
section_obj.pointer_to_raw_data = section.pointer_to_raw_data & ~(self.header.opt.file_align - 1)
section_obj.characteristics = section.characteristics
self.sections.append(section_obj)
self.symbols_by_name["_start"] = self.entry()
self.symbols_by_addr[self.entry()] = "_start"
if self.header.opt.data_dir_count >= 2:
self.imports = self.tree.array(0, "imports")
for i in xrange(0, self.data_dirs[1].size / 20):
if self.read(self.image_base + self.data_dirs[1].virtual_address + (i * 20), 4) == "\0\0\0\0":
break
if self.read(self.image_base + self.data_dirs[1].virtual_address + (i * 20) + 16, 4) == "\0\0\0\0":
break
self.imports.append()
dll = self.imports[i]
dll.seek(
self.virtual_address_to_file_offset(self.image_base + self.data_dirs[1].virtual_address)
+ (i * 20)
)
dll.uint32("lookup")
dll.uint32("timestamp")
dll.uint32("forward_chain")
dll.uint32("name")
dll.uint32("iat")
for dll in self.imports:
name = self.read_string(self.image_base + dll.name).split(".")
if len(name) > 1:
name = ".".join(name[0:-1])
else:
name = name[0]
entry_ofs = self.image_base + dll.lookup
iat_ofs = self.image_base + dll.iat
while True:
if self.bits == 32:
entry = self.read_uint32(entry_ofs)
is_ordinal = (entry & 0x80000000) != 0
entry &= 0x7FFFFFFF
else:
entry = self.read_uint64(entry_ofs)
is_ordinal = (entry & 0x8000000000000000) != 0
entry &= 0x7FFFFFFFFFFFFFFF
if (not is_ordinal) and (entry == 0):
break
if is_ordinal:
func = name + "!Ordinal%d" % (entry & 0xFFFF)
else:
func = name + "!" + self.read_string(self.image_base + entry + 2)
self.symbols_by_name[func] = iat_ofs
self.symbols_by_addr[iat_ofs] = func
entry_ofs += self.bits / 8
iat_ofs += self.bits / 8
if (self.header.opt.data_dir_count >= 1) and (self.data_dirs[0].size >= 40):
self.exports = self.tree.struct("Export directory", "exports")
self.exports.seek(
self.virtual_address_to_file_offset(self.image_base + self.data_dirs[0].virtual_address)
)
self.exports.uint32("characteristics")
self.exports.uint32("timestamp")
self.exports.uint16("major_version")
self.exports.uint16("minor_version")
self.exports.uint32("dll_name")
self.exports.uint32("base")
self.exports.uint32("function_count")
self.exports.uint32("name_count")
self.exports.uint32("address_of_functions")
self.exports.uint32("address_of_names")
self.exports.uint32("address_of_name_ordinals")
self.exports.array(self.exports.function_count, "functions")
for i in xrange(0, self.exports.function_count):
self.exports.functions[i].seek(
self.virtual_address_to_file_offset(self.image_base + self.exports.address_of_functions)
+ (i * 4)
)
self.exports.functions[i].uint32("address")
self.exports.array(self.exports.name_count, "names")
for i in xrange(0, self.exports.name_count):
self.exports.names[i].seek(
self.virtual_address_to_file_offset(self.image_base + self.exports.address_of_names) + (i * 4)
)
self.exports.names[i].uint32("address_of_name")
self.exports.array(self.exports.name_count, "name_ordinals")
for i in xrange(0, self.exports.name_count):
self.exports.name_ordinals[i].seek(
self.virtual_address_to_file_offset(self.image_base + self.exports.address_of_name_ordinals)
+ (i * 2)
)
self.exports.name_ordinals[i].uint16("ordinal")
for i in xrange(0, self.exports.name_count):
function_index = self.exports.name_ordinals[i].ordinal - self.exports.base
address = self.image_base + self.exports.functions[function_index].address
name = self.read_string(self.image_base + self.exports.names[i].address_of_name)
self.symbols_by_addr[address] = name
self.symbols_by_name[name] = address
self.tree.complete()
self.valid = True
except:
self.valid = False
if self.valid:
self.data.add_callback(self)
def read_string(self, addr):
result = ""
while True:
ch = self.read(addr, 1)
addr += 1
if (len(ch) == 0) or (ch == "\0"):
break
result += ch
return result
def virtual_address_to_file_offset(self, addr):
for i in self.sections:
if (
(addr >= (self.image_base + i.virtual_address))
and (addr < (self.image_base + i.virtual_address + i.virtual_size))
) and (i.virtual_size != 0):
cur = i
if cur == None:
return None
ofs = addr - (self.image_base + cur.virtual_address)
return cur.pointer_to_raw_data + ofs
def read(self, ofs, len):
result = ""
while len > 0:
cur = None
for i in self.sections:
if (
(ofs >= (self.image_base + i.virtual_address))
and (ofs < (self.image_base + i.virtual_address + i.virtual_size))
) and (i.virtual_size != 0):
cur = i
if cur == None:
break
prog_ofs = ofs - (self.image_base + cur.virtual_address)
mem_len = cur.virtual_size - prog_ofs
file_len = cur.size_of_raw_data - prog_ofs
if mem_len > len:
mem_len = len
if file_len > len:
file_len = len
if file_len <= 0:
result += "\x00" * mem_len
len -= mem_len
ofs += mem_len
continue
result += self.data.read(cur.pointer_to_raw_data + prog_ofs, file_len)
len -= file_len
ofs += file_len
return result
def next_valid_addr(self, ofs):
result = -1
for i in self.sections:
if (
((self.image_base + i.virtual_address) >= ofs)
and (i.virtual_size != 0)
and ((result == -1) or ((self.image_base + i.virtual_address) < result))
):
result = self.image_base + i.virtual_address
return result
def get_modification(self, ofs, len):
result = []
while len > 0:
cur = None
for i in self.sections:
if (
(ofs >= (self.image_base + i.virtual_address))
and (ofs < (self.image_base + i.virtual_address + i.virtual_size))
) and (i.virtual_size != 0):
cur = i
if cur == None:
break
prog_ofs = ofs - (self.image_base + cur.virtual_address)
mem_len = cur.virtual_size - prog_ofs
file_len = cur.size_of_raw_data - prog_ofs
if mem_len > len:
mem_len = len
if file_len > len:
file_len = len
if file_len <= 0:
result += [DATA_ORIGINAL] * mem_len
len -= mem_len
ofs += mem_len
continue
result += self.data.get_modification(cur.pointer_to_raw_data + prog_ofs, file_len)
len -= file_len
ofs += file_len
return result
def write(self, ofs, data):
result = 0
while len(data) > 0:
cur = None
for i in self.sections:
if (
(ofs >= (self.image_base + i.virtual_address))
and (ofs < (self.image_base + i.virtual_address + i.virtual_size))
) and (i.virtual_size != 0):
cur = i
if cur == None:
break
prog_ofs = ofs - (self.image_base + cur.virtual_address)
mem_len = cur.virtual_size - prog_ofs
file_len = cur.size_of_raw_data - prog_ofs
if mem_len > len:
mem_len = len
if file_len > len:
file_len = len
if file_len <= 0:
break
result += self.data.write(cur.pointer_to_raw_data + prog_ofs, data[0:file_len])
data = data[file_len:]
ofs += file_len
return result
def insert(self, ofs, data):
return 0
def remove(self, ofs, size):
return 0
def notify_data_write(self, data, ofs, contents):
# Find sections that hold data backed by updated regions of the file
for i in self.sections:
if (
((ofs + len(contents)) > i.pointer_to_raw_data)
and (ofs < (i.pointer_to_raw_data + i.size_of_raw_data))
and (i.virtual_size != 0)
):
# This section has been updated, compute which region has been changed
from_start = ofs - i.pointer_to_raw_data
data_ofs = 0
length = len(contents)
if from_start < 0:
length += from_start
data_ofs -= from_start
from_start = 0
if (from_start + length) > i.size_of_raw_data:
length = i.size_of_raw_data - from_start
# Notify callbacks
if length > 0:
for cb in self.callbacks:
if hasattr(cb, "notify_data_write"):
cb.notify_data_write(
self,
self.image_base + i.virtual_address + from_start,
contents[data_ofs : (data_ofs + length)],
)
def save(self, filename):
self.data.save(filename)
def start(self):
return self.image_base
def entry(self):
return self.image_base + self.header.opt.address_of_entry
def __len__(self):
max = None
for i in self.sections:
if ((max == None) or ((self.image_base + i.virtual_address + i.virtual_size) > max)) and (
i.virtual_size != 0
):
max = self.image_base + i.virtual_address + i.virtual_size
return max - self.start()
def is_pe(self):
if self.data.read(0, 2) != "MZ":
return False
ofs = self.data.read(0x3C, 4)
if len(ofs) != 4:
return False
ofs = struct.unpack("<I", ofs)[0]
if self.data.read(ofs, 4) != "PE\0\0":
return False
magic = self.data.read(ofs + 24, 2)
if len(magic) != 2:
return False
magic = struct.unpack("<H", magic)[0]
return (magic == 0x10B) or (magic == 0x20B)
def architecture(self):
if self.header.machine == 0x14C:
return "x86"
if self.header.machine == 0x8664:
return "x86_64"
if self.header.machine == 0x166:
return "mips"
if self.header.machine == 0x266:
return "mips16"
if self.header.machine == 0x366:
return "mips"
if self.header.machine == 0x466:
return "mips16"
if self.header.machine == 0x1F0:
return "ppc"
if self.header.machine == 0x1F1:
return "ppc"
if self.header.machine == 0x1C0:
return "arm"
if self.header.machine == 0x1C2:
return "thumb"
if self.header.machine == 0x1C4:
return "thumb"
if self.header.machine == 0xAA64:
return "arm64"
if self.header.machine == 0x200:
return "ia64"
return None
def decorate_plt_name(self, name):
return name + "@IAT"
def create_symbol(self, addr, name):
self.symbols_by_name[name] = addr
self.symbols_by_addr[addr] = name
def delete_symbol(self, addr, name):
if name in self.symbols_by_name:
del (self.symbols_by_name[name])
if addr in self.symbols_by_addr:
del (self.symbols_by_addr[addr])
def add_callback(self, cb):
self.callbacks.append(cb)
def remove_callback(self, cb):
self.callbacks.remove(cb)
def is_modified(self):
return self.data.is_modified()
def find(self, regex, addr):
while (addr < self.end()) and (addr != -1):
data = self.read(addr, 0xFFFFFFFFF)
match = regex.search(data)
if match != None:
return match.start() + addr
addr += len(data)
addr = self.next_valid_addr(addr)
return -1
def has_undo_actions(self):
return self.data.has_undo_actions()
def commit_undo(self, before_loc, after_loc):
self.data.commit_undo(before_loc, after_loc)
def undo(self):
self.data.undo()
def redo(self):
self.data.redo()
def calc_covBP_from_sto(stoFn,
querySeq=None,
query_id_pattern=None,
full_seq=None,
allpair=False,
min_seqs=8,
min_score=0.4):
"""
Calculate the RNAalignfold_stack score for all base pairs in stockholm
Please note that the refseq in stockholm file should be consistent with querySeq you provide
stoFn -- Stockholm file
querySeq -- Query sequence, the reference sequence (#=GC RF) will aligned with querySeq
If no querySeq provided, the ID match query_id_pattern will used as query sequence
query_id_pattern -- The query ID mattern, if querySeq is provided, will be ignored
full_seq -- If full_seq is provide and querySeq=None, the sequence index by query_id_pattern will
be search in full_seq, the left and right index will be re-indexed
allpair -- Calculate all base pairs, not only pairs in structure
min_seqs -- Minimum sequences in the alignments
min_score -- Minimum score to record
Return [ [left, right, score],.... ]
"""
import Structure, re
id2seq_dict, refStr, refSeq = General.load_stockholm(stoFn)[0]
if len(id2seq_dict) < min_seqs:
return []
columns = collect_columns([value for key, value in id2seq_dict.items()])
alignLen = len(refSeq)
if allpair:
ct = []
for i in range(alignLen):
for j in range(i + 1, alignLen):
ct.append([i, j])
else:
ct = Structure.dot2ct(refStr)
covary_bps = []
for b1, b2 in ct:
rnaalignscore2 = calc_RNAalignfold_stack(columns, b1, b2)
if rnaalignscore2 > min_score:
covary_bps.append([b1, b2, round(rnaalignscore2, 3)])
refSeq = refSeq.replace('~', '-').replace(':', '-').replace(
'.', '-').upper().replace('U', 'T')
if querySeq:
refSeq_realigned, query_aligned = Structure.multi_alignment(
[refSeq.replace('-', ''),
querySeq.upper().replace('U', 'T')])
### 构建一个字典：从refSeq的坐标指向querySeq的坐标
refseq2query = {}
pos_ref, pos_refrealign = 0, 0
query_pos = 0
while pos_ref < len(refSeq):
while pos_ref < len(refSeq) and refSeq[pos_ref] == '-':
pos_ref += 1
if pos_ref == len(refSeq): break
while refSeq_realigned[pos_refrealign] == '-':
if query_aligned[pos_refrealign] != '-':
query_pos += 1
pos_refrealign += 1
#if pos_refrealign==len(query_aligned): break
refseq2query[pos_ref + 1] = query_pos + 1
pos_ref += 1
pos_refrealign += 1
query_pos += 1
i = 0
while i < len(covary_bps):
if covary_bps[i][0] in refseq2query and covary_bps[i][
1] in refseq2query:
covary_bps[i][0] = refseq2query[covary_bps[i][0]]
covary_bps[i][1] = refseq2query[covary_bps[i][1]]
i += 1
else:
del covary_bps[i]
elif query_id_pattern:
tid_list = [
name for name in id2seq_dict.keys()
if re.match(query_id_pattern, name)
]
assert len(tid_list) == 1, f"Error: {len(tid_list)} tid match pattern"
query_tid = tid_list[0]
clean_query_seq = id2seq_dict[query_tid].replace('~', '').replace(
':', '').replace('.', '').replace('-',
'').upper().replace('U', 'T')
if full_seq:
full_seq = full_seq.replace('U', 'T')
start_pos = full_seq.find(clean_query_seq)
if start_pos == -1:
raise RuntimeError('queryseq not in full_seq')
if full_seq.find(clean_query_seq, start_pos + 1) != -1:
raise RuntimeError('queryseq are find in multiple position')
else:
start_pos = 0
aligned_query_seq = id2seq_dict[query_tid].upper().replace('U', 'T')
alignedPos2cleanPos = get_alignedPos2cleanPos_dict(
id2seq_dict[query_tid])
i = 0
while i < len(covary_bps):
l, r = covary_bps[i][:2]
if alignedPos2cleanPos[l] and alignedPos2cleanPos[r]:
nl = alignedPos2cleanPos[l] + start_pos
nr = alignedPos2cleanPos[r] + start_pos
covary_bps[i][0] = nl
covary_bps[i][1] = nr
if full_seq:
assert aligned_query_seq[l - 1] == full_seq[
nl - 1], "Exception: seq not same with full_seq"
assert aligned_query_seq[r - 1] == full_seq[
nr - 1], "Exception: seq not same with full_seq"
i += 1
else:
del covary_bps[i]
else:
raise RuntimeError('querySeq or query_id_pattern must be specified')
return covary_bps
## s = Structure('gens-pdb-kcenters-dih-1.8/Gen%d.pdb'%i, args.lam*energies[i], expdata_filename, use_log_normal_distances=False)
# model_protectionfactor = loadtxt('FH/pf_state%d.txt'%i)
# model_protectionfactor = loadtxt('../HDX/PF/test/pf_state%d.txt'%i)
model_protectionfactor = loadtxt('new_traj/PF/T5/pf_state%d.txt'%i)
model_chemicalshift_H = loadtxt('new_traj/H/T5/cs_%d.txt'%i)
model_chemicalshift_N = loadtxt('new_traj/N/T5/cs_%d.txt'%i)
model_chemicalshift_Ca = loadtxt('new_traj/Ca/T5/cs_%d.txt'%i)
# model_chemicalshift_H = loadtxt('EGH/H/cs_%d.txt'%i)
# model_chemicalshift_N = loadtxt('EGH/N/cs_%d.txt'%i)
# model_chemicalshift_Ca = loadtxt('EGH/Ca/cs_%d.txt'%i)
# s = Structure('Gens/Gens%d.pdb'%i, args.lam*energies[i], expdata_filename_noe, expdata_filename_J, expdata_filename_cs, use_log_normal_distances=False, dloggamma=np.log(1.01), gamma_min=0.2, gamma_max=5.0)
# s = Structure('new_traj/pdb/state%d.pdb'%i, args.lam*energies[i], expdata_filename_cs_H=expdata_filename_cs_H, expdata_filename_cs_N=expdata_filename_cs_N,expdata_filename_cs_Ca=expdata_filename_cs_Ca, use_log_normal_distances=False, dloggamma=np.log(1.01), gamma_min=0.2, gamma_max=5.0, dalpha=0.1, alpha_min=0.1, alpha_max=5.0) #GYH
# s = Structure('../HDX/states/test/state%d.pdb'%i, args.lam*energies[i], expdata_filename_PF=expdata_filename_PF, use_log_normal_distances=False, dloggamma=np.log(1.01), gamma_min=0.2, gamma_max=5.0, dalpha=0.001, alpha_min=3.831, alpha_max=3.832) #GYH
s = Structure('new_traj/pdb/T5/state%d.pdb'%i, args.lam*energies[i], expdata_filename_cs_H=expdata_filename_cs_H, expdata_filename_cs_N=expdata_filename_cs_N,expdata_filename_cs_Ca=expdata_filename_cs_Ca, expdata_filename_PF=expdata_filename_PF, use_log_normal_distances=False, dloggamma=np.log(1.01), gamma_min=0.2, gamma_max=5.0, dalpha=0.1, alpha_min=0.0, alpha_max=10.0) #GYH
# s = Structure('new_traj/pdb/state%d.pdb'%i, args.lam*energies[i], expdata_filename_PF=expdata_filename_PF, use_log_normal_distances=False, dloggamma=np.log(1.01), gamma_min=0.2, gamma_max=5.0, dalpha=0.1, alpha_min=-10.0, alpha_max=10.0) #GYH
# s = Structure('Gens/Gens%d.pdb'%i, args.lam*energies[i], expdata_filename_noe=expdata_filename_noe, use_log_normal_distances=False, dloggamma=np.log(1.01), gamma_min=0.2, gamma_max=5.0)
# NOTE: Upon instantiation, each Structure() object computes the distances from the given PDB.
# However, our clusters represent averaged conformation states, and so we
# need to replace these values with our own r^-6-averaged, precomputed ones
# replace PDB distances with r^-6-averaged distances
# print 'len(s.distance_restraints)', len(s.distance_restraints)
# for j in range(len(s.distance_restraints)):
# print s.distance_restraints[j].i, s.distance_restraints[j].j, model_distances[j]
# s.distance_restraints[j].model_distance = model_distances[j]
import socket
import Structure
import pickle
r = Structure.RegisterStructure('chenzehan3', 'siwei')
data = pickle.dumps(r)
s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
s.bind(('127.0.0.1', 10003))
aim_addr = ('127.0.0.1', 10086)
s.sendto(data, aim_addr)
def __init__(self, data):
self.data = data
self.valid = False
self.callbacks = []
self.symbols_by_name = {}
self.symbols_by_addr = {}
if not self.is_elf():
return
try:
self.tree = Structure(self.data)
self.header = self.tree.struct("ELF header", "header")
self.header.struct("ELF identification", "ident")
self.header.ident.uint32("magic")
self.header.ident.uint8("file_class")
self.header.ident.uint8("encoding")
self.header.ident.uint8("version")
self.header.ident.uint8("abi")
self.header.ident.uint8("abi_version")
self.header.ident.bytes(7, "pad")
self.header.uint16("type")
self.header.uint16("arch")
self.header.uint32("version")
self.symbol_table_section = None
self.dynamic_symbol_table_section = None
if self.header.ident.file_class == 1: # 32-bit
self.header.uint32("entry")
self.header.uint32("program_header_offset")
self.header.uint32("section_header_offset")
self.header.uint32("flags")
self.header.uint16("header_size")
self.header.uint16("program_header_size")
self.header.uint16("program_header_count")
self.header.uint16("section_header_size")
self.header.uint16("section_header_count")
self.header.uint16("string_table")
try:
self.sections = self.tree.array(self.header.section_header_count, "sections")
for i in range(0, self.header.section_header_count):
section = self.sections[i]
section.seek(self.header.section_header_offset + (i * 40))
section.uint32("name")
section.uint32("type")
section.uint32("flags")
section.uint32("addr")
section.uint32("offset")
section.uint32("size")
section.uint32("link")
section.uint32("info")
section.uint32("align")
section.uint32("entry_size")
if section.type == 2:
self.symbol_table_section = section
elif section.type == 11:
self.dynamic_symbol_table_section = section
except:
# Section headers are not required to load an ELF, skip errors
self.sections = self.tree.array(0, "sections")
pass
self.program_headers = self.tree.array(self.header.program_header_count, "programHeaders")
for i in range(0, self.header.program_header_count):
header = self.program_headers[i]
header.seek(self.header.program_header_offset + (i * 32))
header.uint32("type")
header.uint32("offset")
header.uint32("virtual_addr")
header.uint32("physical_addr")
header.uint32("file_size")
header.uint32("memory_size")
header.uint32("flags")
header.uint32("align")
# Parse symbol tables
self.symbols_by_name["_start"] = self.entry()
self.symbols_by_addr[self.entry()] = "_start"
try:
if self.symbol_table_section:
self.symbol_table = self.tree.array(self.symbol_table_section.size / 16, "Symbols", "symbols")
self.parse_symbol_table_32(self.symbol_table, self.symbol_table_section, self.sections[self.symbol_table_section.link])
if self.dynamic_symbol_table_section:
self.dynamic_symbol_table = self.tree.array(self.dynamic_symbol_table_section.size / 16, "Symbols", "symbols")
self.parse_symbol_table_32(self.dynamic_symbol_table, self.dynamic_symbol_table_section, self.sections[self.dynamic_symbol_table_section.link])
except:
# Skip errors in symbol table
pass
# Parse relocation tables
self.plt = {}
for section in self.sections:
if section.type == 9:
self.parse_reloc_32(section)
elif section.type == 4:
self.parse_reloca_32(section)
elif self.header.ident.file_class == 2: # 64-bit
self.header.uint64("entry")
self.header.uint64("program_header_offset")
self.header.uint64("section_header_offset")
self.header.uint32("flags")
self.header.uint16("header_size")
self.header.uint16("program_header_size")
self.header.uint16("program_header_count")
self.header.uint16("section_header_size")
self.header.uint16("section_header_count")
self.header.uint16("string_table")
try:
self.sections = self.tree.array(self.header.section_header_count, "sections")
for i in range(0, self.header.section_header_count):
section = self.sections[i]
section.seek(self.header.section_header_offset + (i * 64))
section.uint32("name")
section.uint32("type")
section.uint64("flags")
section.uint64("addr")
section.uint64("offset")
section.uint64("size")
section.uint32("link")
section.uint32("info")
section.uint64("align")
section.uint64("entry_size")
if section.type == 2:
self.symbol_table_section = section
elif section.type == 11:
self.dynamic_symbol_table_section = section
except:
# Section headers are not required to load an ELF, skip errors
self.sections = self.tree.array(0, "sections")
pass
self.program_headers = self.tree.array(self.header.program_header_count, "program_headers")
for i in range(0, self.header.program_header_count):
header = self.program_headers[i]
header.seek(self.header.program_header_offset + (i * 56))
header.uint32("type")
header.uint32("flags")
header.uint64("offset")
header.uint64("virtual_addr")
header.uint64("physical_addr")
header.uint64("file_size")
header.uint64("memory_size")
header.uint64("align")
# Parse symbol tables
self.symbols_by_name["_start"] = self.entry()
self.symbols_by_addr[self.entry()] = "_start"
try:
if self.symbol_table_section:
self.symbol_table = self.tree.array(self.symbol_table_section.size / 24, "Symbols", "symbols")
self.parse_symbol_table_64(self.symbol_table, self.symbol_table_section, self.sections[self.symbol_table_section.link])
if self.dynamic_symbol_table_section:
self.dynamic_symbol_table = self.tree.array(self.dynamic_symbol_table_section.size / 24, "Symbols", "symbols")
self.parse_symbol_table_64(self.dynamic_symbol_table, self.dynamic_symbol_table_section, self.sections[self.dynamic_symbol_table_section.link])
except:
# Skip errors in symbol table
pass
# Parse relocation tables
self.plt = {}
for section in self.sections:
if section.type == 9:
self.parse_reloc_64(section)
elif section.type == 4:
self.parse_reloca_64(section)
self.tree.complete()
self.valid = True
except:
self.valid = False
if self.valid:
self.data.add_callback(self)
## Node level attributes
NodeA = NodeA.NodeAttribute(G)
## Edge level attributes
EdgeA = EdgeA.EdgeAttribute(G)
graph.walkNodes()
graph.walkEdges()
##### init edge attributes
attribute_type = 3
attribute_name = "srcLabel"
EdgeA.walkGraphEdgeAttributes(attribute_type, attribute_name)
structure = structure.Structure(G, graphName)
structure.genGraphInfo()
centrality = centrality.Centrality(G, graphName)
# degCentrVals = centrality.genCentrValues(centrality.genGraphInfo("_degree-centrality", snap.GetDegreeCentr))
# closenessCentrVals = centrality.genCentrValues(
# centrality.genGraphInfo("_closeness-centrality", snap.GetClosenessCentr))
#
# harmonicClosenessCentrVals = centrality.genCentrValues(
# centrality.genGraphInfo("_harmonic-closeness-centrality", centrality.getHarmonicClosenessCentr))
#
# localccCentrVals = centrality.genCentrValues(
# centrality.genGraphInfo("_local-clustering-coefficient", snap.GetNodeClustCf))
def add_secondary_structure_section(self):
"""HELIX, SHEET, TURN
PDB files do not put separate secondary structure descriptions
within MODEL definitions, so you have to hope the models
do not differ in secondary structure. mmLib allows separate
MODELs to have different secondary structure, but one MODEL must
be chosen for the PDF file, so the default Model of the Structure
is used.
"""
## HELIX
serial_num = 0
for alpha_helix in self.struct.iter_alpha_helicies():
serial_num += 1
helix = PDB.HELIX()
self.pdb_file.append(helix)
helix["serNum"] = serial_num
helix["helixID"] = alpha_helix.helix_id
helix["helixClass"] = alpha_helix.helix_class
helix["initResName"] = alpha_helix.res_name1
helix["initChainID"] = alpha_helix.chain_id1
try:
helix["initSeqNum"], helix["initICode"] = Structure.fragment_id_split(alpha_helix.fragment_id1)
except ValueError:
pass
helix["endResName"] = alpha_helix.res_name2
helix["endChainID"] = alpha_helix.chain_id2
try:
helix["endSeqNum"], helix["endICode"] = Structure.fragment_id_split(alpha_helix.fragment_id2)
except ValueError:
pass
helix["comment"] = alpha_helix.details
helix["initChainID"] = alpha_helix.chain_id1
helix["length"] = alpha_helix.helix_length
## SHEET
for beta_sheet in self.struct.iter_beta_sheets():
num_strands = len(beta_sheet.strand_list)
strand_num = 0
for strand in beta_sheet.iter_strands():
strand_num += 1
sheet = PDB.SHEET()
self.pdb_file.append(sheet)
sheet["strand"] = strand_num
sheet["sheetID"] = beta_sheet.sheet_id
sheet["numStrands"] = num_strands
sheet["initResName"] = strand.res_name1
sheet["initChainID"] = strand.chain_id1
try:
sheet["initSeqNum"], sheet["initICode"] = Structure.fragment_id_split(strand.fragment_id1)
except ValueError:
pass
sheet["endResName"] = strand.res_name2
sheet["endChainID"] = strand.chain_id2
try:
sheet["endSeqNum"], sheet["endICode"] = Structure.fragment_id_split(strand.fragment_id2)
except ValueError:
pass
sheet["curAtom"] = strand.reg_atom
sheet["curResName"] = strand.reg_res_name
sheet["curChainID"] = strand.reg_chain_id
try:
sheet["curSeqNum"], sheet["curICode"] = Structure.fragment_id_split(strand.reg_fragment_id)
except ValueError:
pass
sheet["prevAtom"] = strand.reg_prev_atom
sheet["prevResName"] = strand.reg_prev_res_name
sheet["prevChainID"] = strand.reg_prev_chain_id
try:
sheet["prevSeqNum"],sheet["prevICode"] = Structure.fragment_id_split(strand.reg_prev_fragment_id)
except ValueError:
pass
#Swift = ArmSwift.uArmSwift() #initX, initY, initZ = Swift.get_position() #Kinematics.Camera2Arm(Cx, Cy, Cz) Angle = [0,0,0] PositionStart = [250, 200, 10] PositionEnd = [200, -200, 10] Position = PositionStart X,Y,Z = Position Angle[0],Angle[1],Angle[2] = Kinematics.XYZ2Angle(X,Y,Z) uArm = Build_uArm.BuildArm(Angle[0],Angle[1],Angle[2]) objStart = Structure.Obstacle(Center = PositionStart[0:2], Angle = 0, Size = [40, 40, PositionStart[2]], Type = 'cuboid') objEnd = copy.deepcopy(objStart) objEnd.Center = PositionEnd[0:2] objEnd.H = PositionEnd[2] obst1 = Structure.Obstacle(Center = [200,0], Angle = -45, Size = [80, 40, 60], Type = 'cuboid') obst2 = Structure.Obstacle(Center = [150,-50], Angle = 15, Size = [50, 50, 50], Type = 'cuboid') ShowState(uArm,Target=None,ShowAxes=True,xlim=[-300,300],ylim=[-300,300],Height=300,obstacle1 = obst1,obstacle2 = obst2) time.sleep(1) plt.close() ControlPoint, deltaAngle = Planning.GenControlPoint(obj1=objStart,obj2=objEnd,obst=obst1) #print(ControlPoint) #ControlPID = Planning.pid() #path = Planning.StrightPath(Position1=PositionStart, Position2=PositionEnd) path = Planning.BezierPath(PositionStart, PositionEnd, ControlPoint) flag = Planning.CheckPath(path)
def createStructureObject(pdbFilePath):
with open(pdbFilePath, 'r') as f:
pdb = [line.split() for line in f.readlines()]
structureObject = Structure.Structure(pdb)
return structureObject
def add_ATOM(self, rec_type, atm):
"""Adds ATOM/SIGATM/ANISOU/SIGUIJ/TER/HETATM
"""
self.atom_count += 1
if rec_type == "ATOM":
atom_rec = PDB.ATOM()
elif rec_type == "HETATM":
atom_rec = PDB.HETATM()
self.pdb_file.append(atom_rec)
serial = self.new_atom_serial(atm)
res_seq, icode = Structure.fragment_id_split(atm.fragment_id)
atom_rec["serial"] = serial
atom_rec["chainID"] = atm.chain_id
atom_rec["resName"] = atm.res_name
atom_rec["resSeq"] = res_seq
atom_rec["iCode"] = icode
atom_rec["name"] = atm.name
atom_rec["element"] = atm.element
atom_rec["altLoc"] = atm.alt_loc
if atm.position is not None:
if atm.position[0] is not None:
atom_rec["x"] = atm.position[0]
if atm.position[1] is not None:
atom_rec["y"] = atm.position[1]
if atm.position[2] is not None:
atom_rec["z"] = atm.position[2]
if atm.occupancy is not None:
atom_rec["occupancy"] = atm.occupancy
if atm.temp_factor is not None:
atom_rec["tempFactor"] = atm.temp_factor
if atm.column6768 is not None:
atom_rec["column6768"] = atm.column6768
if atm.charge is not None:
atom_rec["charge"] = atm.charge
def atom_common(arec1, arec2):
if arec1.has_key("serial"):
arec2["serial"] = arec1["serial"]
if arec1.has_key("chainID"):
arec2["chainID"] = arec1["chainID"]
if arec1.has_key("resName"):
arec2["resName"] = arec1["resName"]
if arec1.has_key("resSeq"):
arec2["resSeq"] = arec1["resSeq"]
if arec1.has_key("iCode"):
arec2["iCode"] = arec1["iCode"]
if arec1.has_key("name"):
arec2["name"] = arec1["name"]
if arec1.has_key("altLoc"):
arec2["altLoc"] = arec1["altLoc"]
if arec1.has_key("element"):
arec2["element"] = arec1["element"]
if arec1.has_key("charge"):
arec2["charge"] = arec1["charge"]
if atm.sig_position is not None:
sigatm_rec = PDB.SIGATM()
self.pdb_file.append(sigatm_rec)
atom_common(atom_rec, sigatm_rec)
if atm.sig_position[0] is not None:
sigatm_rec["sigX"] = atm.sig_position[0]
if atm.sig_position[1] is not None:
sigatm_rec["sigY"] = atm.sig_position[1]
if atm.sig_position[2] is not None:
sigatm_rec["sigZ"] = atm.sig_position[2]
if atm.sig_temp_factor is not None:
sigatm_rec["sigTempFactor"] = atm.sig_temp_factor
if atm.sig_occupancy is not None:
sigatm_rec["sigOccupancy"] = atm.sig_occupancy
if atm.U is not None:
anisou_rec = PDB.ANISOU()
self.pdb_file.append(anisou_rec)
atom_common(atom_rec, anisou_rec)
if atm.U[0, 0] is not None:
anisou_rec["u[0][0]"] = int(round(atm.U[0, 0] * 10000.0))
if atm.U[1, 1] is not None:
anisou_rec["u[1][1]"] = int(round(atm.U[1, 1] * 10000.0))
if atm.U[2, 2] is not None:
anisou_rec["u[2][2]"] = int(round(atm.U[2, 2] * 10000.0))
if atm.U[0, 1] is not None:
anisou_rec["u[0][1]"] = int(round(atm.U[0, 1] * 10000.0))
if atm.U[0, 2] is not None:
anisou_rec["u[0][2]"] = int(round(atm.U[0, 2] * 10000.0))
if atm.U[1, 2] is not None:
anisou_rec["u[1][2]"] = int(round(atm.U[1, 2] * 10000.0))
if atm.sig_U is not None:
siguij_rec = PDB.SIGUIJ()
self.pdb_file.append(siguij_rec)
atom_common(atom_rec, siguij_rec)
if atm.sig_U[0, 0] is not None:
siguij_rec["u[0][0]"] = int(round(atm.sig_U[0, 0] * 10000.0))
if atm.sig_U[1, 1] is not None:
siguij_rec["u[1][1]"] = int(round(atm.sig_U[1, 1] * 10000.0))
if atm.sig_U[2, 2] is not None:
siguij_rec["u[2][2]"] = int(round(atm.sig_U[2, 2] * 10000.0))
if atm.sig_U[0, 1] is not None:
siguij_rec["u[0][1]"] = int(round(atm.sig_U[0, 1] * 10000.0))
if atm.sig_U[0, 2] is not None:
siguij_rec["u[0][2]"] = int(round(atm.sig_U[0, 2] * 10000.0))
if atm.sig_U[1, 2] is not None:
siguij_rec["u[1][2]"] = int(round(atm.sig_U[1, 2] * 10000.0))
for o in range(len(allowed_xcs)):
for q in range(len(allowed_bs)):
infile_Nc='input/Nc/Nc_x%0.1f_b%d_state%03d.npy'%(allowed_xcs[o], allowed_bs[q],T1[i])
Ncs[o,q,:] = (np.load(infile_Nc))
for p in range(len(allowed_xhs)):
# for q in range(len(allowed_bs)):
# infile_Nc='input/Nc/Nc_x%0.1f_b%d_state%03d.npy'%(allowed_xcs[o], allowed_bs[q],T1[i])
infile_Nh='input/Nh/Nh_x%0.1f_b%d_state%03d.npy'%(allowed_xhs[p], allowed_bs[q],T1[i])
# Ncs[o,q,:] = (np.load(infile_Nc))
Nhs[p,q,:] = (np.load(infile_Nh))
# sys.exit()
# print Ncs[0,0,0]
# print Nhs.shape
# sys.exit()
s = Structure('new_traj/pdb/state%d.pdb'%T1[i], args.lam*energies[i], expdata_filename_PF=expdata_filename_PF, use_log_normal_distances=False, dloggamma=np.log(1.01), gamma_min=0.2, gamma_max=5.0, dbeta_c=0.005, beta_c_min=0.02, beta_c_max=0.03, dbeta_h=0.05, beta_h_min=0.00, beta_h_max=0.10, dbeta_0=0.2, beta_0_min=0.0, beta_0_max=0.4, dxcs=0.5, xcs_min=5.0, xcs_max=6.0, dxhs=0.1, xhs_min=2.0, xhs_max=2.1, dbs=1.0, bs_min=3.0, bs_max=5.0, Ncs=Ncs, Nhs=Nhs) #GYH
# sys.exit()
# s = Structure('Gens/Gens%d.pdb'%i, args.lam*energies[i], expdata_filename_noe=expdata_filename_noe, use_log_normal_distances=False, dloggamma=np.log(1.01), gamma_min=0.2, gamma_max=5.0)
# NOTE: Upon instantiation, each Structure() object computes the distances from the given PDB.
# However, our clusters represent averaged conformation states, and so we
# need to replace these values with our own r^-6-averaged, precomputed ones
# replace PDB distances with r^-6-averaged distances
# print 'len(s.distance_restraints)', len(s.distance_restraints)
# for j in range(len(s.distance_restraints)):
# print s.distance_restraints[j].i, s.distance_restraints[j].j, model_distances[j]
# s.distance_restraints[j].model_distance = model_distances[j]
def add_secondary_structure_section(self):
"""HELIX, SHEET, TURN
PDB files do not put separate secondary structure descriptions
within MODEL definitions, so you have to hope the models
do not differ in secondary structure. mmLib allows separate
MODELs to have different secondary structure, but one MODEL must
be chosen for the PDF file, so the default Model of the Structure
is used.
"""
## HELIX
serial_num = 0
for alpha_helix in self.struct.iter_alpha_helicies():
serial_num += 1
helix = PDB.HELIX()
self.pdb_file.append(helix)
helix["serNum"] = serial_num
helix["helixID"] = alpha_helix.helix_id
helix["helixClass"] = alpha_helix.helix_class
helix["initResName"] = alpha_helix.res_name1
helix["initChainID"] = alpha_helix.chain_id1
try:
helix["initSeqNum"], helix[
"initICode"] = Structure.fragment_id_split(
alpha_helix.fragment_id1)
except ValueError:
pass
helix["endResName"] = alpha_helix.res_name2
helix["endChainID"] = alpha_helix.chain_id2
try:
helix["endSeqNum"], helix[
"endICode"] = Structure.fragment_id_split(
alpha_helix.fragment_id2)
except ValueError:
pass
helix["comment"] = alpha_helix.details
helix["initChainID"] = alpha_helix.chain_id1
helix["length"] = alpha_helix.helix_length
## SHEET
for beta_sheet in self.struct.iter_beta_sheets():
num_strands = len(beta_sheet.strand_list)
strand_num = 0
for strand in beta_sheet.iter_strands():
strand_num += 1
sheet = PDB.SHEET()
self.pdb_file.append(sheet)
sheet["strand"] = strand_num
sheet["sheetID"] = beta_sheet.sheet_id
sheet["numStrands"] = num_strands
sheet["initResName"] = strand.res_name1
sheet["initChainID"] = strand.chain_id1
try:
sheet["initSeqNum"], sheet[
"initICode"] = Structure.fragment_id_split(
strand.fragment_id1)
except ValueError:
pass
sheet["endResName"] = strand.res_name2
sheet["endChainID"] = strand.chain_id2
try:
sheet["endSeqNum"], sheet[
"endICode"] = Structure.fragment_id_split(
strand.fragment_id2)
except ValueError:
pass
sheet["curAtom"] = strand.reg_atom
sheet["curResName"] = strand.reg_res_name
sheet["curChainID"] = strand.reg_chain_id
try:
sheet["curSeqNum"], sheet[
"curICode"] = Structure.fragment_id_split(
strand.reg_fragment_id)
except ValueError:
pass
sheet["prevAtom"] = strand.reg_prev_atom
sheet["prevResName"] = strand.reg_prev_res_name
sheet["prevChainID"] = strand.reg_prev_chain_id
try:
sheet["prevSeqNum"], sheet[
"prevICode"] = Structure.fragment_id_split(
strand.reg_prev_fragment_id)
except ValueError:
pass
def __init__(self, data):
self.data = data
self.valid = False
self.callbacks = []
self.symbols_by_name = {}
self.symbols_by_addr = {}
self.plt = {}
if not self.is_macho():
return
try:
self.tree = Structure(self.data)
self.header = self.tree.struct("Mach-O header", "header")
self.header.uint32_le("magic")
if (self.header.magic == 0xfeedface) or (self.header.magic == 0xfeedfacf):
self.header.uint32_le("cputype")
self.header.uint32_le("cpusubtype")
self.header.uint32_le("filetype")
self.header.uint32_le("cmds")
self.header.uint32_le("cmdsize")
self.header.uint32_le("flags")
if self.header.magic == 0xfeedfacf:
self.header.uint32_le("reserved")
self.bits = 64
else:
self.bits = 32
self.big_endian = False
elif (self.header.magic == 0xcefaedfe) or (self.header.magic == 0xcffaedfe):
self.header.uint32_be("cputype")
self.header.uint32_be("cpusubtype")
self.header.uint32_be("filetype")
self.header.uint32_be("cmds")
self.header.uint32_be("cmdsize")
self.header.uint32_be("flags")
if self.header.magic == 0xcffaedfe:
self.header.uint32_be("reserved")
self.bits = 64
else:
self.bits = 32
self.big_endian = True
self.symbol_table = None
self.dynamic_symbol_table = None
# Parse loader commands
self.commands = self.tree.array(self.header.cmds, "commands")
self.segments = []
self.sections = []
offset = self.header.getSize()
for i in xrange(0, self.header.cmds):
cmd = self.commands[i]
cmd.seek(offset)
if self.big_endian:
cmd.uint32_be("cmd")
cmd.uint32_be("size")
else:
cmd.uint32_le("cmd")
cmd.uint32_le("size")
if cmd.cmd == 1: # SEGMENT
cmd.bytes(16, "name")
if self.big_endian:
cmd.uint32_be("vmaddr")
cmd.uint32_be("vmsize")
cmd.uint32_be("fileoff")
cmd.uint32_be("filesize")
cmd.uint32_be("maxprot")
cmd.uint32_be("initprot")
cmd.uint32_be("nsects")
cmd.uint32_be("flags")
else:
cmd.uint32_le("vmaddr")
cmd.uint32_le("vmsize")
cmd.uint32_le("fileoff")
cmd.uint32_le("filesize")
cmd.uint32_le("maxprot")
cmd.uint32_le("initprot")
cmd.uint32_le("nsects")
cmd.uint32_le("flags")
if cmd.initprot != 0: # Ignore __PAGE_ZERO or anything like it
self.segments.append(cmd)
cmd.array(cmd.nsects, "sections")
for i in xrange(0, cmd.nsects):
section = cmd.sections[i]
section.bytes(16, "name")
section.bytes(16, "segment")
if self.big_endian:
section.uint32_be("addr")
section.uint32_be("size")
section.uint32_be("offset")
section.uint32_be("align")
section.uint32_be("reloff")
section.uint32_be("nreloc")
section.uint32_be("flags")
section.uint32_be("reserved1")
section.uint32_be("reserved2")
else:
section.uint32_le("addr")
section.uint32_le("size")
section.uint32_le("offset")
section.uint32_le("align")
section.uint32_le("reloff")
section.uint32_le("nreloc")
section.uint32_le("flags")
section.uint32_le("reserved1")
section.uint32_le("reserved2")
self.sections.append(section)
for i in xrange(0, cmd.nsects):
section = cmd.sections[i]
section.array(section.nreloc, "relocs")
for j in xrange(0, section.nreloc):
reloc = section.relocs[j]
reloc.seek(section.reloff + (j * 8))
if self.big_endian:
reloc.uint32_be("addr")
reloc.uint32_be("value")
else:
reloc.uint32_le("addr")
reloc.uint32_le("value")
elif cmd.cmd == 25: # SEGMENT_64
cmd.bytes(16, "name")
if self.big_endian:
cmd.uint64_be("vmaddr")
cmd.uint64_be("vmsize")
cmd.uint64_be("fileoff")
cmd.uint64_be("filesize")
cmd.uint32_be("maxprot")
cmd.uint32_be("initprot")
cmd.uint32_be("nsects")
cmd.uint32_be("flags")
else:
cmd.uint64_le("vmaddr")
cmd.uint64_le("vmsize")
cmd.uint64_le("fileoff")
cmd.uint64_le("filesize")
cmd.uint32_le("maxprot")
cmd.uint32_le("initprot")
cmd.uint32_le("nsects")
cmd.uint32_le("flags")
if cmd.initprot != 0: # Ignore __PAGE_ZERO or anything like it
self.segments.append(cmd)
cmd.array(cmd.nsects, "sections")
for i in xrange(0, cmd.nsects):
section = cmd.sections[i]
section.bytes(16, "name")
section.bytes(16, "segment")
if self.big_endian:
section.uint64_be("addr")
section.uint64_be("size")
section.uint32_be("offset")
section.uint32_be("align")
section.uint32_be("reloff")
section.uint32_be("nreloc")
section.uint32_be("flags")
section.uint32_be("reserved1")
section.uint32_be("reserved2")
section.uint32_be("reserved3")
else:
section.uint64_le("addr")
section.uint64_le("size")
section.uint32_le("offset")
section.uint32_le("align")
section.uint32_le("reloff")
section.uint32_le("nreloc")
section.uint32_le("flags")
section.uint32_le("reserved1")
section.uint32_le("reserved2")
section.uint32_le("reserved3")
self.sections.append(section)
for i in xrange(0, cmd.nsects):
section = cmd.sections[i]
section.array(section.nreloc, "relocs")
for j in xrange(0, section.nreloc):
reloc = section.relocs[j]
reloc.seek(section.reloff + (j * 8))
if self.big_endian:
reloc.uint32_be("addr")
reloc.uint32_be("value")
else:
reloc.uint32_le("addr")
reloc.uint32_le("value")
elif cmd.cmd == 5: # UNIX_THREAD
if self.header.cputype == 7: # x86
cmd.uint32_le("flavor")
cmd.uint32_le("count")
for reg in ["eax", "ebx", "ecx", "edx", "edi", "esi", "ebp", "esp", "ss", "eflags",
"eip", "cs", "ds", "es", "fs", "gs"]:
cmd.uint32_le(reg)
self.entry_addr = cmd.eip
elif self.header.cputype == 0x01000007: # x86_64
cmd.uint32_le("flavor")
cmd.uint32_le("count")
for reg in ["rax", "rbx", "rcx", "rdx", "rdi", "rsi", "rbp", "rsp", "r8", "r9",
"r10", "r11", "r12", "r13", "r14", "r15", "rip", "rflags", "cs", "fs", "gs"]:
cmd.uint64_le(reg)
self.entry_addr = cmd.rip
elif self.header.cputype == 18: # PPC32
cmd.uint32_be("flavor")
cmd.uint32_be("count")
for reg in ["srr0", "srr1"] + ["r%d" % i for i in xrange(0, 32)] + ["cr", "xer",
"lr", "ctr", "mq", "vrsave"]:
cmd.uint32_be(reg)
self.entry_addr = cmd.srr0
elif self.header.cputype == 0x01000012: # PPC64
cmd.uint32_be("flavor")
cmd.uint32_be("count")
for reg in ["srr0", "srr1"] + ["r%d" % i for i in xrange(0, 32)] + ["cr", "xer",
"lr", "ctr", "mq", "vrsave"]:
cmd.uint64_be(reg)
self.entry_addr = cmd.srr0
elif self.header.cputype == 12: # ARM
cmd.uint32_le("flavor")
cmd.uint32_le("count")
for reg in ["r%d" % i for i in xrange(0, 13)] + ["sp", "lr", "pc", "cpsr"]:
cmd.uint32_le(reg)
self.entry_addr = cmd.pc
elif cmd.cmd == 2: # SYMTAB
if self.big_endian:
cmd.uint32_be("symoff")
cmd.uint32_be("nsyms")
cmd.uint32_be("stroff")
cmd.uint32_be("strsize")
else:
cmd.uint32_le("symoff")
cmd.uint32_le("nsyms")
cmd.uint32_le("stroff")
cmd.uint32_le("strsize")
self.symbol_table = self.tree.array(cmd.nsyms, "symtab")
strings = self.data.read(cmd.stroff, cmd.strsize)
sym_offset = cmd.symoff
for j in xrange(0, cmd.nsyms):
entry = self.symbol_table[j]
entry.seek(sym_offset)
if self.big_endian:
entry.uint32_be("strx")
entry.uint8("type")
entry.uint8("sect")
entry.uint16_be("desc")
if self.bits == 32:
entry.uint32_be("value")
else:
entry.uint64_be("value")
else:
entry.uint32_le("strx")
entry.uint8("type")
entry.uint8("sect")
entry.uint16_le("desc")
if self.bits == 32:
entry.uint32_le("value")
else:
entry.uint64_le("value")
str_end = strings.find("\x00", entry.strx)
entry.name = strings[entry.strx:str_end]
if self.bits == 32:
sym_offset += 12
else:
sym_offset += 16
elif cmd.cmd == 11: # DYSYMTAB
if self.big_endian:
cmd.uint32_be("ilocalsym")
cmd.uint32_be("nlocalsym")
cmd.uint32_be("iextdefsym")
cmd.uint32_be("nextdefsym")
cmd.uint32_be("iundefsym")
cmd.uint32_be("nundefsym")
cmd.uint32_be("tocoff")
cmd.uint32_be("ntoc")
cmd.uint32_be("modtaboff")
cmd.uint32_be("nmodtab")
cmd.uint32_be("extrefsymoff")
cmd.uint32_be("nextrefsyms")
cmd.uint32_be("indirectsymoff")
cmd.uint32_be("nindirectsyms")
cmd.uint32_be("extreloff")
cmd.uint32_be("nextrel")
cmd.uint32_be("locreloff")
cmd.uint32_be("nlocrel")
else:
cmd.uint32_le("ilocalsym")
cmd.uint32_le("nlocalsym")
cmd.uint32_le("iextdefsym")
cmd.uint32_le("nextdefsym")
cmd.uint32_le("iundefsym")
cmd.uint32_le("nundefsym")
cmd.uint32_le("tocoff")
cmd.uint32_le("ntoc")
cmd.uint32_le("modtaboff")
cmd.uint32_le("nmodtab")
cmd.uint32_le("extrefsymoff")
cmd.uint32_le("nextrefsyms")
cmd.uint32_le("indirectsymoff")
cmd.uint32_le("nindirectsyms")
cmd.uint32_le("extreloff")
cmd.uint32_le("nextrel")
cmd.uint32_le("locreloff")
cmd.uint32_le("nlocrel")
elif (cmd.cmd & 0x7fffffff) == 0x22: # DYLD_INFO
self.dynamic_symbol_table = cmd
if self.big_endian:
cmd.uint32_be("rebaseoff")
cmd.uint32_be("rebasesize")
cmd.uint32_be("bindoff")
cmd.uint32_be("bindsize")
cmd.uint32_be("weakbindoff")
cmd.uint32_be("weakbindsize")
cmd.uint32_be("lazybindoff")
cmd.uint32_be("lazybindsize")
cmd.uint32_be("exportoff")
cmd.uint32_be("exportsize")
else:
cmd.uint32_le("rebaseoff")
cmd.uint32_le("rebasesize")
cmd.uint32_le("bindoff")
cmd.uint32_le("bindsize")
cmd.uint32_le("weakbindoff")
cmd.uint32_le("weakbindsize")
cmd.uint32_le("lazybindoff")
cmd.uint32_le("lazybindsize")
cmd.uint32_le("exportoff")
cmd.uint32_le("exportsize")
offset += cmd.size
# Add symbols from symbol table
if self.symbol_table:
for i in xrange(0, len(self.symbol_table)):
symbol = self.symbol_table[i]
# Only use symbols that are within a section
if ((symbol.type & 0xe) == 0xe) and (symbol.sect <= len(self.sections)):
self.create_symbol(symbol.value, symbol.name)
# If there is a DYLD_INFO section, parse it and add PLT entries
if self.dynamic_symbol_table:
self.parse_dynamic_tables([[self.dynamic_symbol_table.bindoff, self.dynamic_symbol_table.bindsize],
[self.dynamic_symbol_table.lazybindoff, self.dynamic_symbol_table.lazybindsize]])
self.tree.complete()
self.valid = True
except:
self.valid = False
raise
if self.valid:
self.data.add_callback(self)
utr_5 = [1, 299]
utr_3 = [29536, 29870]
seq5 = seq[utr_5[0] - 1:utr_5[1]]
shape5 = shape[utr_5[0] - 1:utr_5[1]]
seq3 = seq[utr_3[0] - 1:utr_3[1]]
shape3 = shape[utr_3[0] - 1:utr_3[1]]
dot['RF03120_UTR5'][0] == seq5
dot['RF03125_UTR3'][0] == seq3
######## 预测二级结构
prob3, pfs3 = Structure.partition(seq3,
shape_list=shape3,
si=-0.4,
sm=1.5,
md=300,
return_pfs=True)
maxexpect_dot3 = Structure.maxExpect(input_pfs_file=pfs3, delete_pfs=True)[0]
clean_dot3 = SARS2.remove_lowporb_bp(prob3,
maxexpect_dot3,
minprob=0.6,
minDist=150)
bp_prob3 = SARS2.collect_bpprob(prob3, clean_dot3)
auc3 = General.calc_AUC_v2(clean_dot3, shape3)
cmd3 = Visual.Plot_RNAStructure_Shape(seq3,
clean_dot3,
shape3,
mode='label',
bpprob=bp_prob3,
bpprob_cutofflist=[0.6, 0.8, 0.95],
