def _parse_symbol_section(self, section):
sh_link = section.sh_link
if self.b64:
pass
else:
pass
Symbol = Struct(
"st_name" / self.Word,
"st_value" / self.Addr,
"st_size" / self.Word,
"st_info" / BitStruct(
"st_bind" / BitsInteger(4),
"st_type" / BitsInteger(4),
),
"st_other" / Int8ul,
"symbol_name" / Computed(lambda ctx: self.get_string(sh_link, ctx.st_name)),
"st_shndx" / self.Half,
"section_name" / Computed(lambda ctx: defs.section_name(ctx.st_shndx)),
"hidden" / Computed(lambda ctx: ctx.st_other in
(defs.SymbolVisibility.STV_HIDDEN, defs.SymbolVisibility.STV_INTERNAL)),
)
"""
typedef struct {
Elf32_Word st_name;
Elf32_Addr st_value;
Elf32_Word st_size;
unsigned char st_info;
unsigned char st_other;
Elf32_Half st_shndx;
} Elf32_Sym;
typedef struct {
Elf64_Word st_name;
unsigned char st_info;
unsigned char st_other;
Elf64_Half st_shndx;
Elf64_Addr st_value;
Elf64_Xword st_size;
} Elf64_Sym;
"""
num_symbols = len(section.image) // Symbol.sizeof()
for offset in range(0, len(section.image), Symbol.sizeof()):
sym = Symbol.parse(section.image[offset : offset + Symbol.sizeof()])
db_sym = Elf_Symbol(st_name = sym.st_name, st_value = sym.st_value, st_size = sym.st_size,
st_bind = sym.st_info.st_bind, st_type = sym.st_info.st_type, st_other = sym.st_other,
st_shndx = sym.st_shndx, symbol_name = sym.symbol_name, section_name = sym.section_name,
hidden = sym.hidden)
self.session.add(db_sym)
#print(db_sym)
self.session.commit()
query = self.session.query(Elf_Symbol)
aaa = query.filter(Elf_Symbol.section_name == "SHN_ABS").all()
print("SECTIONS", [a.symbol_name for a in aaa])
def _extract_elf_corefile(self, exe_name=None):
"""
Reads the ELF formatted core dump image and parse it
"""
self.core_elf_file.write(self.core_src.data)
# Need to be closed before read. Otherwise the result will be wrong
self.core_elf_file.close()
core_elf = ESPCoreDumpElfFile(self.core_elf_file.name)
# Read note segments from core file which are belong to tasks (TCB or stack)
for seg in core_elf.note_segments:
for note_sec in seg.note_secs:
# Check for version info note
if note_sec.name == 'ESP_CORE_DUMP_INFO' \
and note_sec.type == ESPCoreDumpElfFile.PT_INFO \
and exe_name:
exe_elf = ElfFile(exe_name)
app_sha256 = binascii.hexlify(exe_elf.sha256)
coredump_sha256_struct = Struct(
'ver' / Int32ul,
'sha256' / Bytes(64) # SHA256 as hex string
)
coredump_sha256 = coredump_sha256_struct.parse(
note_sec.desc[:coredump_sha256_struct.sizeof()])
if coredump_sha256.sha256 != app_sha256:
raise ESPCoreDumpLoaderError(
'Invalid application image for coredump: coredump SHA256({}) != app SHA256({}).'
.format(coredump_sha256, app_sha256))
if coredump_sha256.ver != self.version:
raise ESPCoreDumpLoaderError(
'Invalid application image for coredump: coredump SHA256 version({}) != app SHA256 version({}).'
.format(coredump_sha256.ver, self.version))
def _parse_program_headers(self):
ProgramHeaders = Struct(
"segments" / Array(lambda ctx: self.e_phnum,
"segment" / IfThenElse(lambda ctx: self.b64,
Struct(
"p_type" / self.Word,
"p_flags" / self.Word,
"p_offset" / self.Off,
"p_vaddr" / self.Addr,
"p_paddr" / self.Addr,
"p_filesz" / self.Xword,
"p_memsz" / self.Xword,
"p_align" / self.Xword,
),
Struct(
"p_type" / self.Word,
"p_offset" / self.Off,
"p_vaddr" / self.Addr,
"p_paddr" / self.Addr,
"p_filesz" / self.Word,
"p_memsz" / self.Word,
"p_flags" / self.Word,
"p_align" / self.Word,
)
),
)
)
if hasattr(self, 'e_shnum'):
if self.e_shnum:
print("PG_size: {}".format(ProgramHeaders.sizeof() / self.e_phnum))
self._program_headers = ProgramHeaders.parse(self.fp[self.e_phoff : ])
def _parse_symbol_section(self, section):
sh_link = section.sh_link
symbols = []
Symbol = Struct(
"st_name" / self.Word,
"st_value" / self.Addr,
"st_size" / self.Word,
"st_info" / BitStruct(
"st_bind" / BitsInteger(4),
"st_type" / BitsInteger(4),
),
"st_other" / Int8ul,
"symbol_name" /
Computed(lambda ctx: self.get_string(sh_link, ctx.st_name)),
"st_shndx" / self.Half,
)
symbol_cache = {}
num_symbols = len(section.image) // Symbol.sizeof()
for offset in range(0, len(section.image), Symbol.sizeof()):
sym = Symbol.parse(section.image[offset:offset + Symbol.sizeof()])
section_header = None
if sym.st_shndx in defs.SpecialSections:
section_name = defs.special_section_name(sym.st_shndx)
else:
if not sym.st_shndx in symbol_cache:
section_header = self.session.query(model.Elf_Section).\
filter(model.Elf_Section.index == sym.st_shndx).first()
if section_header:
section_name = section_header.section_name
else:
section_name = str(sym.st_shndx)
db_sym = model.Elf_Symbol(
st_name=sym.st_name,
st_value=sym.st_value,
st_size=sym.st_size,
st_bind=sym.st_info.st_bind,
st_type=sym.st_info.st_type,
st_other=sym.st_other,
st_shndx=sym.st_shndx,
symbol_name=sym.symbol_name,
section_name=section_name,
access=section_header.sh_flags if section_header else 0)
symbols.append(db_sym)
self.session.bulk_save_objects(symbols)
self.session.commit()
def get_hci_version(self):
"""Gets the HCI version"""
local_version = Struct(
"status" / Byte,
"hci_version" / Byte,
"hci_revision" / Bytes(2),
"lmp_version" / Byte,
"manufacturer_name" / Bytes(2),
"lmp_subversion" / Bytes(2),
)
try:
resp = self.backend.send_req(self.socket, OGF_INFO_PARAM, OCF_READ_LOCAL_VERSION,
EVT_CMD_COMPLETE, local_version.sizeof(), bytes(), 0)
return HCIVersion(GreedyRange(local_version).parse(resp)[0]["hci_version"])
except (ConstructError, NotImplementedError):
return HCIVersion.BT_CORE_SPEC_1_0
def get_registers_from_stack(data, grows_down):
# type: (bytes, bool) -> Tuple[list[int], Optional[dict[int, int]]]
extra_regs = {v: 0 for v in ExceptionRegisters().registers.values()}
regs = [0] * REG_NUM
# TODO: support for growing up stacks
if not grows_down:
raise ESPCoreDumpLoaderError(
'Growing up stacks are not supported for now!')
ex_struct = Struct('stack' / Int32ul[XT_STK_FRMSZ])
if len(data) < ex_struct.sizeof():
raise ESPCoreDumpLoaderError(
'Too small stack to keep frame: %d bytes!' % len(data))
stack = ex_struct.parse(data).stack
# Stack frame type indicator is always the first item
rc = stack[XT_STK_EXIT]
if rc != 0:
regs[REG_PC_IDX] = stack[XT_STK_PC]
regs[REG_PS_IDX] = stack[XT_STK_PS]
for i in range(XT_STK_AR_NUM):
regs[REG_AR_START_IDX + i] = stack[XT_STK_AR_START + i]
regs[REG_SAR_IDX] = stack[XT_STK_SAR]
regs[REG_LB_IDX] = stack[XT_STK_LBEG]
regs[REG_LE_IDX] = stack[XT_STK_LEND]
regs[REG_LC_IDX] = stack[XT_STK_LCOUNT]
# FIXME: crashed and some running tasks (e.g. prvIdleTask) have EXCM bit set
# and GDB can not unwind callstack properly (it implies not windowed call0)
if regs[REG_PS_IDX] & (1 << 5):
regs[REG_PS_IDX] &= ~(1 << 4)
if stack[XT_STK_EXCCAUSE] in XTENSA_EXCEPTION_CAUSE_DICT:
extra_regs[
ExceptionRegisters.EXCCAUSE_IDX] = stack[XT_STK_EXCCAUSE]
else:
extra_regs[
ExceptionRegisters.EXCCAUSE_IDX] = INVALID_CAUSE_VALUE
extra_regs[
ExceptionRegisters.EXCVADDR_IDX] = stack[XT_STK_EXCVADDR]
else:
regs[REG_PC_IDX] = stack[XT_SOL_PC]
regs[REG_PS_IDX] = stack[XT_SOL_PS]
for i in range(XT_SOL_AR_NUM):
regs[REG_AR_START_IDX + i] = stack[XT_SOL_AR_START + i]
# nxt = stack[XT_SOL_NEXT]
return regs, extra_regs
def __get_struct(is_wiiu: bool) -> Struct:
if is_wiiu:
swap = lambda x: x # noop # noqa: E731
encoding = 'utf-16-be'
# tga file
icon_struct = Struct('128' / Bytes(0x2c + 128 * 128 * 4))
icon_struct = 'icons_tga' / Padded(icon_struct.sizeof() + 4,
icon_struct)
region_all = 0xffffffff
else:
swap = ByteSwapped
encoding = 'utf-16-le'
# raw RGB565
icon_struct = 'icons_raw' / Struct('24' / Bytes(24 * 24 * 2),
'48' / Bytes(48 * 48 * 2))
region_all = 0x7fffffff
return InliningStruct(
'checksum' / ChecksumValue(hashlib.sha256, this),
ChecksumSourceData(
InlineStruct(
'title_id' / swap(common.TitleID), 'version' / swap(Int32ub),
'_unk1' / Bytes(4), 'regions' / swap(
FlagsEnum(Int32ub,
JP=1 << 0,
US=1 << 1,
EU=1 << 2,
AU=1 << 3,
CN=1 << 4,
KO=1 << 5,
TW=1 << 6,
ALL=region_all)), '_unk2' / Bytes(0x1c),
'title_info' / DictZipAdapter(
languages,
Array(
16,
Struct('short_name' / PaddedString(0x80, encoding),
'long_name' / PaddedString(0x100, encoding),
'publisher' / PaddedString(0x80, encoding)))),
icon_struct)), VerifyOrWriteChecksums, Terminated)
def get_data(binary_data):
"""Get data from binary string."""
raw_header, raw_body = binary_data[:HEADER_LENTH], binary_data[HEADER_LENTH+1:]
# TODO !!! Uncomment when will take valid data !!!
#header = HEADER_FORMAT.parse(raw_header)
# if binascii.crc_hqx(raw_body, 0xffff) != header.checksum:
# logger.exception('Checksums is not match')
# raise InputError(binascii.crc_hqx(raw_body, 0xffff), 'Checksums is not match')
result = {}
while True:
current_type = raw_body[:1]
if not current_type:
return result
type_of_struct = "0x{}".format(binascii.hexlify(current_type))
data = STRUCT_TYPES.get(type_of_struct, None)
if not data:
logger.exception('Data type is not exist')
raise AttributeError
data_size = data[1] if not isinstance(data[1], int) else Byte[data[1]]
body_format = Struct(
"type" / Int8ul,
"data" / data_size,
)
# TODO !!! Remove that exception when will take valid data
try:
current = body_format.parse(raw_body[:data_size.sizeof()+1])
except :
return result
result[data[0]] = current.data
raw_body = raw_body[body_format.sizeof():]
def _parse_section_headers(self):
SectionHeaders = Struct(
"sections" / Array(lambda ctx: self.e_shnum,
"section" / Struct(
"sh_name" / self.Word,
"sh_type" / self.Word,
"sh_flags" / self.Xword,
"sh_addr" / self.Addr,
"sh_offset" / self.Off,
"sh_size" / self.Xword,
"sh_link" / self.Word,
"sh_info" / self.Word,
"sh_addralign" / self.Xword,
"sh_entsize" /self.Xword,
"allocate" / Computed(lambda ctx: (ctx.sh_type not in (0, 8) and ctx.sh_size > 0)),
)
)
)
if hasattr(self, 'e_shnum'):
if self.e_shnum:
print("SH_size: {}".format(SectionHeaders.sizeof() / self.e_shnum))
self._section_headers = SectionHeaders.parse(self.fp[self.e_shoff : ])
for idx, section in enumerate(self._section_headers.sections):
if section.allocate:
image = self.fp[section.sh_offset : section.sh_offset + section.sh_size]
else:
image = None
self._images[idx] = image
section.image = image
for section in self._section_headers.sections:
name = self.get_string(self.e_shstrndx, section.sh_name)
section.name = name
self._sections_by_name[name] = section
if section.sh_type == defs.SectionType.SHT_NOTE:
print("NOTE!!! {:08X} {:04X}".format(section.sh_offset, section.sh_size))
self._parse_note(self.fp[section.sh_offset : section.sh_offset + section.sh_size])
elif section.sh_type in (defs.SectionType.SHT_SYMTAB, defs.SectionType.SHT_DYNSYM):
self._parse_symbol_section(section)
def _extract_elf_corefile(self,
exe_name=None,
e_machine=ESPCoreDumpElfFile.EM_XTENSA
): # type: (str, int) -> None
"""
Reads the ELF formatted core dump image and parse it
"""
with open(self.core_elf_file, 'wb') as fw: # type: ignore
fw.write(self.core_src.data) # type: ignore
core_elf = ESPCoreDumpElfFile(self.core_elf_file,
e_machine=e_machine) # type: ignore
# Read note segments from core file which are belong to tasks (TCB or stack)
for seg in core_elf.note_segments:
for note_sec in seg.note_secs:
# Check for version info note
if note_sec.name == 'ESP_CORE_DUMP_INFO' \
and note_sec.type == ESPCoreDumpElfFile.PT_INFO \
and exe_name:
exe_elf = ElfFile(exe_name)
app_sha256 = binascii.hexlify(exe_elf.sha256)
coredump_sha256_struct = Struct(
'ver' / Int32ul,
'sha256' / Bytes(64) # SHA256 as hex string
)
coredump_sha256 = coredump_sha256_struct.parse(
note_sec.desc[:coredump_sha256_struct.sizeof()])
if coredump_sha256.sha256 != app_sha256:
raise ESPCoreDumpLoaderError(
'Invalid application image for coredump: coredump SHA256({!r}) != app SHA256({!r}).'
.format(coredump_sha256, app_sha256))
if coredump_sha256.ver != self.version:
raise ESPCoreDumpLoaderError(
'Invalid application image for coredump: coredump SHA256 version({}) != app SHA256 version({}).'
.format(coredump_sha256.ver, self.version))
blockref = Struct(
"value" / Int24ul,
"control" / Int8ul,
)
anchor = Struct(
Const(b"\xff" * 4),
"akttab" / blockref,
"clorX" / blockref,
Const(b"\xff" * 4 * 3),
"taskRoot" / blockref,
Const(b"\xff" * 4),
) * "System anchor block"
assert pagesize // blockref.sizeof() == 128
blockTable = Array(pagesize // blockref.sizeof(), blockref)
# XXX: skip const
segmentTable = Sequence(Const(2 * blockref.sizeof() * b'\xff'),
Array(14, blockref))
drinfo = Struct(
"count" / blockref * "Number of blocks/pages allocated",
"blocks" / Array(3, blockref) *
"Direct block references for page 1, 2 and 3",
"blockTables" / Array(2, blockref) * "Block references to block tables",
"segmentTables" / Array(2, blockref) *
"Block references to segment tables, which refer to block tables",
) * "Dataspace descriptor"
class USBTransport(object):
'''Implement USB transport.'''
def __init__(self, *args, **kwargs):
device = kwargs.get('device', None)
'''Instantiate the first available PTP device over USB'''
logger.debug('Init USB')
self.__setup_constructors()
# If no device is specified, find all devices claiming to be Cameras
# and get the USB endpoints for the first one that works.
if device is None:
logger.debug('No device provided, probing all USB devices.')
if isinstance(device, six.string_types):
name = device
logger.debug(
'Device name provided, probing all USB devices for {}.'
.format(name)
)
device = None
else:
name = None
devs = (
[device] if (device is not None)
else find_usb_cameras(name=name)
)
self.__claimed = False
self.__acquire_camera(devs)
self.__event_queue = Queue()
self.__event_shutdown = Event()
# Locks for different end points.
self.__inep_lock = RLock()
self.__intep_lock = RLock()
self.__outep_lock = RLock()
# Slightly redundant transaction lock to avoid catching other request's
# response
self.__transaction_lock = RLock()
self.__event_proc = Thread(
name='EvtPolling',
target=self.__poll_events
)
self.__event_proc.daemon = False
atexit.register(self._shutdown)
self.__event_proc.start()
def __available_cameras(self, devs):
for dev in devs:
if self.__setup_device(dev):
logger.debug('Found USB PTP device {}'.format(dev))
yield
else:
message = 'No USB PTP device found.'
logger.error(message)
raise PTPError(message)
def __acquire_camera(self, devs):
'''From the cameras given, get the first one that does not fail'''
for _ in self.__available_cameras(devs):
# Stop system drivers
try:
if self.__dev.is_kernel_driver_active(
self.__intf.bInterfaceNumber):
try:
self.__dev.detach_kernel_driver(
self.__intf.bInterfaceNumber)
except usb.core.USBError:
message = (
'Could not detach kernel driver. '
'Maybe the camera is mounted?'
)
logger.error(message)
except NotImplementedError as e:
logger.debug('Ignoring unimplemented function: {}'.format(e))
# Claim camera
try:
logger.debug('Claiming {}'.format(repr(self.__dev)))
usb.util.claim_interface(self.__dev, self.__intf)
self.__claimed = True
except Exception as e:
logger.warn('Failed to claim PTP device: {}'.format(e))
continue
self.__dev.reset()
break
else:
message = (
'Could not acquire any camera.'
)
logger.error(message)
raise PTPError(message)
def _shutdown(self):
logger.debug('Shutdown request')
self.__event_shutdown.set()
# Free USB resource on shutdown.
# Only join a running thread.
if self.__event_proc.is_alive():
self.__event_proc.join(2)
try:
if self.__claimed:
logger.debug('Release {}'.format(repr(self.__dev)))
usb.util.release_interface(self.__dev, self.__intf)
except Exception as e:
logger.warn(e)
# Helper methods.
# ---------------------
def __setup_device(self, dev):
'''Get endpoints for a device. True on success.'''
self.__inep = None
self.__outep = None
self.__intep = None
self.__cfg = None
self.__dev = None
self.__intf = None
# Attempt to find the USB in, out and interrupt endpoints for a PTP
# interface.
for cfg in dev:
for intf in cfg:
if intf.bInterfaceClass == PTP_USB_CLASS:
for ep in intf:
ep_type = endpoint_type(ep.bmAttributes)
ep_dir = endpoint_direction(ep.bEndpointAddress)
if ep_type == ENDPOINT_TYPE_BULK:
if ep_dir == ENDPOINT_IN:
self.__inep = ep
elif ep_dir == ENDPOINT_OUT:
self.__outep = ep
elif ((ep_type == ENDPOINT_TYPE_INTR) and
(ep_dir == ENDPOINT_IN)):
self.__intep = ep
if not (self.__inep and self.__outep and self.__intep):
self.__inep = None
self.__outep = None
self.__intep = None
else:
logger.debug('Found {}'.format(repr(self.__inep)))
logger.debug('Found {}'.format(repr(self.__outep)))
logger.debug('Found {}'.format(repr(self.__intep)))
self.__cfg = cfg
self.__dev = dev
self.__intf = intf
return True
return False
def __setup_constructors(self):
'''Set endianness and create transport-specific constructors.'''
# Set endianness of constructors before using them.
self._set_endian('little')
self.__Length = Int32ul
self.__Type = Enum(
Int16ul,
default=Pass,
Undefined=0x0000,
Command=0x0001,
Data=0x0002,
Response=0x0003,
Event=0x0004,
)
# This is just a convenience constructor to get the size of a header.
self.__Code = Int16ul
self.__Header = Struct(
'Length' / self.__Length,
'Type' / self.__Type,
'Code' / self.__Code,
'TransactionID' / self._TransactionID,
)
# These are the actual constructors for parsing and building.
self.__CommandHeader = Struct(
'Length' / self.__Length,
'Type' / self.__Type,
'OperationCode' / self._OperationCode,
'TransactionID' / self._TransactionID,
)
self.__ResponseHeader = Struct(
'Length' / self.__Length,
'Type' / self.__Type,
'ResponseCode' / self._ResponseCode,
'TransactionID' / self._TransactionID,
)
self.__EventHeader = Struct(
'Length' / self.__Length,
'Type' / self.__Type,
'EventCode' / self._EventCode,
'TransactionID' / self._TransactionID,
)
# Apparently nobody uses the SessionID field. Even though it is
# specified in ISO15740:2013(E), no device respects it and the session
# number is implicit over USB.
self.__Param = Range(0, 5, self._Parameter)
self.__CommandTransactionBase = Struct(
Embedded(self.__CommandHeader),
'Payload' / Bytes(
lambda ctx, h=self.__Header: ctx.Length - h.sizeof()
)
)
self.__CommandTransaction = ExprAdapter(
self.__CommandTransactionBase,
encoder=lambda obj, ctx, h=self.__Header: Container(
Length=len(obj.Payload) + h.sizeof(),
**obj
),
decoder=lambda obj, ctx: obj,
)
self.__ResponseTransactionBase = Struct(
Embedded(self.__ResponseHeader),
'Payload' / Bytes(
lambda ctx, h=self.__Header: ctx.Length - h.sizeof())
)
self.__ResponseTransaction = ExprAdapter(
self.__ResponseTransactionBase,
encoder=lambda obj, ctx, h=self.__Header: Container(
Length=len(obj.Payload) + h.sizeof(),
**obj
),
decoder=lambda obj, ctx: obj,
)
def __parse_response(self, usbdata):
'''Helper method for parsing USB data.'''
# Build up container with all PTP info.
logger.debug('Transaction:')
usbdata = bytearray(usbdata)
if logger.isEnabledFor(logging.DEBUG):
for l in hexdump(
six.binary_type(usbdata[:512]),
result='generator'
):
logger.debug(l)
transaction = self.__ResponseTransaction.parse(usbdata)
response = Container(
SessionID=self.session_id,
TransactionID=transaction.TransactionID,
)
logger.debug('Interpreting {} transaction'.format(transaction.Type))
if transaction.Type == 'Response':
response['ResponseCode'] = transaction.ResponseCode
response['Parameter'] = self.__Param.parse(transaction.Payload)
elif transaction.Type == 'Event':
event = self.__EventHeader.parse(
usbdata[0:self.__Header.sizeof()]
)
response['EventCode'] = event.EventCode
response['Parameter'] = self.__Param.parse(transaction.Payload)
else:
command = self.__CommandHeader.parse(
usbdata[0:self.__Header.sizeof()]
)
response['OperationCode'] = command.OperationCode
response['Data'] = transaction.Payload
return response
def __recv(self, event=False, wait=False, raw=False):
'''Helper method for receiving data.'''
# TODO: clear stalls automatically
ep = self.__intep if event else self.__inep
lock = self.__intep_lock if event else self.__inep_lock
usbdata = array.array('B', [])
with lock:
tries = 0
# Attempt to read a header
while len(usbdata) < self.__Header.sizeof() and tries < 5:
if tries > 0:
logger.debug('Data smaller than a header')
logger.debug(
'Requesting {} bytes of data'
.format(ep.wMaxPacketSize)
)
try:
usbdata += ep.read(
ep.wMaxPacketSize
)
except usb.core.USBError as e:
# Return None on timeout or busy for events
if (
(e.errno is None and
('timeout' in e.strerror.decode() or
'busy' in e.strerror.decode())) or
(e.errno == 110 or e.errno == 16 or e.errno == 5)
):
if event:
return None
else:
logger.warning('Ignored exception: {}'.format(e))
else:
logger.error(e)
raise e
tries += 1
logger.debug('Read {} bytes of data'.format(len(usbdata)))
if len(usbdata) == 0:
if event:
return None
else:
raise PTPError('Empty USB read')
if (
logger.isEnabledFor(logging.DEBUG) and
len(usbdata) < self.__Header.sizeof()
):
logger.debug('Incomplete header')
for l in hexdump(
six.binary_type(bytearray(usbdata)),
result='generator'
):
logger.debug(l)
header = self.__ResponseHeader.parse(
bytearray(usbdata[0:self.__Header.sizeof()])
)
if header.Type not in ['Response', 'Data', 'Event']:
raise PTPError(
'Unexpected USB transfer type. '
'Expected Response, Event or Data but received {}'
.format(header.Type)
)
while len(usbdata) < header.Length:
usbdata += ep.read(
min(
header.Length - len(usbdata),
# Up to 64kB
64 * 2**10
)
)
if raw:
return usbdata
else:
return self.__parse_response(usbdata)
def __send(self, ptp_container, event=False):
'''Helper method for sending data.'''
ep = self.__intep if event else self.__outep
lock = self.__intep_lock if event else self.__outep_lock
transaction = self.__CommandTransaction.build(ptp_container)
with lock:
try:
sent = 0
while sent < len(transaction):
sent = ep.write(
# Up to 64kB
transaction[sent:(sent + 64*2**10)]
)
except usb.core.USBError as e:
# Ignore timeout or busy device once.
if (
(e.errno is None and
('timeout' in e.strerror.decode() or
'busy' in e.strerror.decode())) or
(e.errno == 110 or e.errno == 16 or e.errno == 5)
):
logger.warning('Ignored USBError {}'.format(e.errno))
ep.write(transaction)
def __send_request(self, ptp_container):
'''Send PTP request without checking answer.'''
# Don't modify original container to keep abstraction barrier.
ptp = Container(**ptp_container)
# Don't send unused parameters
try:
while not ptp.Parameter[-1]:
ptp.Parameter.pop()
if len(ptp.Parameter) == 0:
break
except IndexError:
# The Parameter list is already empty.
pass
# Send request
ptp['Type'] = 'Command'
ptp['Payload'] = self.__Param.build(ptp.Parameter)
self.__send(ptp)
def __send_data(self, ptp_container, data):
'''Send data without checking answer.'''
# Don't modify original container to keep abstraction barrier.
ptp = Container(**ptp_container)
# Send data
ptp['Type'] = 'Data'
ptp['Payload'] = data
self.__send(ptp)
@property
def _dev(self):
return None if self.__event_shutdown.is_set() else self.__dev
@_dev.setter
def _dev(self, value):
raise ValueError('Read-only property')
# Actual implementation
# ---------------------
def send(self, ptp_container, data):
'''Transfer operation with dataphase from initiator to responder'''
datalen = len(data)
logger.debug('SEND {} {} bytes{}'.format(
ptp_container.OperationCode,
datalen,
' ' + str(list(map(hex, ptp_container.Parameter)))
if ptp_container.Parameter else '',
))
with self.__transaction_lock:
self.__send_request(ptp_container)
self.__send_data(ptp_container, data)
# Get response and sneak in implicit SessionID and missing
# parameters.
response = self.__recv()
logger.debug('SEND {} {} bytes {}{}'.format(
ptp_container.OperationCode,
datalen,
response.ResponseCode,
' ' + str(list(map(hex, response.Parameter)))
if ptp_container.Parameter else '',
))
return response
def recv(self, ptp_container):
'''Transfer operation with dataphase from responder to initiator.'''
logger.debug('RECV {}{}'.format(
ptp_container.OperationCode,
' ' + str(list(map(hex, ptp_container.Parameter)))
if ptp_container.Parameter else '',
))
with self.__transaction_lock:
self.__send_request(ptp_container)
dataphase = self.__recv()
if hasattr(dataphase, 'Data'):
response = self.__recv()
if not (ptp_container.SessionID ==
dataphase.SessionID ==
response.SessionID):
self.__dev.reset()
raise PTPError(
'Dataphase session ID missmatch: {}, {}, {}.'
.format(
ptp_container.SessionID,
dataphase.SessionID,
response.SessionID
)
)
if not (ptp_container.TransactionID ==
dataphase.TransactionID ==
response.TransactionID):
self.__dev.reset()
raise PTPError(
'Dataphase transaction ID missmatch: {}, {}, {}.'
.format(
ptp_container.TransactionID,
dataphase.TransactionID,
response.TransactionID
)
)
if not (ptp_container.OperationCode ==
dataphase.OperationCode):
self.__dev.reset()
raise PTPError(
'Dataphase operation code missmatch: {}, {}.'.
format(
ptp_container.OperationCode,
dataphase.OperationCode
)
)
response['Data'] = dataphase.Data
else:
response = dataphase
logger.debug('RECV {} {}{}{}'.format(
ptp_container.OperationCode,
response.ResponseCode,
' {} bytes'.format(len(response.Data))
if hasattr(response, 'Data') else '',
' ' + str(list(map(hex, response.Parameter)))
if response.Parameter else '',
))
return response
def mesg(self, ptp_container):
'''Transfer operation without dataphase.'''
logger.debug('MESG {}{}'.format(
ptp_container.OperationCode,
' ' + str(list(map(hex, ptp_container.Parameter)))
if ptp_container.Parameter else '',
))
with self.__transaction_lock:
self.__send_request(ptp_container)
# Get response and sneak in implicit SessionID and missing
# parameters for FullResponse.
response = self.__recv()
logger.debug('MESG {} {}{}'.format(
ptp_container.OperationCode,
response.ResponseCode,
' ' + str(list(map(hex, response.Parameter)))
if response.Parameter else '',
))
return response
def event(self, wait=False):
'''Check event.
If `wait` this function is blocking. Otherwise it may return None.
'''
evt = None
usbdata = None
if wait:
usbdata = self.__event_queue.get(block=True)
elif not self.__event_queue.empty():
usbdata = self.__event_queue.get(block=False)
if usbdata is not None:
evt = self.__parse_response(usbdata)
return evt
def __poll_events(self):
'''Poll events, adding them to a queue.'''
while not self.__event_shutdown.is_set() and _main_thread_alive():
try:
evt = self.__recv(event=True, wait=False, raw=True)
if evt is not None:
logger.debug('Event queued')
self.__event_queue.put(evt)
except usb.core.USBError as e:
logger.error(
'{} polling exception: {}'.format(repr(self.__dev), e)
)
# check if disconnected
if e.errno == 19:
break
except Exception as e:
logger.error(
'{} polling exception: {}'.format(repr(self.__dev), e)
)
from construct import Struct, Union, Enum, SLInt32
announcement_type = Enum(
SLInt32("type"),
LAUNCH=0,
EXIT=1,
)
announcement = Struct(
"announcement",
announcement_type,
Union(
"args",
SLInt32("rank"),
),
)
ANNOUNCEMENT_PACKET_LEN = announcement.sizeof()
class BOMAdapter(Adapter):
def _encode(self, obj, context, path):
data = Struct("entries" / ENTRY[context.n_entries],
"zlength" / GreedyRange(Int16ub)).build(obj)
return encrypt_bom(data)
def _decode(self, obj, context, path):
data = decrypt_bom(obj)
return Struct("entries" / ENTRY[context.n_entries],
"zlength" / GreedyRange(Int16ub)).parse(data)
VERSION = 65540
ENTRY_SIZE = ENTRY.sizeof()
BLOCK_SIZE = 2**16
ARCHIVE_FLAGS = 4
HEADER = Struct(
"MAGIC" / Const(b"PSAR"),
"VERSION" / Const(Int32ub.build(VERSION)),
"COMPRESSION" / Const(b"zlib"),
"header_size" / Int32ub,
"ENTRY_SIZE" / Const(Int32ub.build(ENTRY_SIZE)),
"n_entries" / Int32ub,
"BLOCK_SIZE" / Const(Int32ub.build(BLOCK_SIZE)),
"ARCHIVE_FLAGS" / Const(Int32ub.build(ARCHIVE_FLAGS)),
"bom" / BOMAdapter(Bytes(this.header_size - 32)),
)
Padding(12), # reserved
"physical_drive_number" / Byte,
Padding(1), # reserved
"extended_bootsignature" / Byte,
"volume_id" / Int32ul,
"volume_label" / Bytes(11),
"fs_type" / Bytes(8),
"boot_code" / Bytes(420),
"boot_sector_signature" / Bytes(2),
)
# ready to use bootsector definition for FAT12 and FAT16
FAT12_16_BOOTSECTOR = Struct(
Embedded(FAT_CORE_BOOTSECTOR),
Embedded(FAT12_16_EXTENDED_BOOTSECTOR),
Padding(this.sector_size - FAT_CORE_BOOTSECTOR.sizeof() -
FAT12_16_EXTENDED_BOOTSECTOR.sizeof()),
)
# ready to use bootsector definition for FAT32
FAT32_BOOTSECTOR = Struct(
Embedded(FAT_CORE_BOOTSECTOR),
Embedded(FAT32_EXTENDED_BOOTSECTOR),
Padding(this.sector_size - FAT_CORE_BOOTSECTOR.sizeof()
- FAT32_EXTENDED_BOOTSECTOR.sizeof()
),
Embedded(FS_INFORMATION_SECTOR),
Padding(this.sector_size
- FS_INFORMATION_SECTOR.sizeof()
),
)
from construct import Struct, Byte, Bytes, Int16ul, Int32ul, Enum, Padding, Embedded, Pass, BitStruct, Flag, Const, this
Fat16Header = Struct(
"jumpInstruction" / Bytes(3), "creatingSystemId" / Bytes(8), "sectorSize" /
Int16ul, "sectorsPerCluster" / Byte, "reservedSectorCount" / Int16ul,
"fatCount" / Byte, "rootdirEntryCount" / Int16ul,
"sectorCount_small" / Int16ul, "mediaId" / Byte, "sectorsPerFat" / Int16ul,
"sectorsPerTrack" / Int16ul, "sideCount" / Int16ul,
"hiddenSectorCount" / Int32ul, "sectorCount_large" / Int32ul,
"physicalDriveNumber" / Byte, "currentHead" / Byte,
"extendedBootSignature" / Byte, "volumeId" / Bytes(4),
"volumeLabel" / Bytes(11), "fsType" / Const(b"FAT16 "),
"bootCode" / Bytes(448), "bootSectorSignature" / Const(b"\x55\xaa"))
BootSector = Struct(Embedded(Fat16Header),
Padding(this.sectorSize - Fat16Header.sizeof()))
FatEntry = Enum(Int16ul,
free_cluster=0x0000,
bad_cluster=0xfff7,
last_cluster=0xffff,
default=Pass)
DirEntry = Struct(
"name" / Bytes(8),
"extension" / Bytes(3),
"attributes" /
BitStruct("unused" / Flag, "device" / Flag, "archive" / Flag,
"subDirectory" / Flag, "volumeLabel" / Flag, "system" / Flag,
"hidden" / Flag, "readonly" / Flag),
# reserved
class IPTransport(object):
'''Implement IP transport.'''
def __init__(self, device=None):
'''Instantiate the first available PTP device over IP'''
self.__setup_constructors()
logger.debug('Init IP')
self.__dev = device
if device is None:
raise NotImplementedError(
'IP discovery not implemented. Please provide a device.'
)
self.__device = device
# Signal usable implicit session
self.__implicit_session_open = Event()
# Signal implicit session is shutting down
self.__implicit_session_shutdown = Event()
self.__check_session_lock = Lock()
self.__transaction_lock = Lock()
self.__event_queue = Queue()
atexit.register(self._shutdown)
def _shutdown(self):
try:
self.__close_implicit_session()
except Exception as e:
logger.error(e)
@contextmanager
def __implicit_session(self):
'''Manage implicit sessions with responder'''
# There is now an implicit session
self.__check_session_lock.acquire()
if not self.__implicit_session_open.is_set():
try:
self.__open_implicit_session()
self.__check_session_lock.release()
yield
except Exception as e:
logger.error(e)
raise PTPError('Failed to open PTP/IP implicit session')
finally:
if self.__implicit_session_open.is_set():
self.__close_implicit_session()
if self.__check_session_lock.locked():
self.__check_session_lock.release()
else:
self.__check_session_lock.release()
yield
def __open_implicit_session(self):
'''Establish implicit session with responder'''
self.__implicit_session_shutdown.clear()
# Establish Command and Event connections
if type(self.__device) is tuple:
host, port = self.__device
self.__setup_connection(host, port)
else:
self.__setup_connection(self.__device)
self.__implicit_session_open.set()
# Prepare Event and Probe threads
self.__event_proc = Thread(
name='EvtPolling',
target=self.__poll_events
)
self.__event_proc.daemon = False
self.__ping_pong_proc = Thread(
name='PingPong',
target=self.__ping_pong
)
self.__ping_pong_proc.daemon = False
# Launch Event and Probe threads
self.__event_proc.start()
self.__ping_pong_proc.start()
def __close_implicit_session(self):
'''Terminate implicit session with responder'''
self.__implicit_session_shutdown.set()
if not self.__implicit_session_open.is_set():
return
# Only join running threads.
if self.__event_proc.is_alive():
self.__event_proc.join(2)
if self.__ping_pong_proc.is_alive():
self.__ping_pong_proc.join(2)
logger.debug('Close connections for {}'.format(repr(self.__dev)))
try:
self.__evtcon.shutdown(socket.SHUT_RDWR)
except socket.error as e:
if e.errno == 107:
pass
else:
raise e
try:
self.__cmdcon.shutdown(socket.SHUT_RDWR)
except socket.error as e:
if e.errno == 107:
pass
else:
raise e
self.__evtcon.close()
self.__cmdcon.close()
self.__implicit_session_open.clear()
def __setup_connection(self, host=None, port=15740):
'''Establish a PTP/IP session for a given host'''
logger.debug(
'Establishing PTP/IP connection with {}:{}'
.format(host, port)
)
socket.setdefaulttimeout(5)
hdrlen = self.__Header.sizeof()
# Command Connection Establishment
self.__cmdcon = create_connection((host, port))
# Send InitCommand
# TODO: Allow users to identify as an arbitrary initiator.
init_cmd_req_payload = self.__InitCommand.build(
Container(
InitiatorGUID=16*[0xFF],
InitiatorFriendlyName='PTPy',
InitiatorProtocolVersion=Container(
Major=100,
Minor=000,
),
))
init_cmd_req = self.__Packet.build(
Container(
Type='InitCommand',
Payload=init_cmd_req_payload,
)
)
actual_socket(self.__cmdcon).sendall(init_cmd_req)
# Get ACK/NACK
init_cmd_req_rsp = actual_socket(self.__cmdcon).recv(72)
init_cmd_rsp_hdr = self.__Header.parse(
init_cmd_req_rsp[0:hdrlen]
)
if init_cmd_rsp_hdr.Type == 'InitCommandAck':
cmd_ack = self.__InitCommandACK.parse(init_cmd_req_rsp[hdrlen:])
logger.debug(
'Command connection ({}) established'
.format(cmd_ack.ConnectionNumber)
)
elif init_cmd_rsp_hdr.Type == 'InitFail':
cmd_nack = self.__InitFail.parse(init_cmd_req_rsp[hdrlen:])
msg = 'InitCommand failed, Reason: {}'.format(
cmd_nack
)
logger.error(msg)
raise PTPError(msg)
else:
msg = 'Unexpected response Type to InitCommand : {}'.format(
init_cmd_rsp_hdr.Type
)
logger.error(msg)
raise PTPError(msg)
# Event Connection Establishment
self.__evtcon = create_connection((host, port))
self.__evtcon.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
self.__evtcon.setsockopt(socket.IPPROTO_TCP, socket.SO_KEEPALIVE, 1)
# Send InitEvent
payload = self.__InitEvent.build(Container(
ConnectionNumber=cmd_ack.ConnectionNumber,
))
evt_req = self.__Packet.build(
Container(
Type='InitEvent',
Payload=payload,
)
)
actual_socket(self.__evtcon).sendall(evt_req)
# Get ACK/NACK
init_evt_req_rsp = actual_socket(self.__evtcon).recv(
hdrlen + self.__InitFail.sizeof()
)
init_evt_rsp_hdr = self.__Header.parse(
init_evt_req_rsp[0:hdrlen]
)
if init_evt_rsp_hdr.Type == 'InitEventAck':
logger.debug(
'Event connection ({}) established'
.format(cmd_ack.ConnectionNumber)
)
elif init_evt_rsp_hdr.Type == 'InitFail':
evt_nack = self.__InitFail.parse(init_evt_req_rsp[hdrlen:])
msg = 'InitEvent failed, Reason: {}'.format(
evt_nack
)
logger.error(msg)
raise PTPError(msg)
else:
msg = 'Unexpected response Type to InitEvent : {}'.format(
init_evt_rsp_hdr.Type
)
logger.error(msg)
raise PTPError(msg)
# Helper methods.
# ---------------------
def __setup_constructors(self):
'''Set endianness and create transport-specific constructors.'''
# Set endianness of constructors before using them.
self._set_endian('little')
self.__Length = Int32ul
self.__Type = Enum(
Int32ul,
Undefined=0x00000000,
InitCommand=0x00000001,
InitCommandAck=0x00000002,
InitEvent=0x00000003,
InitEventAck=0x00000004,
InitFail=0x00000005,
Command=0x00000006,
Response=0x00000007,
Event=0x00000008,
StartData=0x00000009,
Data=0x0000000A,
Cancel=0x0000000B,
EndData=0x0000000C,
Ping=0x0000000D,
Pong=0x0000000E,
)
self.__Header = Struct(
'Length' / self.__Length,
'Type' / self.__Type,
)
self.__Param = Range(0, 5, self._Parameter)
self.__EventParam = Range(0, 3, self._Parameter)
self.__PacketBase = Struct(
Embedded(self.__Header),
'Payload' / Bytes(
lambda ctx, h=self.__Header: ctx.Length - h.sizeof()),
)
self.__Packet = ExprAdapter(
self.__PacketBase,
encoder=lambda obj, ctx, h=self.__Header: Container(
Length=len(obj.Payload) + h.sizeof(),
**obj
),
decoder=lambda obj, ctx: obj,
)
# Yet another arbitrary string type. Max-length CString utf8-encoded
self.__PTPIPString = ExprAdapter(
RepeatUntil(
lambda obj, ctx, lst:
six.unichr(obj) in '\x00' or len(lst) == 40, Int16ul
),
encoder=lambda obj, ctx:
[] if len(obj) == 0 else[ord(c) for c in six.text_type(obj)]+[0],
decoder=lambda obj, ctx:
u''.join(
[six.unichr(o) for o in obj]
).split('\x00')[0],
)
# PTP/IP packets
# Command
self.__ProtocolVersion = Struct(
'Major' / Int16ul,
'Minor' / Int16ul,
)
self.__InitCommand = Embedded(Struct(
'InitiatorGUID' / Array(16, Int8ul),
'InitiatorFriendlyName' / self.__PTPIPString,
'InitiatorProtocolVersion' / self.__ProtocolVersion,
))
self.__InitCommandACK = Embedded(Struct(
'ConnectionNumber' / Int32ul,
'ResponderGUID' / Array(16, Int8ul),
'ResponderFriendlyName' / self.__PTPIPString,
'ResponderProtocolVersion' / self.__ProtocolVersion,
))
# Event
self.__InitEvent = Embedded(Struct(
'ConnectionNumber' / Int32ul,
))
# Common to Events and Command requests
self.__Reason = Enum(
# TODO: Verify these codes...
Int32ul,
Undefined=0x0000,
RejectedInitiator=0x0001,
Busy=0x0002,
Unspecified=0x0003,
)
self.__InitFail = Embedded(Struct(
'Reason' / self.__Reason,
))
self.__DataphaseInfo = Enum(
Int32ul,
Undefined=0x00000000,
In=0x00000001,
Out=0x00000002,
)
self.__Command = Embedded(Struct(
'DataphaseInfo' / self.__DataphaseInfo,
'OperationCode' / self._OperationCode,
'TransactionID' / self._TransactionID,
'Parameter' / self.__Param,
))
self.__Response = Embedded(Struct(
'ResponseCode' / self._ResponseCode,
'TransactionID' / self._TransactionID,
'Parameter' / self.__Param,
))
self.__Event = Embedded(Struct(
'EventCode' / self._EventCode,
'TransactionID' / self._TransactionID,
'Parameter' / self.__EventParam,
))
self.__StartData = Embedded(Struct(
'TransactionID' / self._TransactionID,
'TotalDataLength' / Int64ul,
))
# TODO: Fix packing and unpacking dataphase data
self.__Data = Embedded(Struct(
'TransactionID' / self._TransactionID,
'Data' / Bytes(
lambda ctx:
ctx._.Length -
self.__Header.sizeof() -
self._TransactionID.sizeof()
),
))
self.__EndData = Embedded(Struct(
'TransactionID' / self._TransactionID,
'Data' / Bytes(
lambda ctx:
ctx._.Length -
self.__Header.sizeof() -
self._TransactionID.sizeof()
),
))
self.__Cancel = Embedded(Struct(
'TransactionID' / self._TransactionID,
))
# Convenience construct for parsing packets
self.__PacketPayload = Debugger(Struct(
'Header' / Embedded(self.__Header),
'Payload' / Embedded(Switch(
lambda ctx: ctx.Type,
{
'InitCommand': self.__InitCommand,
'InitCommandAck': self.__InitCommandACK,
'InitEvent': self.__InitEvent,
'InitFail': self.__InitFail,
'Command': self.__Command,
'Response': self.__Response,
'Event': self.__Event,
'StartData': self.__StartData,
'Data': self.__Data,
'EndData': self.__EndData,
},
default=Pass,
))
))
def __parse_response(self, ipdata):
'''Helper method for parsing data.'''
# Build up container with all PTP info.
response = self.__PacketPayload.parse(ipdata)
# Sneak in an implicit Session ID
response['SessionID'] = self.session_id
return response
def __recv(self, event=False, wait=False, raw=False):
'''Helper method for receiving packets.'''
hdrlen = self.__Header.sizeof()
with self.__implicit_session():
ip = (
actual_socket(self.__evtcon)
if event
else actual_socket(self.__cmdcon)
)
data = bytes()
while True:
try:
ipdata = ip.recv(hdrlen)
except socket.timeout:
if event:
return None
else:
ipdata = ip.recv(hdrlen)
if len(ipdata) == 0 and not event:
raise PTPError('Command connection dropped')
elif event:
return None
# Read a single entire header
while len(ipdata) < hdrlen:
ipdata += ip.recv(hdrlen - len(ipdata))
header = self.__Header.parse(
ipdata[0:hdrlen]
)
# Read a single entire packet
while len(ipdata) < header.Length:
ipdata += ip.recv(header.Length - len(ipdata))
# Run sanity checks.
if header.Type not in [
'Cancel',
'Data',
'Event',
'Response',
'StartData',
'EndData',
]:
raise PTPError(
'Unexpected PTP/IP packet type {}'
.format(header.Type)
)
if header.Type not in ['StartData', 'Data', 'EndData']:
break
else:
response = self.__parse_response(ipdata)
if header.Type == 'StartData':
expected = response.TotalDataLength
current_transaction = response.TransactionID
elif (
header.Type == 'Data' and
response.TransactionID == current_transaction
):
data += response.Data
elif (
header.Type == 'EndData' and
response.TransactionID == current_transaction
):
data += response.Data
datalen = len(data)
if datalen != expected:
logger.warning(
'{} data than expected {}/{}'
.format(
'More' if datalen > expected else 'Less',
datalen,
expected
)
)
response['Data'] = data
response['Type'] = 'Data'
return response
if raw:
# TODO: Deal with raw Data packets??
return ipdata
else:
return self.__parse_response(ipdata)
def __send(self, ptp_container, event=False):
'''Helper method for sending packets.'''
packet = self.__Packet.build(ptp_container)
ip = (
actual_socket(self.__evtcon)
if event
else actual_socket(self.__cmdcon)
)
while ip.sendall(packet) is not None:
logger.debug('Failed to send {} packet'.format(ptp_container.Type))
def __send_request(self, ptp_container):
'''Send PTP request without checking answer.'''
# Don't modify original container to keep abstraction barrier.
ptp = Container(**ptp_container)
# Send unused parameters always
ptp['Parameter'] += [0] * (5 - len(ptp.Parameter))
# Send request
ptp['Type'] = 'Command'
ptp['DataphaseInfo'] = 'In'
ptp['Payload'] = self.__Command.build(ptp)
self.__send(ptp)
def __send_data(self, ptp_container, data):
'''Send data without checking answer.'''
# Don't modify original container to keep abstraction barrier.
ptp = Container(**ptp_container)
# Send data
ptp['Type'] = 'Data'
ptp['DataphaseInfo'] = 'Out'
ptp['Payload'] = data
self.__send(ptp)
# Actual implementation
# ---------------------
def send(self, ptp_container, data):
'''Transfer operation with dataphase from initiator to responder'''
logger.debug('SEND {}{}'.format(
ptp_container.OperationCode,
' ' + str(list(map(hex, ptp_container.Parameter)))
if ptp_container.Parameter else '',
))
with self.__implicit_session():
with self.__transaction_lock:
self.__send_request(ptp_container)
self.__send_data(ptp_container, data)
# Get response and sneak in implicit SessionID and missing
# parameters.
return self.__recv()
def recv(self, ptp_container):
'''Transfer operation with dataphase from responder to initiator.'''
logger.debug('RECV {}{}'.format(
ptp_container.OperationCode,
' ' + str(list(map(hex, ptp_container.Parameter)))
if ptp_container.Parameter else '',
))
with self.__implicit_session():
with self.__transaction_lock:
self.__send_request(ptp_container)
dataphase = self.__recv()
if hasattr(dataphase, 'Data'):
response = self.__recv()
if (
(ptp_container.TransactionID != dataphase.TransactionID) or
(ptp_container.SessionID != dataphase.SessionID) or
(dataphase.TransactionID != response.TransactionID) or
(dataphase.SessionID != response.SessionID)
):
raise PTPError(
'Dataphase does not match with requested operation'
)
response['Data'] = dataphase.Data
return response
else:
return dataphase
def mesg(self, ptp_container):
'''Transfer operation without dataphase.'''
op = ptp_container['OperationCode']
if op == 'OpenSession':
self.__open_implicit_session()
with self.__implicit_session():
with self.__transaction_lock:
self.__send_request(ptp_container)
# Get response and sneak in implicit SessionID and missing
# parameters for FullResponse.
response = self.__recv()
rc = response['ResponseCode']
if op == 'OpenSession':
if rc != 'OK':
self.__close_implicit_session()
elif op == 'CloseSession':
if rc == 'OK':
self.__close_implicit_session()
return response
def event(self, wait=False):
'''Check event.
If `wait` this function is blocking. Otherwise it may return None.
'''
evt = None
ipdata = None
timeout = None if wait else 0.001
if not self.__event_queue.empty():
ipdata = self.__event_queue.get(block=not wait, timeout=timeout)
if ipdata is not None:
evt = self.__parse_response(ipdata)
return evt
def __poll_events(self):
'''Poll events, adding them to a queue.'''
logger.debug('Start')
while (
not self.__implicit_session_shutdown.is_set() and
self.__implicit_session_open.is_set() and
_main_thread_alive()
):
try:
evt = self.__recv(event=True, wait=False, raw=True)
except OSError as e:
if e.errno == 9 and not self.__implicit_session_open.is_set():
break
else:
raise e
if evt is not None:
logger.debug('Event queued')
self.__event_queue.put(evt)
sleep(5e-3)
logger.debug('Stop')
def __ping_pong(self):
'''Poll events, adding them to a queue.'''
logger.debug('Start')
last = time()
while (
not self.__implicit_session_shutdown.is_set() and
self.__implicit_session_open.is_set() and
_main_thread_alive()
):
if time() - last > 10:
logger.debug('PING')
# TODO: implement Ping Pong
last = time()
sleep(0.10)
logger.debug('Stop')
"virtualAddress" / Int32ul,
"symbolTableIndex" / Int32ul,
"addressExtension" / Int16ul,
"relocationType" / Int16ul,
#relocationType
)
fileHeader = Struct(
"versionID" / Int16ul,
"numSectionHeaders" / Int16ul,
UTCTimeStamp("timestamp"),
"filePoimter" / Int32ul,
"numSymbols" / Int32ul,
"symbols" / Pointer(
this.filePoimter,
Bytes(this.numSymbols * symbol.sizeof()),
),
"stringTable" / Pointer(
(this.filePoimter + (this.numSymbols * symbol.sizeof()) + 0),
ListToBytesAdapter(GreedyRange(Byte))),
"sizeOptHeader" / Int16ul,
"flags" / FlagsEnum(
Int16ul,
F_RELFLG=0x0001,
F_EXEC=0x0002,
F_LNNO=0x0004,
F_LSYMS=0x0008,
F_LITTLE=0x0100,
F_BIG=0x0200,
F_SYMMERGE=0x1000,
),
Bytes("magic", 8),
ULInt32("kernel_size"),
ULInt32("kernel_addr"),
ULInt32("ramdisk_size"),
ULInt32("ramdisk_addr"),
ULInt32("second_size"),
ULInt32("second_addr"),
ULInt32("tags_addr"),
ULInt32("page_size"),
Array(2, ULInt32("unused")),
Bytes("name", 16),
Bytes("cmdline", 512),
Array(8, ULInt32("id")),
Bytes("extra_cmdline", 1024))
_HEADERLEN = _BOOTIMGHDR.sizeof()
_OUT = "{filename}_{start:08x}-{end:08x}.{name}"
def header_to_yaml(filename):
with open(filename, 'rb') as f:
s = f.read(_HEADERLEN)
h = _BOOTIMGHDR.parse(s)
fields = 'name,kernel_size,kernel_addr,ramdisk_size,ramdisk_addr,second_size,second_addr,tags_addr,cmdline,id,extra_cmdline'
for k in fields.split(','):
print('%s -> %s' % (k, h[k]))
with open(filename + '.header.yaml', 'wb') as f:
f.write(yaml.dump(dict([(k, h[k]) for k in fields.split(',')]), default_flow_style=False))
def __init__(self, data):
ip = Struct(
"ip_header",
EmbeddedBitStruct(Const(Nibble("version"), 4),
Nibble("header_length")),
BitStruct("tos", Bits("precedence", 3), Flag("minimize_delay"),
Flag("high_throuput"), Flag("high_reliability"),
Flag("minimize_cost"), Padding(1)),
UBInt16("total_length"),
UBInt16("id"),
UBInt16("flags"),
UBInt8("ttl"),
Enum(UBInt8("proto"),
UDP=0x11,
TCP=0x06,
HOPOPT=0x00,
ICMP=0x01,
IGMP=0x02,
GGP=0x03,
IPoIP=0x04,
ST=0x05,
CBT=0x07,
EGP=0x08,
IGP=0x09,
NVPII=0x0B,
PUP=0x0C,
ARGUS=0x0D,
EMCON=0x0E,
XNET=0x0F,
CHAOS=0x10,
MUX=0x12,
DCNMEAS=0x13,
HMP=0x14,
PRM=0x15,
RDP=0x1B,
IRTP=0x1C,
ISOTP4=0x1D,
DCCP=0x21,
XTP=0x24,
DDP=0x25,
IL=0x28,
IPv6=0x29,
SDRP=0x2A,
IPv6Route=0x2B,
IPv6Frag=0x2C,
IDRP=0x2D,
RSVP=0x2E,
GRE=0x2F,
MHRP=0x30,
BNA=0x31,
ESP=0x32,
AH=0x33,
SWIPE=0x35,
MOBILE=0x37,
TLSP=0x38,
SKIP=0x39,
IPv6ICMP=0x3A,
IPv6NoNxt=0x3B,
IPv6Opts=0x3C,
CFTP=0x3E,
SATEXPAK=0x40,
KRYPTOLAN=0x41,
RVD=0x42,
IPPC=0x43,
SATMON=0x45,
VISA=0x46,
IPCU=0x47,
CPNX=0x48,
CPHB=0x49,
WSN=0x4A,
PVP=0x4B,
BRSATMON=0x4C,
SUNND=0x4D,
WBMON=0x4E,
WBEXPAK=0x4F,
ISOIP=0x50,
VMTP=0x51,
SECUREVMTP=0x52,
VINES=0x53,
TTP=0x54,
IPTM=0x54,
NSFNETIGP=0x55,
DGP=0x56,
TCF=0x57,
EIGRP=0x58,
OSPF=0x59,
LARP=0x5B,
MTP=0x5C,
IPIP=0x5E,
MICP=0x5F,
SCCSP=0x60,
ETHERIP=0x61,
ENCAP=0x62,
GMTP=0x64,
IFMP=0x65,
PNNI=0x66,
PIM=0x67,
ARIS=0x68,
SCPS=0x69,
QNX=0x6A,
IPComp=0x6C,
SNP=0x6D,
VRRP=0x70,
PGM=0x71,
L2TP=0x73,
DDX=0x74,
IATP=0x75,
STP=0x76,
SRP=0x77,
UTI=0x78,
SMP=0x79,
SM=0x7A,
PTP=0x7B,
ISIS=0x7C,
FIRE=0x7D,
CRTP=0x7E,
CRUDP=0x7F,
SSCOPMCE=0x80,
IPLT=0x81,
SPS=0x82,
SCTP=0x84,
FC=0x85,
UDPLite=0x88,
MPLSoIP=0x89,
manet=0x8A,
HIP=0x8B,
Shim6=0x8C,
WESP=0x8D,
ROHC=0x8E),
UBInt16("checksum"),
UBInt32("src"),
UBInt32("dst"),
)
self.ip = ip.parse(data[:ip.sizeof()])
self.ip.src = utils.IPv4Address(self.ip.src)
self.ip.dst = utils.IPv4Address(self.ip.dst)
data = data[ip.sizeof():]
if self.ip.proto in ('TCP', 'UDP'):
self.proto = Struct(
"proto",
UBInt16("sport"),
UBInt16("dport"),
).parse(data)
self.ip_sport = self.proto.sport
self.ip_dport = self.proto.dport
MINIDUMP_MEMORY_INFO = Struct(
"BaseAddress" / Hex(Int64ul),
"AllocationBase" / Hex(Int64ul),
"AllocationProtect" / Hex(Int32ul),
"__alignment1" / Default(Int32ul, 0),
"RegionSize" / Int64ul,
"State" / Int32ul,
"Protect" / Hex(Int32ul),
"Type" / Int32ul,
"__alignment2" / Default(Int32ul, 0)
)
# https://docs.microsoft.com/en-us/windows/win32/api/minidumpapiset/ns-minidumpapiset-minidump_memory_info_list
MINIDUMP_MEMORY_INFO_LIST = Struct(
"SizeOfHeader" / Default(Int32ul, 16),
"SizeOfEntry" / Default(Int32ul, MINIDUMP_MEMORY_INFO.sizeof()),
"NumberOfEntries" / Int64ul,
"memory_infos" / MINIDUMP_MEMORY_INFO[this.NumberOfEntries]
)
# https://docs.microsoft.com/en-us/windows/win32/api/minidumpapiset/ns-minidumpapiset-minidump_memory_descriptor
minidump_memory_descriptor = Struct(
"StartOfMemoryRange" / Hex(Int64ul),
"Memory" / minidump_location_descriptor,
)
# https://docs.microsoft.com/en-us/windows/win32/api/minidumpapiset/ns-minidumpapiset-minidump_memory_descriptor
MINIDUMP_MEMORY_DESCRIPTOR64 = Struct(
"StartOfMemoryRange" / Hex(Int64ul),
"DataSize" / Hex(Int64ul),
)
"symbolTableIndex" / Int32ul,
"addressExtension" / Int16ul,
"relocationType" / Int16ul,
#relocationType
)
fileHeader = Struct(
"versionID" / Int16ul,
"numSectionHeaders" / Int16ul,
UTCTimeStamp("timestamp"),
"filePoimter" / Int32ul,
"numSymbols" / Int32ul,
"symbols" / Pointer(this.filePoimter,
Bytes(this.numSymbols * symbol.sizeof()),
),
"stringTable" / Pointer( (this.filePoimter + (this.numSymbols * symbol.sizeof()) + 0),
ListToBytesAdapter(GreedyRange(Byte))
),
"sizeOptHeader" / Int16ul,
"flags" / FlagsEnum(Int16ul,
F_RELFLG = 0x0001,
F_EXEC = 0x0002,
F_LNNO = 0x0004,
F_LSYMS = 0x0008,
class USBTransport(object):
'''Implement USB transport.'''
def __init__(self, device=None):
'''Instantiate the first available PTP device over USB'''
logger.debug('Init USB')
self.__setup_constructors()
# If no device is specified, find all devices claiming to be Cameras
# and get the USB endpoints for the first one that works.
if device is None:
logger.debug('No device provided, probing all USB devices.')
if isinstance(device, six.string_types):
name = device
logger.debug(
'Device name provided, probing all USB devices for {}.'.format(
name))
device = None
else:
name = None
devs = ([device] if
(device is not None) else find_usb_cameras(name=name))
self.__acquire_camera(devs)
self.__event_queue = Queue()
self.__event_shutdown = Event()
# Locks for different end points.
self.__inep_lock = RLock()
self.__intep_lock = RLock()
self.__outep_lock = RLock()
self.__event_proc = Thread(name='EvtPolling',
target=self.__poll_events)
self.__event_proc.daemon = False
atexit.register(self._shutdown)
self.__event_proc.start()
def __available_cameras(self, devs):
for dev in devs:
if self.__setup_device(dev):
logger.debug('Found USB PTP device {}'.format(dev))
yield
else:
message = 'No USB PTP device found.'
logger.error(message)
raise PTPError(message)
def __acquire_camera(self, devs):
'''From the cameras given, get the first one that does not fail'''
for _ in self.__available_cameras(devs):
try:
if self.__dev.is_kernel_driver_active(
self.__intf.bInterfaceNumber):
try:
self.__dev.detach_kernel_driver(
self.__intf.bInterfaceNumber)
usb.util.claim_interface(self.__dev, self.__intf)
except usb.core.USBError:
message = ('Could not detach kernel driver. '
'Maybe the camera is mounted?')
logger.error(message)
logger.debug('Claiming {}'.format(repr(self.__dev)))
usb.util.claim_interface(self.__dev, self.__intf)
except Exception as e:
logger.debug('{}'.format(e))
continue
break
else:
message = ('Could acquire any camera.')
logger.error(message)
raise PTPError(message)
def _shutdown(self):
logger.debug('Shutdown request')
self.__event_shutdown.set()
# Free USB resource on shutdown.
# Only join a running thread.
if self.__event_proc.is_alive():
self.__event_proc.join(2)
logger.debug('Release {}'.format(repr(self.__dev)))
usb.util.release_interface(self.__dev, self.__intf)
# Helper methods.
# ---------------------
def __setup_device(self, dev):
'''Get endpoints for a device. True on success.'''
self.__inep = None
self.__outep = None
self.__intep = None
self.__cfg = None
self.__dev = None
self.__intf = None
# Attempt to find the USB in, out and interrupt endpoints for a PTP
# interface.
for cfg in dev:
for intf in cfg:
if intf.bInterfaceClass == PTP_USB_CLASS:
for ep in intf:
ep_type = endpoint_type(ep.bmAttributes)
ep_dir = endpoint_direction(ep.bEndpointAddress)
if ep_type == ENDPOINT_TYPE_BULK:
if ep_dir == ENDPOINT_IN:
self.__inep = ep
elif ep_dir == ENDPOINT_OUT:
self.__outep = ep
elif ((ep_type == ENDPOINT_TYPE_INTR)
and (ep_dir == ENDPOINT_IN)):
self.__intep = ep
if not (self.__inep and self.__outep and self.__intep):
self.__inep = None
self.__outep = None
self.__intep = None
else:
logger.debug('Found {}'.format(repr(self.__inep)))
logger.debug('Found {}'.format(repr(self.__outep)))
logger.debug('Found {}'.format(repr(self.__intep)))
self.__cfg = cfg
self.__dev = dev
self.__intf = intf
return True
return False
def __setup_constructors(self):
'''Set endianness and create transport-specific constructors.'''
# Set endianness of constructors before using them.
self._set_endian('little')
self.__Length = Int32ul
self.__Type = Enum(
Int16ul,
default=Pass,
Undefined=0x0000,
Command=0x0001,
Data=0x0002,
Response=0x0003,
Event=0x0004,
)
# This is just a convenience constructor to get the size of a header.
self.__Code = Int16ul
self.__Header = Struct(
'Length' / self.__Length,
'Type' / self.__Type,
'Code' / self.__Code,
'TransactionID' / self._TransactionID,
)
# These are the actual constructors for parsing and building.
self.__CommandHeader = Struct(
'Length' / self.__Length,
'Type' / self.__Type,
'OperationCode' / self._OperationCode,
'TransactionID' / self._TransactionID,
)
self.__ResponseHeader = Struct(
'Length' / self.__Length,
'Type' / self.__Type,
'ResponseCode' / self._ResponseCode,
'TransactionID' / self._TransactionID,
)
self.__EventHeader = Struct(
'Length' / self.__Length,
'Type' / self.__Type,
'EventCode' / self._EventCode,
'TransactionID' / self._TransactionID,
)
# Apparently nobody uses the SessionID field. Even though it is
# specified in ISO15740:2013(E), no device respects it and the session
# number is implicit over USB.
self.__Param = Range(0, 5, self._Parameter)
self.__CommandTransactionBase = Struct(
Embedded(self.__CommandHeader), 'Payload' /
Bytes(lambda ctx, h=self.__Header: ctx.Length - h.sizeof()))
self.__CommandTransaction = ExprAdapter(
self.__CommandTransactionBase,
encoder=lambda obj, ctx, h=self.__Header: Container(
Length=len(obj.Payload) + h.sizeof(), **obj),
decoder=lambda obj, ctx: obj,
)
self.__ResponseTransactionBase = Struct(
Embedded(self.__ResponseHeader), 'Payload' /
Bytes(lambda ctx, h=self.__Header: ctx.Length - h.sizeof()))
self.__ResponseTransaction = ExprAdapter(
self.__ResponseTransactionBase,
encoder=lambda obj, ctx, h=self.__Header: Container(
Length=len(obj.Payload) + h.sizeof(), **obj),
decoder=lambda obj, ctx: obj,
)
def __parse_response(self, usbdata):
'''Helper method for parsing USB data.'''
# Build up container with all PTP info.
usbdata = bytearray(usbdata)
transaction = self.__ResponseTransaction.parse(usbdata)
response = Container(
SessionID=self.session_id,
TransactionID=transaction.TransactionID,
)
if transaction.Type == 'Response':
response['ResponseCode'] = transaction.ResponseCode
response['Parameter'] = self.__Param.parse(transaction.Payload)
elif transaction.Type == 'Event':
event = self.__EventHeader.parse(usbdata[0:self.__Header.sizeof()])
response['EventCode'] = event.EventCode
response['Parameter'] = self.__Param.parse(transaction.Payload)
else:
command = self.__CommandHeader.parse(
usbdata[0:self.__Header.sizeof()])
response['OperationCode'] = command.OperationCode
response['Data'] = transaction.Payload
return response
def __recv(self, event=False, wait=False, raw=False):
'''Helper method for receiving data.'''
# TODO: clear stalls automatically
ep = self.__intep if event else self.__inep
lock = self.__intep_lock if event else self.__inep_lock
with lock:
try:
usbdata = ep.read(ep.wMaxPacketSize, timeout=0 if wait else 5)
except usb.core.USBError as e:
# Ignore timeout or busy device once.
if e.errno == 110 or e.errno == 16:
if event:
return None
else:
usbdata = ep.read(ep.wMaxPacketSize, timeout=5000)
else:
raise e
header = self.__ResponseHeader.parse(
bytearray(usbdata[0:self.__Header.sizeof()]))
if header.Type not in ['Response', 'Data', 'Event']:
raise PTPError(
'Unexpected USB transfer type.'
'Expected Response, Event or Data but received {}'.format(
header.Type))
while len(usbdata) < header.Length:
usbdata += ep.read(ep.wMaxPacketSize, timeout=5000)
if raw:
return usbdata
else:
return self.__parse_response(usbdata)
def __send(self, ptp_container, event=False):
'''Helper method for sending data.'''
ep = self.__intep if event else self.__outep
lock = self.__intep_lock if event else self.__outep_lock
transaction = self.__CommandTransaction.build(ptp_container)
with lock:
try:
ep.write(transaction, timeout=1)
except usb.core.USBError as e:
# Ignore timeout or busy device once.
if e.errno == 110 or e.errno == 16:
ep.write(transaction, timeout=5000)
def __send_request(self, ptp_container):
'''Send PTP request without checking answer.'''
# Don't modify original container to keep abstraction barrier.
ptp = Container(**ptp_container)
# Don't send unused parameters
try:
while not ptp.Parameter[-1]:
ptp.Parameter.pop()
if len(ptp.Parameter) == 0:
break
except IndexError:
# The Parameter list is already empty.
pass
# Send request
ptp['Type'] = 'Command'
ptp['Payload'] = self.__Param.build(ptp.Parameter)
self.__send(ptp)
def __send_data(self, ptp_container, data):
'''Send data without checking answer.'''
# Don't modify original container to keep abstraction barrier.
ptp = Container(**ptp_container)
# Send data
ptp['Type'] = 'Data'
ptp['Payload'] = data
self.__send(ptp)
# Actual implementation
# ---------------------
def send(self, ptp_container, data):
'''Transfer operation with dataphase from initiator to responder'''
datalen = len(data)
logger.debug('SEND {} {} bytes{}'.format(
ptp_container.OperationCode,
datalen,
' ' + str(list(map(hex, ptp_container.Parameter)))
if ptp_container.Parameter else '',
))
self.__send_request(ptp_container)
self.__send_data(ptp_container, data)
# Get response and sneak in implicit SessionID and missing parameters.
response = self.__recv()
logger.debug('SEND {} {} bytes {}{}'.format(
ptp_container.OperationCode,
datalen,
response.ResponseCode,
' ' + str(list(map(hex, response.Parameter)))
if ptp_container.Parameter else '',
))
return response
def recv(self, ptp_container):
'''Transfer operation with dataphase from responder to initiator.'''
logger.debug('RECV {}{}'.format(
ptp_container.OperationCode,
' ' + str(list(map(hex, ptp_container.Parameter)))
if ptp_container.Parameter else '',
))
self.__send_request(ptp_container)
dataphase = self.__recv()
if hasattr(dataphase, 'Data'):
response = self.__recv()
if ((ptp_container.OperationCode != dataphase.OperationCode)
or (ptp_container.TransactionID != dataphase.TransactionID)
or (ptp_container.SessionID != dataphase.SessionID)
or (dataphase.TransactionID != response.TransactionID)
or (dataphase.SessionID != response.SessionID)):
raise PTPError(
'Dataphase does not match with requested operation.')
response['Data'] = dataphase.Data
else:
response = dataphase
logger.debug('RECV {} {}{}{}'.format(
ptp_container.OperationCode,
response.ResponseCode,
' {} bytes'.format(len(response.Data)) if hasattr(
response, 'Data') else '',
' ' + str(list(map(hex, response.Parameter)))
if response.Parameter else '',
))
return response
def mesg(self, ptp_container):
'''Transfer operation without dataphase.'''
logger.debug('MESG {}{}'.format(
ptp_container.OperationCode,
' ' + str(list(map(hex, ptp_container.Parameter)))
if ptp_container.Parameter else '',
))
self.__send_request(ptp_container)
# Get response and sneak in implicit SessionID and missing parameters
# for FullResponse.
response = self.__recv()
logger.debug('MESG {} {}{}'.format(
ptp_container.OperationCode,
response.ResponseCode,
' ' + str(list(map(hex, response.Parameter)))
if response.Parameter else '',
))
return response
def event(self, wait=False):
'''Check event.
If `wait` this function is blocking. Otherwise it may return None.
'''
evt = None
usbdata = None
timeout = None if wait else 0.001
if not self.__event_queue.empty():
usbdata = self.__event_queue.get(block=not wait, timeout=timeout)
if usbdata is not None:
evt = self.__parse_response(usbdata)
return evt
def __poll_events(self):
'''Poll events, adding them to a queue.'''
while not self.__event_shutdown.is_set() and _main_thread_alive():
evt = self.__recv(event=True, wait=False, raw=True)
if evt is not None:
logger.debug('Event queued')
self.__event_queue.put(evt)
def checksum_16(data):
c = 0
for i in range(len(data) // 2):
c = c + (data[i * 2 + 1] << 8) + data[i * 2]
return c & 0xFFFF
WriteCommand = Struct("address" / Int16ul, "value" / Int16ul)
WriteMessageRequest = Struct(
"fields" / RawCopy(
Struct(
"length" / Rebuild(
Int16ul,
len_(this.items) * WriteCommand.sizeof() // Int16ul.sizeof() +
2,
),
"command" /
Const(vlxDevConstants.WS_WEB_UI_COMMAND_WRITE_DATA, Int16ul),
"items" / WriteCommand[len_(this.items)],
)),
"checksum" /
Checksum(Int16ul, lambda data: checksum_16(data), this.fields.data),
)
ReadTableResponse = GreedyRange(Int16ub)
ReadTableRequest = Struct(
"fields" / RawCopy(
Struct(
def ssexy_linux(fname, *eips):
import elf32
from construct import Struct, ULInt32, ULInt16, ULInt8, Array, CString
from construct import OptionalGreedyRange
# assume low-endian binary
elf32_rel = Struct('elf32_rel', ULInt32('r_offset'), ULInt32('r_info'))
ELF32_R_SYM = lambda x: x.r_info >> 8
ELF32_R_TYPE = lambda x: x.r_info & 0xff
R_386_PC32 = 2
elf32_sym = Struct('elf32_sym', ULInt32('st_name'), ULInt32('st_value'),
ULInt32('st_size'), ULInt8('st_info'), ULInt8('st_other'),
ULInt16('st_shndx'))
elf = elf32.elf32_file.parse_stream(file(fname, 'rb'))
# retrieve section by name
elf32_section = lambda elf, name: [x for x in elf.sections
if x.name == name][0]
# for now we assume that all code is in the .text section
code_section = [x for x in elf.sections if x.name == '.text']
if not len(code_section):
raise Exception('your binary doesn\'t have a .text section..')
relocs = [x.data.value for x in elf.sections if x.name == '.rel.dyn']
if not len(relocs):
raise Exception('no relocs available, compile with -pie')
# read all relocations
relocs = Array(len(relocs[0]) / elf32_rel.sizeof(),
elf32_rel).parse(relocs[0])
# now get the offsets of the relocations
relocs = set([x.r_offset for x in relocs])
imports = {}
# a list of addresses that were used.
addresses = []
# a list of all m128 values we use
m128s = []
# a list of all dword values we use
m32s = []
instructions = pyasm2.block()
# get string at offset
dynstr = lambda x: CString(None).parse(
elf32_section(elf, '.dynstr').data.value[x:])
# read the symbol table
imports = OptionalGreedyRange(elf32_sym).parse(elf32_section(elf,
'.dynsym').data.value)
# resolve relocations
section = elf32_section(elf, '.rel.dyn')
relocates = {}
for x in xrange(0, section.size, elf32_rel.sizeof()):
x = elf32_rel.parse(section.data.value[x:x+elf32_rel.sizeof()])
# relocation to fixup addresses to imports
if ELF32_R_TYPE(x) == R_386_PC32:
relocates[x.r_offset] = dynstr(imports[ELF32_R_SYM(x)].st_name)
# walk each section, find those that are executable and disassemble those
section = elf32_section(elf, '.text')
g = distorm3.DecomposeGenerator(section.addr, section.data.value,
distorm3.Decode32Bits)
for instr in g:
# useless instruction?
if str(instr) in ('NOP', 'ADD [EAX], AL', 'LEA ESI, [ESI]',
'INT 3') or str(instr)[:2] == 'DB':
continue
# a jump to one of the imports?
#if instr.mnemonic == 'JMP' and instr.operands[0].type == \
# distorm3.OPERAND_ABSOLUTE_ADDRESS and \
# instr.operands[0].disp in imports:
# iat_label[instr.address] = imports[instr.operands[0].disp]
# continue
# quite hackery, but when the jumps with thunk address have been
# processed, we can be fairly sure that there will be no (legit)
# code anymore.
#if len(iat_label):
# break
#print str(instr)
#print str(instr)
# convert the instruction from distorm3 format to pyasm2 format.
instr = distorm3_to_pyasm2(instr)
# we create the block already here, otherwise our `labelnr' is
# not defined.
#block = pyasm2.block(pyasm2.Label('%08x' % instr.address), instr)
offset_flat = None
addr = instr.address
# now we check if this instruction has a relocation inside it
# not a very efficient way, but oke.
reloc = instr.length > 4 and relocs.intersection(range(
instr.address, instr.address + instr.length - 3))
if reloc:
# make an immediate with `addr' set to True
enable_addr = lambda x: Immediate(int(x), addr=True)
# TODO support for two relocations in one instruction
# (displacement *and* immediate)
reloc = reloc.pop()
if not hasattr(instr, 'op1'):
instr.op1, instr.op2 = None, None
# there is only one operand, that's easy
if not instr.op2:
#sys.stderr.write('reloc in op1 %s??\n' % instr.op1)
if isinstance(instr.op1, pyasm2.MemoryAddress):
# special occassion, this memory addres is an import
if instr.op1.reg1 is None and \
instr.op1.reg2 is None and \
int(instr.op1.disp) in imports:
instr.op1 = imports[int(instr.op1.disp)]
else:
addresses.append(int(instr.op1.disp))
# change the displacement to a label
#instr.op1 = str(instr.op1).replace('0x',
# '__lbl_00')
instr.op1 = enable_addr(instr.op1)
elif isinstance(instr.op1, pyasm2.Immediate):
addresses.append(int(instr.op1))
offset_flat = int(instr.op1)
#instr.op1 = str(instr.op1).replace('0x',
# 'offset flat:__lbl_00')
# if the second operand is an immediate and the relocation is
# in the last four bytes of the instruction, then this
# immediate is the reloc. Otherwise, if the second operand is
# a memory address, then it's the displacement.
elif isinstance(instr.op2, pyasm2.Immediate) and reloc == \
instr.address + instr.length - 4:
# keep this address
addresses.append(int(instr.op2))
# make a label from this address
# TODO: fix this horrible hack
offset_flat = int(instr.op2)
#instr.op2 = pyasm2.Label('offset flat:__lbl_%08x' %
# int(instr.op2), prepend=False)
elif isinstance(instr.op2, pyasm2.MemoryAddress) and \
reloc == instr.address + instr.length - 4:
addresses.append(int(instr.op2.disp))
# change the displacement to a label
instr.op2 = enable_addr(instr.op2)
#instr.op2 = str(instr.op2).replace('0x', '__lbl_00')
#sys.stderr.write('reloc in op2 memaddr %s\n' %
# str(instr.op2))
# the relocation is not inside the second operand, it must be
# inside the first operand after all.
elif isinstance(instr.op1, pyasm2.MemoryAddress):
addresses.append(int(instr.op1.disp))
instr.op1 = enable_addr(instr.op1)
#instr.op1 = str(instr.op1).replace('0x', '__lbl_00')
#sys.stderr.write('reloc in op1 memaddr %s\n' %
# str(instr.op1))
elif isinstance(instr.op1, pyasm2.Immediate):
addresses.append(int(instr.op1))
instr.op1 = enable_addr(instr.op1)
#instr.op1 = '__lbl_%08x' % int(instr.op1)
#sys.stderr.write('reloc in op1 imm %s\n' % instr.op1)
else:
sys.stderr.write('Invalid Relocation!\n')
#print instr
m32len = len(m32s)
instr = translate.Translater(instr, m128s, m32s).translate()
if offset_flat:
encode_offset_flat = lambda x: str(x).replace('0x',
'offset flat:__lbl_') if isinstance(x, (int, long,
pyasm2.imm)) and int(x) == offset_flat or isinstance(x,
pyasm2.mem) and x.disp == offset_flat else x
if isinstance(instr, pyasm2.block):
for x in instr.instructions:
x.op1 = encode_offset_flat(x.op1)
x.op2 = encode_offset_flat(x.op2)
else:
x.op1 = encode_offset_flat(x.op1)
x.op2 = encode_offset_flat(x.op2)
# update stuff
m32s = m32s[:m32len] + [x.replace('0x%08x' % offset_flat,
'offset flat:__lbl_%08x' % offset_flat)
for x in m32s[m32len:]]
instructions += pyasm2.block(pyasm2.Label('%08x' % addr), instr)
# remove any addresses that are from within the current section
newlist = addresses[:]
for i in xrange(len(addresses)):
if addresses[i] >= code_section[0].addr and addresses[i] < \
code_section[0].addr + code_section[0].size:
newlist[i] = None
addresses = filter(lambda x: x is not None, newlist)
# walk over each instruction, if it has references, we update them
for instr in instructions.instructions:
# we can skip labels
if isinstance(instr, pyasm2.Label):
continue
# check for references to imports
if isinstance(instr, pyasm2.RelativeJump):
# not very good, but for now (instead of checking relocs) we check
# if the index is in the iat tabel..
#if int(instr.lbl.index, 16) in iat_label:
#instr.lbl.index = iat_label[int(instr.lbl.index, 16)]
#instr.lbl.prepend = False
continue
program = ['.file "ssexy.c"', '.intel_syntax noprefix']
# we walk over each section, if a reference to this section has been found
# then we will dump the entire section as bytecode.. with matching labels
for section in elf.sections:
base = section.addr
data = section.data.value
addr = set(range(base, base + section.size)).intersection(addresses)
if addr:
# create a header for this section
program.append('.section %s' % section.name)
# for now we do it the easy way.. one line and label per byte, lol
for addr in xrange(section.size):
program.append('__lbl_%08x: .byte 0x%02x' % (base + addr,
ord(data[addr])))
# empty line..
program.append('')
# now we define all xmm's etc we gathered
program.append('.align 4')
program += m32s
program.append('.align 16')
program += m128s
# time to define 'main'
program.append('.text')
program.append('.globl Main')
program.append('.type Main, @function')
OEP = elf.entry
# f****d up shit
relocates = dict(('jmp __lbl_%08x' % k, 'jmp ' + v)
for k, v in relocates.items())
eips = ['__lbl_%08x' % x for x in eips]
# append each instruction
for instr in instructions.instructions:
# if this is an label, we want a colon as postfix
if isinstance(instr, pyasm2.Label):
program.append(str(instr) + ':')
# if OEP is at this address, we will also add the `_main' label
if str(instr) == '__lbl_%08x' % OEP:
program.append('Main:')
# we have to initialize the stack register, so..
# for now we assume esp gpr is stored as first gpr in xmm7
program.append('movd xmm7, esp')
# if the label is in the list of addresses to which we have to add
# an "movd xmm7, esp" instruction, then add it (e.g. callback
# function for pthread_create)
if str(instr) in eips:
program.append('movd xmm7, esp')
else:
# TODO: fix this terrible hack as well
program.append(str(instr).replace('byte', 'byte ptr').replace(
'word', 'word ptr').replace('retn', 'ret').replace(
'__lbl_00400000', '0x400000').replace('oword ptr', ''))
if program[-1] in relocates:
program[-1] = relocates[program[-1]]
print '\n'.join(program)
def ssexy_linux(fname, *eips):
import elf32
from construct import Struct, ULInt32, ULInt16, ULInt8, Array, CString
from construct import OptionalGreedyRange
# assume low-endian binary
elf32_rel = Struct('elf32_rel', ULInt32('r_offset'), ULInt32('r_info'))
ELF32_R_SYM = lambda x: x.r_info >> 8
ELF32_R_TYPE = lambda x: x.r_info & 0xff
R_386_PC32 = 2
elf32_sym = Struct('elf32_sym', ULInt32('st_name'), ULInt32('st_value'),
ULInt32('st_size'), ULInt8('st_info'),
ULInt8('st_other'), ULInt16('st_shndx'))
elf = elf32.elf32_file.parse_stream(file(fname, 'rb'))
# retrieve section by name
elf32_section = lambda elf, name: [
x for x in elf.sections if x.name == name
][0]
# for now we assume that all code is in the .text section
code_section = [x for x in elf.sections if x.name == '.text']
if not len(code_section):
raise Exception('your binary doesn\'t have a .text section..')
relocs = [x.data.value for x in elf.sections if x.name == '.rel.dyn']
if not len(relocs):
raise Exception('no relocs available, compile with -pie')
# read all relocations
relocs = Array(len(relocs[0]) / elf32_rel.sizeof(),
elf32_rel).parse(relocs[0])
# now get the offsets of the relocations
relocs = set([x.r_offset for x in relocs])
imports = {}
# a list of addresses that were used.
addresses = []
# a list of all m128 values we use
m128s = []
# a list of all dword values we use
m32s = []
instructions = pyasm2.block()
# get string at offset
dynstr = lambda x: CString(None).parse(
elf32_section(elf, '.dynstr').data.value[x:])
# read the symbol table
imports = OptionalGreedyRange(elf32_sym).parse(
elf32_section(elf, '.dynsym').data.value)
# resolve relocations
section = elf32_section(elf, '.rel.dyn')
relocates = {}
for x in xrange(0, section.size, elf32_rel.sizeof()):
x = elf32_rel.parse(section.data.value[x:x + elf32_rel.sizeof()])
# relocation to fixup addresses to imports
if ELF32_R_TYPE(x) == R_386_PC32:
relocates[x.r_offset] = dynstr(imports[ELF32_R_SYM(x)].st_name)
# walk each section, find those that are executable and disassemble those
section = elf32_section(elf, '.text')
g = distorm3.DecomposeGenerator(section.addr, section.data.value,
distorm3.Decode32Bits)
for instr in g:
# useless instruction?
if str(instr) in ('NOP', 'ADD [EAX], AL', 'LEA ESI, [ESI]',
'INT 3') or str(instr)[:2] == 'DB':
continue
# a jump to one of the imports?
#if instr.mnemonic == 'JMP' and instr.operands[0].type == \
# distorm3.OPERAND_ABSOLUTE_ADDRESS and \
# instr.operands[0].disp in imports:
# iat_label[instr.address] = imports[instr.operands[0].disp]
# continue
# quite hackery, but when the jumps with thunk address have been
# processed, we can be fairly sure that there will be no (legit)
# code anymore.
#if len(iat_label):
# break
#print str(instr)
#print str(instr)
# convert the instruction from distorm3 format to pyasm2 format.
instr = distorm3_to_pyasm2(instr)
# we create the block already here, otherwise our `labelnr' is
# not defined.
#block = pyasm2.block(pyasm2.Label('%08x' % instr.address), instr)
offset_flat = None
addr = instr.address
# now we check if this instruction has a relocation inside it
# not a very efficient way, but oke.
reloc = instr.length > 4 and relocs.intersection(
range(instr.address, instr.address + instr.length - 3))
if reloc:
# make an immediate with `addr' set to True
enable_addr = lambda x: Immediate(int(x), addr=True)
# TODO support for two relocations in one instruction
# (displacement *and* immediate)
reloc = reloc.pop()
if not hasattr(instr, 'op1'):
instr.op1, instr.op2 = None, None
# there is only one operand, that's easy
if not instr.op2:
#sys.stderr.write('reloc in op1 %s??\n' % instr.op1)
if isinstance(instr.op1, pyasm2.MemoryAddress):
# special occassion, this memory addres is an import
if instr.op1.reg1 is None and \
instr.op1.reg2 is None and \
int(instr.op1.disp) in imports:
instr.op1 = imports[int(instr.op1.disp)]
else:
addresses.append(int(instr.op1.disp))
# change the displacement to a label
#instr.op1 = str(instr.op1).replace('0x',
# '__lbl_00')
instr.op1 = enable_addr(instr.op1)
elif isinstance(instr.op1, pyasm2.Immediate):
addresses.append(int(instr.op1))
offset_flat = int(instr.op1)
#instr.op1 = str(instr.op1).replace('0x',
# 'offset flat:__lbl_00')
# if the second operand is an immediate and the relocation is
# in the last four bytes of the instruction, then this
# immediate is the reloc. Otherwise, if the second operand is
# a memory address, then it's the displacement.
elif isinstance(instr.op2, pyasm2.Immediate) and reloc == \
instr.address + instr.length - 4:
# keep this address
addresses.append(int(instr.op2))
# make a label from this address
# TODO: fix this horrible hack
offset_flat = int(instr.op2)
#instr.op2 = pyasm2.Label('offset flat:__lbl_%08x' %
# int(instr.op2), prepend=False)
elif isinstance(instr.op2, pyasm2.MemoryAddress) and \
reloc == instr.address + instr.length - 4:
addresses.append(int(instr.op2.disp))
# change the displacement to a label
instr.op2 = enable_addr(instr.op2)
#instr.op2 = str(instr.op2).replace('0x', '__lbl_00')
#sys.stderr.write('reloc in op2 memaddr %s\n' %
# str(instr.op2))
# the relocation is not inside the second operand, it must be
# inside the first operand after all.
elif isinstance(instr.op1, pyasm2.MemoryAddress):
addresses.append(int(instr.op1.disp))
instr.op1 = enable_addr(instr.op1)
#instr.op1 = str(instr.op1).replace('0x', '__lbl_00')
#sys.stderr.write('reloc in op1 memaddr %s\n' %
# str(instr.op1))
elif isinstance(instr.op1, pyasm2.Immediate):
addresses.append(int(instr.op1))
instr.op1 = enable_addr(instr.op1)
#instr.op1 = '__lbl_%08x' % int(instr.op1)
#sys.stderr.write('reloc in op1 imm %s\n' % instr.op1)
else:
sys.stderr.write('Invalid Relocation!\n')
#print instr
m32len = len(m32s)
instr = translate.Translater(instr, m128s, m32s).translate()
if offset_flat:
encode_offset_flat = lambda x: str(x).replace(
'0x', 'offset flat:__lbl_') if isinstance(
x, (int, long, pyasm2.imm
)) and int(x) == offset_flat or isinstance(
x, pyasm2.mem) and x.disp == offset_flat else x
if isinstance(instr, pyasm2.block):
for x in instr.instructions:
x.op1 = encode_offset_flat(x.op1)
x.op2 = encode_offset_flat(x.op2)
else:
x.op1 = encode_offset_flat(x.op1)
x.op2 = encode_offset_flat(x.op2)
# update stuff
m32s = m32s[:m32len] + [
x.replace('0x%08x' % offset_flat,
'offset flat:__lbl_%08x' % offset_flat)
for x in m32s[m32len:]
]
instructions += pyasm2.block(pyasm2.Label('%08x' % addr), instr)
# remove any addresses that are from within the current section
newlist = addresses[:]
for i in xrange(len(addresses)):
if addresses[i] >= code_section[0].addr and addresses[i] < \
code_section[0].addr + code_section[0].size:
newlist[i] = None
addresses = filter(lambda x: x is not None, newlist)
# walk over each instruction, if it has references, we update them
for instr in instructions.instructions:
# we can skip labels
if isinstance(instr, pyasm2.Label):
continue
# check for references to imports
if isinstance(instr, pyasm2.RelativeJump):
# not very good, but for now (instead of checking relocs) we check
# if the index is in the iat tabel..
#if int(instr.lbl.index, 16) in iat_label:
#instr.lbl.index = iat_label[int(instr.lbl.index, 16)]
#instr.lbl.prepend = False
continue
program = ['.file "ssexy.c"', '.intel_syntax noprefix']
# we walk over each section, if a reference to this section has been found
# then we will dump the entire section as bytecode.. with matching labels
for section in elf.sections:
base = section.addr
data = section.data.value
addr = set(range(base, base + section.size)).intersection(addresses)
if addr:
# create a header for this section
program.append('.section %s' % section.name)
# for now we do it the easy way.. one line and label per byte, lol
for addr in xrange(section.size):
program.append('__lbl_%08x: .byte 0x%02x' %
(base + addr, ord(data[addr])))
# empty line..
program.append('')
# now we define all xmm's etc we gathered
program.append('.align 4')
program += m32s
program.append('.align 16')
program += m128s
# time to define 'main'
program.append('.text')
program.append('.globl Main')
program.append('.type Main, @function')
OEP = elf.entry
# f****d up shit
relocates = dict(
('jmp __lbl_%08x' % k, 'jmp ' + v) for k, v in relocates.items())
eips = ['__lbl_%08x' % x for x in eips]
# append each instruction
for instr in instructions.instructions:
# if this is an label, we want a colon as postfix
if isinstance(instr, pyasm2.Label):
program.append(str(instr) + ':')
# if OEP is at this address, we will also add the `_main' label
if str(instr) == '__lbl_%08x' % OEP:
program.append('Main:')
# we have to initialize the stack register, so..
# for now we assume esp gpr is stored as first gpr in xmm7
program.append('movd xmm7, esp')
# if the label is in the list of addresses to which we have to add
# an "movd xmm7, esp" instruction, then add it (e.g. callback
# function for pthread_create)
if str(instr) in eips:
program.append('movd xmm7, esp')
else:
# TODO: fix this terrible hack as well
program.append(
str(instr).replace('byte', 'byte ptr').replace(
'word', 'word ptr').replace('retn', 'ret').replace(
'__lbl_00400000', '0x400000').replace('oword ptr', ''))
if program[-1] in relocates:
program[-1] = relocates[program[-1]]
print '\n'.join(program)
