class PositionMapping:
__slots__ = ('_pos', '_posmap')
DUPLICATION_CHECK = True
def __init__(self):
self._pos = 0
self._posmap = SortedDict()
def items(self):
return self._posmap.items()
#
# Properties
#
@property
def pos(self):
return self._pos
@pos.setter
def pos(self, v):
self._pos = v
#
# Public methods
#
def add_mapping(self, start_pos, length, obj):
# duplication check
if self.DUPLICATION_CHECK:
try:
pre = next(self._posmap.irange(maximum=start_pos, reverse=True))
if start_pos in self._posmap[pre]:
raise ValueError("New mapping is overlapping with an existing element.")
except StopIteration:
pass
self._posmap[start_pos] = PositionMappingElement(start_pos, length, obj)
def tick_pos(self, delta):
self._pos += delta
def get_node(self, pos):
element = self.get_element(pos)
if element is None:
return None
return element.obj
def get_element(self, pos):
try:
pre = next(self._posmap.irange(maximum=pos, reverse=True))
except StopIteration:
return None
element = self._posmap[pre]
if pos in element:
return element
return None
class Index:
def __init__(self, *keys):
self.keys = keys
self.keyer = operator.attrgetter(*keys)
self.list = SortedDict()
def reindex(self):
rows = [i for i in self.list.values()]
self.list = SortedDict()
for i in rows:
self.add(i)
def add(self, object):
key = self.keyer(object)
self.list[key] = object
def remove(self, object):
key = self.keyer(object)
del self.list[key]
def find(self, start, reverse=False):
if reverse:
return (self.list[i]
for i in self.list.irange(maximum=start, reverse=True))
return (self.list[i] for i in self.list.irange(start))
class PositionMapping:
__slots__ = ('_pos', '_posmap')
DUPLICATION_CHECK = True
def __init__(self):
self._pos = 0
self._posmap = SortedDict()
def items(self):
return self._posmap.items()
#
# Properties
#
@property
def pos(self):
return self._pos
@pos.setter
def pos(self, v):
self._pos = v
#
# Public methods
#
def add_mapping(self, start_pos, length, obj):
# duplication check
if self.DUPLICATION_CHECK:
try:
pre = next(self._posmap.irange(maximum=start_pos, reverse=True))
if start_pos in self._posmap[pre]:
raise ValueError("New mapping is overlapping with an existing element.")
except StopIteration:
pass
self._posmap[start_pos] = PositionMappingElement(start_pos, length, obj)
def tick_pos(self, delta):
self._pos += delta
def get_node(self, pos):
element = self.get_element(pos)
if element is None:
return None
return element.obj
def get_element(self, pos):
try:
pre = next(self._posmap.irange(maximum=pos, reverse=True))
except StopIteration:
return None
element = self._posmap[pre]
if pos in element:
return element
return None
def test_irange():
mapping = [(val, pos) for pos, val in enumerate(string.ascii_lowercase)]
temp = SortedDict(7, mapping)
for start in range(26):
for stop in range(start + 1, 26):
result = list(string.ascii_lowercase[start:stop])
assert list(temp.irange(result[0], result[-1])) == result
def test_irange():
mapping = [(val, pos) for pos, val in enumerate(string.ascii_lowercase)]
temp = SortedDict(7, mapping)
for start in range(26):
for stop in range(start + 1, 26):
result = list(string.ascii_lowercase[start:stop])
assert list(temp.irange(result[0], result[-1])) == result
def print_events_between_dates(events: SortedDict, start_date: datetime,
end_date: datetime):
date_keys_between_dates = events.irange(minimum=start_date,
maximum=end_date,
inclusive=(True, True))
events = [(event, date_key) for date_key in date_keys_between_dates
for event in events[date_key]]
for event, date in events:
print("{event} | {date}".format(event=event, date=date))
def simulate(arrival_rate,exercise_duration,exercise_number,alpha,graph,start_node,duration,mean_fitness,std_fitness):
arrival_duration = 1/arrival_rate
geom_p = 1/(exercise_number+1)
queue = 0
available_benches = set([node for node in graph.nodes if node[0]=='B'])
occupied_benches = dict([])
available_weights = SortedDict(zip(map(lambda x:x*5,range(len(graph.nodes))),filter(lambda x:x[0]=='S',graph.nodes)))
events=[]
time=np.random.exponential(arrival_duration)
event='arrival'
requests=set([])
schedule_items=0
cur_node = start_node
shortest_paths = nx.all_pairs_dijkstra(graph)
while time < duration:
event_type=event[0]
if event_type == 'arrival':
if len(available_benches) == 0:
queue+=1
else:
fitness=np.random.normal(mean_fitness,std_fitness)
bench=available_benches.pop()
occupied_benches[bench]=fitness
max_weight=np.random.uniform(high=2.0)*fitness
weight=next(available_weights.irange(max_weight,reverse=true)))
requests.add(Request(time,'delivery',weight,available_weights.pop(weight),bench))
next_arrival=np.random.exponential(arrival_duration)
elif event_type == 'departure':
if queue > 0 :
queue-=1
event=('arrival')
break
elif event_type == 'delivery_request':
bench=event[1]
fitness=occupied_benches[bench]
max_weight=np.random.uniform(high=2.0)*fitness
weight=next(available_weights.irange(max_weight,reverse=true)))
requests.add(Request(time,'delivery',weight,available_weights.pop(weight),bench))
if schedule_items==0:
r.solve_schedule(
elif event_type == 'delivery_completed':
(time,event)=events.pop()
class DependencyViewer(BasePlugin):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.transitions: Set[Tuple[int, int]] = set()
self.covered_blocks = SortedDict()
self.sink_color = Qt.yellow
def color_insn(self, addr, selected) -> Optional[QColor]:
if not selected:
try:
block_addr = next(
self.covered_blocks.irange(maximum=addr, reverse=True))
except StopIteration:
return None
block_size = self.covered_blocks[block_addr]
if block_addr <= addr < block_addr + block_size:
return QColor(0xa5, 0xd0, 0xf3)
return None
FUNC_COLUMNS = ('Vuln Sink', )
def extract_func_column(self, func, idx):
assert idx == 0
func_name = func.demangled_name
if "<" in func_name or "{" in func_name:
func_name = normalize_cpp_function_name(func_name)
if "(" in func_name:
# only take function name
func_name = func_name[:func_name.index("(")]
vulntype_and_sinks = sink_manager.get_function_sinks(func_name)
if not vulntype_and_sinks:
return 0, ''
vuln_type, sink = vulntype_and_sinks[0]
return 1, VulnerabilityType.to_string(vuln_type)
def color_func(self, func) -> Optional[QColor]:
# test if we have a match
match = sink_manager.has_function_sink(func.demangled_name)
if match:
return self.sink_color
return None
class ContactList:
def __init__(self):
self.contact_map = SortedDict()
self.name_map = SortedDict()
self.phone_map = SortedDict()
self.email_map = SortedDict()
self.size = 0
self.counter = 1
def add_contact(self, name='', phone_number='', email=''):
self.contact_map[self.counter] = Contact(name, phone_number, email)
self.name_map[name] = self.counter
self.phone_map[phone_number] = self.counter
self.email_map[email] = self.counter
self.counter += 1
self.size += 1
def get_by_name(self, name):
n_id = self.name_map[name]
return self.contact_map[n_id]
def get_by_phone(self, phone):
p_id = self.phone_map[phone]
return self.contact_map[p_id]
def get_by_email(self, email):
e_id = self.email_map[email]
return self.contact_map[e_id]
def get_by_id(self, c_id):
return self.contact_map[c_id]
def remove(self, c_id):
contact = self.contact_map[c_id]
del self.name_map[contact.name]
del self.phone_map[contact.phone_number]
del self.email_map[contact.email]
del self.contact_map[c_id]
self.size -= 1
def get_contacts_ordered_by_name(self):
name_list = self.name_map.irange('ga', 'hzzz')
ordered_name_list = []
for name in name_list:
ordered_name_list.append(self.contact_map[self.name_map[name]])
return ordered_name_list
class LogSystem:
def __init__(self):
self.tweets = SortedDict()
self.granularity = {
"Year": 0,
"Month": 1,
"Day": 2,
"Hour": 3,
"Minute": 4,
"Second": 5,
}
self.min = ["2000", "01", "01", "00", "00", "00"]
self.max = ["2017", "12", "31", "23", "59", "59"]
def put(self, id: int, timestamp: str) -> None:
# Time Complexity: O(log N)
self.tweets[timestamp] = id
def retrieve(self, s: str, e: str, gra: str) -> List[int]:
# Time Complexity: O(num_answer + log N)
ret = []
gran_s = self.granularize(s, gra, True)
gran_e = self.granularize(e, gra, False)
for timestamp in self.tweets.irange(gran_s, gran_e):
ret.append(self.tweets[timestamp])
return ret
def granularize(self, timestamp: str, gra: str, start: bool) -> int:
last_kept = self.granularity[gra]
components = timestamp.split(":")[:last_kept + 1]
if start:
components += self.min[last_kept + 1:]
else:
components += self.max[last_kept + 1:]
return ":".join(components)
class QFeatureMap(QWidget):
"""
Byte-level map of the memory space.
"""
def __init__(self, disasm_view, parent=None):
super().__init__(parent)
self.disasm_view = disasm_view
self.workspace = disasm_view.workspace
self.instance = self.workspace.instance
self.orientation = Orientation.Vertical
# widgets
self.view = None # type: QFeatureMapView
# items
self._insn_indicators = []
# data instance
self.addr = ObjectContainer(
None, name='The current address of the Feature Map.')
# cached values
self._addr_to_region = SortedDict()
self._regionaddr_to_offset = SortedDict()
self._offset_to_regionaddr = SortedDict()
self._total_size = None
self._regions_painted = False
self._init_widgets()
self._register_events()
def sizeHint(self):
return QSize(25, 25)
#
# Public methods
#
def refresh(self):
if self.view is None:
return
if not self._regions_painted:
self._regions_painted = True
self._paint_regions()
def select_offset(self, offset):
addr = self._get_addr_from_pos(offset)
if addr is None:
return
self.addr.am_obj = addr
self.addr.am_event()
#
# Private methods
#
def _init_widgets(self):
self.view = QFeatureMapView(self)
layout = QHBoxLayout()
layout.addWidget(self.view)
layout.setContentsMargins(0, 0, 0, 0)
self.setLayout(layout)
def _register_events(self):
self.disasm_view.infodock.selected_insns.am_subscribe(
self._paint_insn_indicators)
def _paint_regions(self):
cfb = self.instance.cfb_container.am_obj
if cfb is None:
return
# colors
func_color = Conf.feature_map_color_regular_function
data_color = Conf.feature_map_color_data
unknown_color = Conf.feature_map_color_unknown
delimiter_color = Conf.feature_map_color_delimiter
if self._total_size is None:
# calculate the total number of bytes
b = 0
self._addr_to_region.clear()
self._regionaddr_to_offset.clear()
for mr in cfb.regions:
self._addr_to_region[mr.addr] = mr
self._regionaddr_to_offset[mr.addr] = b
self._offset_to_regionaddr[b] = mr.addr
b += self._adjust_region_size(mr)
self._total_size = b
# iterate through all items and draw the image
offset = 0
total_width = self.width()
current_region = None
height = self.height()
print(total_width)
for addr, obj in cfb.ceiling_items():
# are we in a new region?
new_region = False
if current_region is None or not (
current_region.addr <= addr <
current_region.addr + current_region.size):
current_region_addr = next(
self._addr_to_region.irange(maximum=addr, reverse=True))
current_region = self._addr_to_region[current_region_addr]
new_region = True
# adjust size
adjusted_region_size = self._adjust_region_size(current_region)
adjusted_size = min(
obj.size, current_region.addr + adjusted_region_size - addr)
if adjusted_size <= 0:
continue
pos = offset * total_width // self._total_size
length = adjusted_size * total_width // self._total_size
offset += adjusted_size
# draw a rectangle
if isinstance(obj, Unknown):
pen = QPen(data_color)
brush = QBrush(data_color)
elif isinstance(obj, Block):
# TODO: Check if it belongs to a function or not
pen = QPen(func_color)
brush = QBrush(func_color)
else:
pen = QPen(unknown_color)
brush = QBrush(unknown_color)
rect = QRectF(pos, 0, length, height)
self.view._scene.addRect(rect, pen, brush)
# if at the beginning of a new region, draw a line
if new_region:
pen = QPen(delimiter_color)
self.view._scene.addLine(pos, 0, pos, height, pen)
def _adjust_region_size(self, memory_region):
if isinstance(memory_region.object,
(cle.ExternObject, cle.TLSObject, cle.KernelObject)):
# Draw unnecessary objects smaller
return 80
else:
print(memory_region.size, memory_region.object)
return memory_region.size
def _get_pos_from_addr(self, addr):
# find the region it belongs to
try:
mr_base = next(
self._addr_to_region.irange(maximum=addr, reverse=True))
except StopIteration:
return None
# get the base offset of that region
base_offset = self._regionaddr_to_offset[mr_base]
offset = base_offset + addr - mr_base
return offset * self.width() // self._total_size
def _get_addr_from_pos(self, pos):
offset = int(pos * self._total_size // self.width())
try:
base_offset = next(
self._offset_to_regionaddr.irange(maximum=offset,
reverse=True))
except StopIteration:
return None
region_addr = self._offset_to_regionaddr[base_offset]
return region_addr + offset - base_offset
def _paint_insn_indicators(self):
scene = self.view.scene() # type: QGraphicsScene
for item in self._insn_indicators:
scene.removeItem(item)
self._insn_indicators.clear()
for selected_insn_addr in self.disasm_view.infodock.selected_insns:
pos = self._get_pos_from_addr(selected_insn_addr)
if pos is None:
continue
pos -= 1 # this is the top-left x coordinate of our arrow body (the rectangle)
pen = QPen(Qt.yellow)
brush = QBrush(Qt.yellow)
rect = QRectF(pos, 0, 2, 5)
# rectangle
item = scene.addRect(rect, pen, brush)
self._insn_indicators.append(item)
# triangle
triangle = QPolygonF()
triangle.append(QPointF(pos - 1, 5))
triangle.append(QPointF(pos + 3, 5))
triangle.append(QPointF(pos + 1, 7))
triangle.append(QPointF(pos - 1, 5))
item = scene.addPolygon(triangle, pen, brush)
self._insn_indicators.append(item)
class State:
"""
The state.
:ivar str user: Name of the user.
:ivar int version: Version of the state, starting from 0.
"""
def __init__(self, user, version=None, client=None):
# metadata info
self.user = user # type: str
self.version = version if version is not None else 0 # type: int
self.last_push_func = -1
self.last_push_time = -1
# the client
self.client = client # type: Optional[Client]
# dirty bit
self._dirty = True # type: bool
# data
self.functions = {} # type: Dict[int,Function]
self.comments = SortedDict() # type: Dict[int,str]
self.stack_variables = defaultdict(
dict) # type: Dict[int,Dict[int,StackVariable]]
self.patches = SortedDict()
@property
def dirty(self):
return self._dirty
def ensure_dir_exists(self, dir_name):
if not os.path.exists(dir_name):
os.mkdir(dir_name)
if not os.path.isdir(dir_name):
raise RuntimeError(
"Cannot create directory %s. Maybe it conflicts with an existing file?"
% dir_name)
def dump_metadata(self, index, last_push_time: int, last_push_func: int):
d = {
"user": self.user,
"version": self.version,
"last_push_func": last_push_func,
"last_push_time": last_push_time
}
add_data(index, 'metadata.toml', toml.dumps(d).encode())
def dump(self, index: git.IndexFile, last_push_func: int,
last_push_time: int):
# dump metadata
self.dump_metadata(index, last_push_time, last_push_func)
# dump function
add_data(index, 'functions.toml',
toml.dumps(Function.dump_many(self.functions)).encode())
# dump comments
add_data(index, 'comments.toml',
toml.dumps(Comment.dump_many(self.comments)).encode())
# dump patches
add_data(index, 'patches.toml',
toml.dumps(Patch.dump_many(self.patches)).encode())
# dump stack variables, one file per function
for func_addr, stack_vars in self.stack_variables.items():
path = os.path.join('stack_vars', "%08x.toml" % func_addr)
add_data(index, path,
toml.dumps(StackVariable.dump_many(stack_vars)).encode())
@staticmethod
def load_metadata(tree):
return toml.loads(tree['metadata.toml'].data_stream.read().decode())
@classmethod
def parse(cls, tree, version=None, client=None):
s = cls(None, client=client)
# load metadata
try:
metadata = cls.load_metadata(tree)
except:
# metadata is not found
raise MetadataNotFoundError()
s.user = metadata["user"]
s.version = version if version is not None else metadata["version"]
# load function
try:
funcs_toml = toml.loads(
tree['functions.toml'].data_stream.read().decode())
except:
pass
else:
functions = {}
for func in Function.load_many(funcs_toml):
functions[func.addr] = func
s.functions = functions
# load comments
try:
comments_toml = toml.loads(
tree['comments.toml'].data_stream.read().decode())
except:
pass
else:
comments = {}
for comm in Comment.load_many(comments_toml):
comments[comm.addr] = comm.comment
s.comments = SortedDict(comments)
# load patches
try:
patches_toml = toml.loads(
tree['patches.toml'].data_stream.read().decode())
except:
pass
else:
patches = {}
for patch in Patch.load_many(patches_toml):
patches[patch.offset] = patch
s.patches = SortedDict(patches)
# load stack variables
for func_addr in s.functions:
try:
# TODO use unrebased address for more durability
svs_toml = toml.loads(tree[os.path.join(
'stack_vars',
'%08x.toml' % func_addr)].data_stream.read().decode())
except:
pass
else:
svs = list(StackVariable.load_many(svs_toml))
d = dict((v.stack_offset, v) for v in svs)
if svs:
s.stack_variables[svs[0].func_addr] = d
# clear the dirty bit
s._dirty = False
return s
def copy_state(self, target_state=None):
if target_state == None:
print("Cannot copy an empty state (state == None)")
return
self.functions = target_state.functions.copy()
self.comments = target_state.comments.copy()
self.stack_variables = target_state.stack_variables.copy()
self.patches = target_state.patches.copy()
def save(self):
if self.client is None:
raise RuntimeError("save(): State.client is None.")
self.client.save_state(self)
#
# Pushers
#
@dirty_checker
def update_metadata(self, last_func_push: str, last_push_time: int):
pass
@dirty_checker
def set_function(self, func):
if not isinstance(func, Function):
raise TypeError("Unsupported type %s. Expecting type %s." %
(type(func), Function))
if func.addr in self.functions and self.functions[func.addr] == func:
# no update is required
return False
self.functions[func.addr] = func
return True
@dirty_checker
def set_comment(self, addr, comment):
if addr in self.comments and self.comments[addr] == comment:
# no update is required
return False
self.comments[addr] = comment
return True
@dirty_checker
def remove_comment(self, addr):
if addr in self.comments:
del self.comments[addr]
return True
return False
@dirty_checker
def set_patch(self, addr, patch):
if addr in self.patches and self.patches[addr] == patch:
# no update is required
return False
self.patches[addr] = patch
return True
@dirty_checker
def set_stack_variable(self, func_addr, offset, variable):
if func_addr in self.stack_variables \
and offset in self.stack_variables[func_addr] \
and self.stack_variables[func_addr][offset] == variable:
# no update is required
return False
self.stack_variables[func_addr][offset] = variable
return True
#
# Pullers
#
def get_function(self, addr):
if addr not in self.functions:
raise KeyError("Function %x is not found in the db." % addr)
return self.functions[addr]
def get_comment(self, addr):
if addr not in self.comments:
raise KeyError("There is no comment at address %#x." % addr)
cmt = self.comments[addr]
if is_py2() and isinstance(cmt, unicode):
cmt = str(cmt)
return cmt
def get_comments(self, start_addr, end_addr=None):
for k in self.comments.irange(start_addr, reverse=False):
if k >= end_addr:
break
cmt = self.comments[k]
if is_py2() and isinstance(cmt, unicode):
cmt = str(cmt)
yield cmt
def get_patch(self, addr):
if addr not in self.patches:
raise KeyError("There is no patch at address %#x." % addr)
return self.patches[addr]
def get_patches(self):
return self.patches.values()
def get_stack_variable(self, func_addr, offset):
if func_addr not in self.stack_variables:
raise KeyError("No stack variables are defined for function %#x." %
func_addr)
if offset not in self.stack_variables[func_addr]:
raise KeyError(
"No stack variable exists at offset %d in function %#x." %
(offset, func_addr))
return self.stack_variables[func_addr][offset]
def get_stack_variables(self, func_addr):
if func_addr not in self.stack_variables:
raise KeyError("No stack variables are defined for function %#x." %
func_addr)
return self.stack_variables[func_addr].items()
# TODO: it would be better if we stored the function addr with every state object, like comments
def get_modified_addrs(self):
"""
Gets ever address that has been touched in the current state.
Returns a set of those addresses.
@rtype: Set(int)
"""
moded_addrs = set()
# ==== functions ==== #
for addr in self.functions:
moded_addrs.add(addr)
# ==== comments ==== #
for addr in self.comments:
moded_addrs.add(addr)
# ==== stack vars ==== #
for addr in self.stack_variables:
moded_addrs.add(addr)
return moded_addrs
class VectorNav(object):
TIMEOUT = 0.5 # Serial read timeout, in seconds
MODE_SYNC = 0
MODE_ASYNC = 1
CHECKSUM_8BIT = 0
CHECKSUM_16BIT = 1
INITIAL_CRC = 0
REG_USER_TAG = 0
REG_MODEL_NUM = 1
REG_HARDWARE_REV = 2
REG_SERIAL_NUM = 3
REG_FIRMWARE_VER = 4
REG_SERIAL_BAUD = 5
REG_ASYNC_DOUT_TYPE = 6
REG_ASYNC_DOUT_FREQ = 7
REG_PROTO_CTRL = 30
REG_SYNC_CTRL = 32
REG_SYNC_STATUS = 33
REG_BIN_OUT_1 = 75
REG_BIN_OUT_2 = 76
REG_BIN_OUT_3 = 77
FIX_NONE = 0
FIX_TIME = 1
FIX_2D = 2
FIX_3D = 3
SRC_VECTORNAV = 'VectorNav'
DEFAULT_BAUD = 115200
BINARY_OUTPUT_RATE_DIV = 32 # 800Hz/32 = 25Hz
STATE_TIMEOUT = timedelta(seconds=60)
DEFAULT_VID = 0x0403
DEFAULT_PID = 0x6001
FIELDS = {
0: {
0: ('TimeStartup', '<Q'),
1: ('TimeGps', '<Q'),
2: ('TimeSyncIn', '<Q'),
3: ('Ypr', '<3f'),
4: ('Qtn', '<4f'),
5: ('AngularRate', '<3f'),
6: ('PosLla', '<3d'),
7: ('VelNed', '<3f'),
8: ('Accel', '<3f'),
9: ('Imu', '<3f3f'),
10: ('MagPres', '<5f'),
11: ('DeltaThetaVel', '<7f'),
12: ('InsStatus', '<H'),
13: ('SyncInCnt', '<L'),
14: ('TimeGpsPps', '<Q')
},
1: {
0: ('TimeStartup', '<Q'),
1: ('TimeGps', '<Q'),
2: ('GpsTow', '<Q'),
3: ('GpsWeek', '<H'),
4: ('TimeSyncIn', '<Q'),
5: ('TimeGpsPps', '<Q'),
6: ('TimeUTC', '<bBBBBBH'),
7: ('SyncInCnt', '<L')
},
2: {
0: ('ImuStatus', '<H'),
1: ('UncompMag', '<3f'),
2: ('UncompAccel', '<3f'),
3: ('UncompGyro', '<3f'),
4: ('Temp', '<f'),
5: ('Pres', '<f'),
6: ('DeltaTheta', '<4f'),
7: ('DeltaV', '<3f'),
8: ('Mag', '<3f'),
9: ('Accel', '<3f'),
10: ('AngularRate', '<3f'),
11: ('SensSat', '<H')
},
3: {
0: ('TimeUTC', '<bBBBBBH'),
1: ('GpsTow', '<Q'),
2: ('GpsWeek', '<H'),
3: ('NumSats', '<B'),
4: ('Fix', '<B'),
5: ('GpsPosLla', '<3d'),
6: ('GpsPosEcef', '<3d'),
7: ('GpsVelNed', '<3f'),
8: ('GpsVelEcef', '<3f'),
9: ('GpsPosU', '<3f'),
10: ('GpsVelU', '<f'),
11: ('TimeU', '<L')
},
4: {
0: ('VpeStatus', '<H'),
1: ('Ypr', '<3f'),
2: ('Qtn', '<4f'),
3: ('DCM', '<9f'),
4: ('MagNed', '<3f'),
5: ('AccelNed', '<3f'),
6: ('LinearAccelBody', '<3f'),
7: ('LinearAccelNed', '<3f'),
8: ('YprU', '<3f')
},
5: {
0: ('InsStatus', '<H'),
1: ('PosLla', '<3d'),
2: ('PosEcef', '<3d'),
3: ('VelBody', '<3f'),
4: ('VelNed', '<3f'),
5: ('VelEcef', '<3f'),
6: ('MagEcef', '<3f'),
7: ('AccelEcef', '<3f'),
8: ('LinearAccelEcef', '<3f'),
9: ('PosU', '<f'),
10: ('VelU', '<f')
}
}
def __init__(self, port, baud):
self._port = port
self._baud = baud
self._serial = None
self._csum_mode = self.CHECKSUM_8BIT
self._state = {}
self._tstate = SortedDict()
self._proc = None
self._mode = self.MODE_SYNC
self._offsetYPR = None
self._autocal = True
m = datetime.now()
self._tstate[m] = {}
d = self._tstate[m]
for g in self.FIELDS:
for f in self.FIELDS[g]:
self._state[self.FIELDS[g][f][0]] = None
d[self.FIELDS[g][f][0]] = None
@staticmethod
def create():
port = findSerialDevice(VectorNav.DEFAULT_VID, VectorNav.DEFAULT_PID)
if port is None:
raise Exception('VectorNav FTDI converter not detected')
return VectorNav(port, VectorNav.DEFAULT_BAUD)
def calibrate(self):
view = self.getView()
self._offsetYPR = view.YPR
def open(self, mode):
self.close()
if mode != self.MODE_SYNC and mode != self.MODE_ASYNC:
return
self._mode = mode
if mode == self.MODE_ASYNC:
self._queueState = mp.Queue()
self._queueSync = mp.Queue()
self._proc = mp.Process(target=self._processFunc)
self._proc.start()
else:
self._queueState = None
self._queueSync = None
self._proc = None
self._openPort()
def close(self):
if self._mode == self.MODE_ASYNC:
if self._proc is None:
return
self._queueSync.put(None)
self._syncState()
self._proc.join()
self._proc = None
else:
if self._serial is None:
return
self._closePort()
self._queueState = None
self._queueSync = None
self._mode = self.MODE_SYNC
def _openPort(self):
self._serial = serial.Serial(self._port,
self._baud,
timeout=self.TIMEOUT)
self._configBinaryOutput(0, 'TimeUTC', 'Ypr', 'PosLla', 'VelNed',
'TimeUTC', 'NumSats', 'Fix', 'PosU', 'VelU',
'TimeU')
def _closePort(self):
self._serial.close()
self._serial = None
def _processFunc(self):
self._openPort()
while True:
try:
try:
self._queueSync.get(False)
break
except Queue.Empty:
pass
self._recvResponse()
except KeyboardInterrupt: # Allow Ctrl+C without raising an exception
break
except: # Allow other exceptions
pass
self._closePort()
def _send(self, data):
self._serial.write(data)
def _recv(self, size):
return self._serial.read(size)
def _recvLine(self):
return self._serial.readline()
@staticmethod
def _calcChecksum8(cmd):
csum = 0
for c in cmd:
csum ^= ord(c)
return '%02X' % (csum & 0xFF)
@staticmethod
def _appendCRC(crc, data):
for c in data:
crc = ((crc >> 8) & 0xFF) | (crc << 8)
crc ^= ord(c) & 0xFF
crc ^= (crc & 0xFF) >> 4
crc ^= crc << 12
crc ^= (crc & 0xFF) << 5
return crc & 0xFFFF
@staticmethod
def _calcChecksum16(cmd):
return '%04X' % VectorNav._appendCRC(VectorNav.INITIAL_CRC, cmd)
def _calcChecksum(self, cmd):
if self._csum_mode == self.CHECKSUM_16BIT:
return VectorNav._calcChecksum16(cmd)
return VectorNav._calcChecksum8(cmd)
def _sendCommand(self, id, params):
cmd = 'VN' + id
if len(params) > 0:
cmd += ',' + ','.join(str(p) for p in params)
cmd = '$' + cmd + '*' + self._calcChecksum(cmd) + '\r\n'
self._send(cmd)
def process(self, flush=False):
if self._mode == self.MODE_SYNC:
if flush:
self._serial.flushInput() # Flush the input buffer
self._waitResponse('$B') # Wait for a binary response
def _waitResponse(self, id, process_func=None):
while True:
r = self._recvResponse()
if r:
if id is not None and r[0] == id:
return process_func(
r) if process_func is not None else True
elif r[0] == 'ERR':
return None
return True
def _processTextResponse(self, id, params):
if id == 'GGA':
pass
def _recvResponse(self):
sync = self._recv(1)
if len(sync) < 1:
return None
if sync[0] == '$':
line = str(self._recvLine())
if line:
return self._parseTextResponse(line)
elif sync[0] == '\xFA':
return self._parseBinaryResponse()
return None
def _parseTextResponse(self, line):
if len(line) < 5:
return None
hdr = line[0:2]
id = line[2:5]
if hdr != 'VN':
return None
term = line.rfind('*')
if term < 5:
return None
# TODO: Verify checksum
params = line[5:term].split(',')
self._processTextResponse(id, params)
return (id, params)
def _parseBinaryResponse(self):
groups = self._recv(1)
if len(groups) < 0:
return None
now = datetime.now()
crc = VectorNav._appendCRC(self.INITIAL_CRC, groups)
groups, = unpack('<B', groups)
fields = []
for g in range(0, 6):
if (groups & (1 << g)) != 0:
group_fields = self._recv(2)
if len(group_fields) < 2:
return None
crc = VectorNav._appendCRC(crc, group_fields)
group_fields, = unpack('<H', group_fields)
for f in range(0, 16):
if (group_fields & (1 << f)) != 0:
fields.append((g, f))
state = {}
for f in fields:
if not f[0] in self.FIELDS:
continue
group_fields = self.FIELDS[f[0]]
if not f[1] in group_fields:
continue
field_info = group_fields[f[1]]
size = calcsize(field_info[1])
data = self._recv(size)
if len(data) < size:
return None
crc = VectorNav._appendCRC(crc, data)
state[field_info[0]] = unpack(field_info[1], data)
# Receive the CRC
ref_crc = self._recv(2)
crc = VectorNav._appendCRC(crc, ref_crc)
if crc != 0:
return None # Bad CRC
for st in state:
self._setState(st, state[st], now)
return ('$B', None)
def _setState(self, id, value, timestamp):
state = (id, value, datetime.now())
if self._mode == self.MODE_ASYNC:
self._queueState.put(state)
else:
self._updateState(state)
def _syncState(self):
if self._mode != self.MODE_ASYNC:
return
# Add new entries
try:
while True:
self._updateState(self._queueState.get(False))
except Queue.Empty:
pass
# Remove old entries
keys = list(
self._tstate.irange(minimum=None,
maximum=datetime.now() - self.STATE_TIMEOUT,
inclusive=(True, False)))
for k in keys:
try:
del self._tstate[k]
except:
pass
def _updateState(self, state):
self._state[state[0]] = state[1]
d = self._tstate.setdefault(state[2], {})
d[state[0]] = state[1]
def _registerRead(self, reg_id):
self._sendCommand('RRG', [reg_id])
return self._waitResponse('RRG', lambda r: r[1])
def _registerWrite(self, reg_id, *args):
params = [reg_id]
params.extend(list(args))
self._sendCommand('WRG', params)
return self._waitResponse('WRG', lambda r: r[1])
def _findField(self, id):
for g in self.FIELDS:
for f in self.FIELDS[g]:
if self.FIELDS[g][f][0] == id:
return (g, f)
return None
def _configBinaryOutput(self, index, *fields):
if index < 0 or index > 2:
return
groups = 0
group_fields = [0, 0, 0, 0, 0, 0]
for f in fields:
field = self._findField(f)
if field is None:
continue
groups |= 1 << field[0]
group_fields[field[0]] |= 1 << field[1]
args = [1, self.BINARY_OUTPUT_RATE_DIV, '%02X' % groups]
args.extend('%04X' % x for x in filter(None, group_fields))
args = tuple(args)
self._registerWrite(self.REG_BIN_OUT_1 + index, *args)
def getState(self, id, maxTime=None):
self._syncState()
keys = self._tstate.irange(minimum=None,
maximum=maxTime,
inclusive=(True, True),
reverse=True)
for k in keys:
if id in self._tstate[k]:
value = self._tstate[k][id]
if id == 'Ypr' and value is not None:
if self._offsetYPR is not None:
value = (value[0] - self._offsetYPR[0],
value[1] - self._offsetYPR[1],
value[2] - self._offsetYPR[2])
elif self._autocal:
self._offsetYPR = value
value = (0, 0, 0)
if value is not None and len(value) == 1:
value = value[0]
return value
return None
def getView(self, timestamp=None):
if len(self._tstate) == 0:
return None
try:
if timestamp is None:
maxTime = self._tstate.iloc[-1]
else:
maxTime = next(
self._tstate.irange(minimum=None,
maximum=timestamp,
inclusive=(True, True),
reverse=True))
return VectorNav.View(self, maxTime)
except:
#log.log('Error getting VectorNav view for timestamp {time}'.format(time=timestamp))
print 'Error getting VectorNav view for timestamp {time}'.format(
time=timestamp)
return None
class View(object):
def __init__(self, vn, maxTime):
self._vn = vn
self._maxTime = maxTime
def getState(self, id):
return self._vn.getState(id, self._maxTime)
@property
def timeUTC(self):
t = self.getState('TimeUTC')
time = t[0]
if time is None:
return None
return (datetime(2000 + time[0], time[1] + 1, time[2] + 1, time[3],
time[4], time[5], time[6] * 1000), t[1])
@property
def position(self):
"""Get the latest position as (latitude, longitude, altitude)"""
return self.getState('PosLla')
@property
def velocity(self):
"""Get the latest velocity in NED frame in m/s"""
return self.getState('VelNed')
@property
def YPR(self):
"""Get the latest yaw/pitch/roll values in degrees"""
return self.getState('Ypr')
@property
def numSats(self):
"""Get the latest number of visible satellites"""
return self.getState('NumSats')
@property
def fix(self):
"""Get the latest GPS fix type"""
return self.getState('Fix')
@property
def dict(self):
"""Get the data dictionary"""
d = {
'yaw': 0,
'pitch': 0,
'roll': 0,
'src_att': None,
'hdg': 0,
'src_hdg': None,
'lat': 0,
'lon': 0,
'relative_alt': 0,
'src_gps': None,
'cog': 0,
'src_cog': None
}
ypr = self.YPR
pos = self.position
vel = self.velocity
if ypr is not None:
d['yaw'] = math.radians(ypr[0])
d['pitch'] = math.radians(ypr[1])
d['roll'] = math.radians(ypr[2])
d['hdg'] = ypr[0] * 100
d['src_att'] = VectorNav.SRC_VECTORNAV
d['src_hdg'] = VectorNav.SRC_VECTORNAV
if pos is not None and abs(pos[0]) > 1.0 and abs(pos[1]) > 1.0:
d['lat'] = pos[0] * 1e7
d['lon'] = pos[1] * 1e7
d['relative_alt'] = pos[2] * 1e3
d['src_gps'] = VectorNav.SRC_VECTORNAV
d['fd_timestamp'] = self._maxTime.strftime(TIMESTAMP_SIGNATURE)
#d['cog'] = atan2(vel[1], vel[0])
d['src'] = VectorNav.SRC_VECTORNAV
return d
@property
def json(self):
"""Get the json-encoded data"""
return json.dumps(self.dict)
class TimeSeries(TictsMagicMixin, TictsOperationMixin, PandasMixin,
TictsIOMixin, TictsPlot):
""" TimeSeries object.
Args:
default: The default value of timeseries.
permissive (bool): Whether to allow accessing non-existing values or not.
If is True, getting non existing item returns None.
If is False, getting non existing item raises.
"""
_default_interpolate = "previous"
_meta_keys = ('default', 'name', 'permissive')
@property
def index(self):
return self.data.keys()
@property
def lower_bound(self):
"""Return the lower bound time index."""
if self.empty:
return MINTS
return self.index[0]
@property
def upper_bound(self):
"""Return the upper bound time index."""
if self.empty:
return MAXTS
return self.index[-1]
@property
def _has_default(self):
return self.default != NO_DEFAULT
@property
def _kwargs_special_keys(self):
kwargs = {}
for attr_name in self._meta_keys:
kwargs[attr_name] = getattr(self, attr_name)
return kwargs
@property
def empty(self):
"""Return whether the TimeSeries is empty or not."""
return len(self) == 0
def __init__(self,
data=None,
default=NO_DEFAULT,
name=DEFAULT_NAME,
permissive=True,
tz='UTC'):
""""""
if isinstance(data, self.__class__):
for attr in ('data', *self._meta_keys):
setattr(self, attr, getattr(data, attr))
# Only set 'default' and 'name' if is different from default
if default != NO_DEFAULT:
setattr(self, 'default', default)
if name != DEFAULT_NAME:
setattr(self, 'name', name)
return
if hasattr(default, 'lower') and default.lower() == 'no_default':
# 'no_default' as string is used at JSON serealization time
self.default = NO_DEFAULT
else:
self.default = default
self.name = name
self.permissive = permissive
# Overwrite the name if data is an instance of pd.DataFrame or pd.Series
if isinstance(data, pd.DataFrame):
if len(data.columns) != 1:
msg = ("Can't convert a DataFrame with several columns into "
"one timeseries: {}.")
raise ValueError(msg.format(data.columns))
self.name = data.columns[0]
elif isinstance(data, pd.Series):
self.name = data.name
try:
tz = pytz.timezone(tz)
except pytz.UnknownTimeZoneError:
raise ValueError('{} is not a valid timezone'.format(tz))
# SortedDict.__init__ does not use the __setitem__
# Hence we got to parse datetime keys ourselves.
# SortedDict use the first arg given and check if is a callable
# in case you want to give your custom sorting function.
self.data = SortedDict(None, _process_args(data, tz))
def __setitem__(self, key, value):
if isinstance(key, slice):
return self.set_interval(key.start, key.stop, value)
if key in self._meta_keys:
super().__setitem__(key, value)
else:
key = timestamp_converter(key, self.tz)
self.data[key] = value
def __getitem__(self, key):
"""Get the value of the time series, even in-between measured values by interpolation.
Args:
key (datetime): datetime index
interpolate (str): interpolate operator among ["previous", "linear"]
"""
interpolate = self._default_interpolate
if isinstance(key, tuple):
if len(key) == 2:
key, interpolate = key
elif len(key) > 2:
raise KeyError
if isinstance(key, slice):
return self.slice(key.start, key.stop)
key = timestamp_converter(key, self.tz)
basemsg = "Getting {} but default attribute is not set".format(key)
if self.empty:
if self._has_default:
return self.default
else:
if self.permissive:
return
else:
raise KeyError(
"{} and timeseries is empty".format(basemsg))
if key < self.lower_bound:
if self._has_default:
return self.default
else:
if self.permissive:
return
else:
msg = "{}, can't deduce value before the oldest measurement"
raise KeyError(msg.format(basemsg))
# If the key is already defined:
if key in self.index:
return self.data[key]
if interpolate.lower() == "previous":
fn = self._get_previous
elif interpolate.lower() == "linear":
fn = self._get_linear_interpolate
else:
raise ValueError("'{}' interpolation unknown.".format(interpolate))
return fn(key)
def _get_previous(self, time):
# In this case, bisect_left == bisect_right == bisect
# And idx > 0 as we already handled other cases
previous_idx = self.data.bisect(time) - 1
time_idx = self.index[previous_idx]
return self.data[time_idx]
def _get_linear_interpolate(self, time):
# TODO: put it into a 'get_previous_index' method
idx = self.data.bisect_left(time)
previous_time_idx = self.index[idx - 1]
# TODO: check on left bound case
# out of right bound case:
if idx == len(self):
return self.data[previous_time_idx]
next_time_idx = self.index[idx]
previous_value = self.data[previous_time_idx]
next_value = self.data[next_time_idx]
coeff = (time - previous_time_idx) / (
next_time_idx - previous_time_idx)
value = previous_value + coeff * (next_value - previous_value)
return value
def slice(self, start, end): # noqa A003
"""Slice your timeseries for give interval.
Args:
start (datetime or str): lower bound
end (datetime or str): upper bound
Returns:
TimeSeries sliced
"""
start = timestamp_converter(start, self.tz)
end = timestamp_converter(end, self.tz)
newts = TimeSeries(**self._kwargs_special_keys)
for key in self.data.irange(start, end, inclusive=(True, False)):
newts[key] = self[key]
should_add_left_closure = (start not in newts.index
and start >= self.lower_bound)
if should_add_left_closure:
newts[start] = self[start] # is applying get_previous on self
return newts
def set_interval(self, start, end, value):
"""Set a value for an interval of time.
Args:
start (datetime or str): lower bound
end (datetime or str): upper bound
value: the value to be set
Returns:
self
Raises:
NotImplementedError: when no default is set.
"""
if not self._has_default:
msg = "At the moment, you have to set a default for set_interval"
raise NotImplementedError(msg)
start = timestamp_converter(start, self.tz)
end = timestamp_converter(end, self.tz)
keys = self.data.irange(start, end, inclusive=(True, False))
last_value = self[end]
for key in list(keys):
del self.data[key]
self[start] = value
self[end] = last_value
def compact(self):
"""Convert this instance to a compact version: consecutive measurement of the
same value are discarded.
Returns:
TimeSeries
"""
ts = TimeSeries(**self._kwargs_special_keys)
for time, value in self.items():
should_set_it = ts.empty or (ts[time] != value)
if should_set_it:
ts[time] = value
return ts
def iterintervals(self, end=None):
"""Iterator that contain start, end of intervals.
Args:
end (datetime): right bound of last interval.
"""
lst_keys = SortedList(self.index)
if not end:
end = self.upper_bound
else:
end = timestamp_converter(end, self.tz)
if end not in lst_keys:
lst_keys.add(end)
for i, key in enumerate(lst_keys[:-1]):
next_key = lst_keys[i + 1]
if next_key > end: # stop there
raise StopIteration
yield key, next_key
def equals(self, other, check_default=True, check_name=True):
if not isinstance(other, self.__class__):
raise TypeError("Can't compare {} with {}".format(
self.__class__.__name__, other.__class__.__name__))
is_equal = self.data == other.data
if check_default:
is_equal = is_equal and self.default == other.default
if check_name:
is_equal = is_equal and self.name == other.name
return is_equal
@property
def tz(self):
if self.empty:
return pytz.UTC
return str(self.index[0].tz)
def tz_convert(self, tz):
try:
tz = pytz.timezone(tz)
except pytz.UnknownTimeZoneError:
raise ValueError('{} is not a valid timezone'.format(tz))
ts = deepcopy(self)
for key in ts.index:
ts[key.tz_convert(tz)] = ts.data.pop(key)
return ts
class LogListener(threading.Thread):
lock = threading.Lock()
logfile = open('./test.log', 'w')
def __init__(self, name, default_timeout):
threading.Thread.__init__(self)
self.messages = SortedDict()
self.default_timeout = default_timeout
self.event = threading.Event()
self.is_running = True
self.name = name
self.daemon = True
self.start()
def starts_with_count(self, message):
count = 0
for i in self.messages.irange(minimum=message):
if not i.startswith(message):
break
count += self.messages[i]
return count
def exact_count(self, message):
return self.messages.get(message, 0)
def expect_count(self, message, mincount, timeout=None):
if timeout is None:
timeout = self.default_timeout
now = time.time()
end = now + timeout
curcount = self.exact_count(message)
while (curcount < mincount and now < end):
self.event.wait(end - now)
now = time.time()
curcount = self.exact_count(message)
return curcount
def expect(self, message, expected_count, timeout=None):
count = self.expect_count(message, expected_count, timeout)
if count != expected_count:
print(
'TEST FAILED: message:"{0}", count:{1} (expected {2})'.format(
message, count, expected_count))
traceback.print_stack()
sys.exit()
else:
print('{3}: message:"{0}", count:{1} (expected {2})'.format(
message, count, expected_count, self.name))
def expect_minimum(self, message, mincount, timeout=None):
count = self.expect_count(message, mincount, timeout)
if count < mincount:
print(
'TEST FAILED: message:"{0}" count:{1} (expected at least {2})'.
format(message, count, mincount))
traceback.print_stack()
sys.exit()
else:
print(
'{3}: message:"{0}" count:{1} (expected at least {2})'.format(
message, count, mincount, self.name))
def expect_maximum(self, message, maxcount, timeout=None):
count = self.expect_count(message, maxcount + 1, timeout)
if count > maxcount:
print(
'TEST FAILED: message:"{0}" count:{1} (expected no more than {2})'
.format(message, count, maxcount))
traceback.print_stack()
sys.exit()
else:
print('{3}: message:"{0}" count:{1} (expected no more than {2})'.
format(message, count, maxcount, self.name))
def run(self):
try:
os.mkfifo(self.name)
except OSError as oe:
if oe.errno != errno.EEXIST:
raise
while self.is_running:
with open(self.name) as fifo:
with LogListener.lock:
print("connected to {0}".format(self.name))
while self.is_running:
line = fifo.readline()
if line == '':
break
key = line[:-1]
with LogListener.lock:
LogListener.logfile.write('{0}: {1}: "{2}"\n'.format(
time.strftime('%X %x %Z'), self.name, key))
LogListener.logfile.flush()
self.messages[key] = self.messages.get(key, 0) + 1
self.event.set()
self.event.clear()
def close(self):
self.is_running = False
self.join()
class FreshPondSim:
def __init__(self,
distance,
start_time,
end_time,
entrances,
entrance_weights,
rand_velocities_and_distances_func,
entrance_rate,
entrance_rate_integral=None,
entrance_rate_integral_inverse=None,
interpolate_rate=True,
interpolate_rate_integral=True,
interpolate_res=None,
snap_exit=True):
assert_positive_real(distance, 'distance')
assert_real(start_time, 'start_time')
assert_real(end_time, 'end_time')
if not (start_time < end_time):
raise ValueError(f"start_time should be less than end_time")
assert len(entrances) == len(entrance_weights)
self.start_time = start_time
self.end_time = end_time
self.dist_around = distance
self.entrances = entrances
self.entrance_weights = entrance_weights
self.rand_velocities_and_distances = rand_velocities_and_distances_func
self._snap_exit = snap_exit
if interpolate_rate or interpolate_rate_integral:
if interpolate_res is None:
raise ValueError("Specify interpolate_res for interpolation")
if interpolate_rate:
self.entrance_rate = DynamicBoundedInterpolator(
entrance_rate, start_time, end_time, interpolate_res)
else:
self.entrance_rate = entrance_rate
if interpolate_rate_integral: # Want to interplate the integral function
if entrance_rate_integral is None: # No integral function given
# Do numerical integration and interpolate to speed it up
def integral_func(t):
y, abserr = integrate.quad(entrance_rate, start_time, t)
return y
self.entrance_rate_integral = DynamicBoundedInterpolator(
integral_func, start_time, end_time, interpolate_res)
else: # Integral function was provided
# Use the provided rate integral function but interpolate it
self.entrance_rate_integral = DynamicBoundedInterpolator(
entrance_rate_integral, start_time, end_time, interpolate_res)
else: # Don't want to interpolate the integral function
# If entrance_rate_integral is not None (i.e. is provided) then
# that function will be used as the rate integral.
# If entrance_rate_integral is None, numerical integration will
# be used.
self.entrance_rate_integral = entrance_rate_integral
self.entrance_rate_integral_inverse = entrance_rate_integral_inverse
self.pedestrians = SortedKeyList(key=attrgetter('start_time'))
self._counts = SortedDict()
self._counts[self.start_time] = 0
self._counts_are_correct = True
self.refresh_pedestrians()
def _distance(self, a, b):
"""signed distance of a relative to b"""
return circular_diff(a % self.dist_around, b % self.dist_around,
self.dist_around)
def _distance_from(self, b):
"""returns a function that returns the signed sitance from b"""
return lambda a: self._distance(a, b)
def _abs_distance_from(self, b):
"""returns a function that returns the distance from b"""
return lambda a: abs(self._distance(a, b))
def _closest_exit(self, dist):
"""Returns the closest number to dist that is equivalent mod dist_around
to an element of entrances"""
closest_exit = min(self.entrances, key=self._abs_distance_from(dist))
diff = self._distance(closest_exit, dist)
corrected_dist = dist + diff
return corrected_dist
def refresh_pedestrians(self):
"""Refreshes the pedestrians in the simulation to random ones"""
self.clear_pedestrians()
start_times = list(
random_times(self.start_time, self.end_time,
self.entrance_rate,
self.entrance_rate_integral,
self.entrance_rate_integral_inverse))
n_pedestrians = len(start_times)
entrances = random.choices(population=self.entrances,
weights=self.entrance_weights,
k=n_pedestrians)
velocities, distances = self.rand_velocities_and_distances(
n_pedestrians).T
def pedestrians_generator():
for start_time, entrance, velocity, dist in zip(
start_times, entrances, velocities, distances):
assert dist > 0
if self._snap_exit:
original_exit = entrance + dist * sign(velocity)
corrected_exit = self._closest_exit(original_exit)
corrected_dist = abs(corrected_exit - entrance)
if math.isclose(corrected_dist, 0, abs_tol=1e-10):
corrected_dist = self.dist_around
else:
corrected_dist = dist
yield FreshPondPedestrian(self.dist_around, entrance,
corrected_dist, start_time, velocity)
self.add_pedestrians(pedestrians_generator())
def clear_pedestrians(self):
"""Removes all pedestrains in the simulation"""
self.pedestrians.clear()
self._reset_counts()
self._counts_are_correct = True
def add_pedestrians(self, pedestrians):
"""Adds all the given pedestrians to the simulation"""
def checked_pedestrians():
for p in pedestrians:
self._assert_pedestrian_in_range(p)
yield p
initial_num_pedestrians = self.num_pedestrians()
self.pedestrians.update(checked_pedestrians())
final_num_pedestrians = self.num_pedestrians()
if final_num_pedestrians > initial_num_pedestrians:
self._counts_are_correct = False
else:
assert final_num_pedestrians == initial_num_pedestrians
def _assert_pedestrian_in_range(self, p):
"""Makes sure the pedestrian's start time is in the simulation's
time interval"""
if not (self.start_time <= p.start_time < self.end_time):
raise ValueError(
"Pedestrian start time is not in range [start_time, end_time)")
def add_pedestrian(self, p):
"""Adds a new pedestrian to the simulation"""
self._assert_pedestrian_in_range(p)
self.pedestrians.add(p)
# Update counts only when counts are correct
if self._counts_are_correct:
# add a new breakpoint at the pedestrian's start time if it not there
self._counts[p.start_time] = self.n_people(p.start_time)
# add a new breakpoint at the pedestrian's end time if it not there
self._counts[p.end_time] = self.n_people(p.end_time)
# increment all the counts in the pedestrian's interval of time
# inclusive on the left, exclusive on the right
# If it were inclusive on the right, then the count would be one more
# than it should be in the period after end_time and before the next
# breakpoint after end_time
for t in self._counts.irange(p.start_time,
p.end_time,
inclusive=(True, False)):
self._counts[t] += 1
def _reset_counts(self):
"""Clears _counts and sets count at start_time to 0"""
self._counts.clear()
self._counts[self.start_time] = 0
def _recompute_counts(self):
"""Store how many people there are whenever someone enters or exits so
the number of people at a given time can be found quickly later"""
# print("Recomputing counts")
self._reset_counts()
if self.num_pedestrians() == 0:
return
# pedestrians are already sorted by start time
start_times = [p.start_time for p in self.pedestrians]
end_times = sorted([p.end_time for p in self.pedestrians])
n = len(start_times)
curr_count = 0 # current number of people
start_times_index = 0
end_times_index = 0
starts_done = False # whether all the start times have been added
ends_done = False # whether all the end times have been added
while not (starts_done and ends_done):
# determine whether a start time or an end time should be added next
# store this in the variable take_start which is true if a start
# time should be added next
if starts_done:
# already added all the start times; add an end time
take_start = False
elif ends_done:
# already added all the end times; add a start time
take_start = True
else:
# didn't add all the end times nor all the start times
# add the time that is earliest
next_start_time = start_times[start_times_index]
next_end_time = end_times[end_times_index]
take_start = next_start_time < next_end_time
if take_start:
# add next start
curr_count += 1
start_time = start_times[start_times_index]
self._counts[start_time] = curr_count
start_times_index += 1
if start_times_index == n:
starts_done = True
else:
# add next end
curr_count -= 1
end_time = end_times[end_times_index]
self._counts[end_time] = curr_count
end_times_index += 1
if end_times_index == n:
ends_done = True
def n_unique_people_saw(self, p):
"""Returns the number of unique people that a pedestrian sees"""
n = 0
for q in self.pedestrians:
if p.intersects(q):
n += 1
return n
def n_people_saw(self, p):
"""Returns the number of times a pedestrian sees someone"""
n = 0
for q in self.pedestrians:
if p.end_time > q.start_time and p.start_time < q.end_time:
n += p.n_intersections(q)
return n
def intersection_directions(self, p):
"""Returns the number of people seen going in the same direction and the
number of people seen going in the opposite direction by p as a tuple"""
n_same, n_diff = 0, 0
for q in self.pedestrians:
if p.end_time > q.start_time and p.start_time < q.end_time:
d = q.intersection_direction(p)
if d == 1:
n_same += 1
elif d == -1:
n_diff += 1
return n_same, n_diff
def intersection_directions_total(self, p):
n_same, n_diff = 0, 0
for q in self.pedestrians:
if p.end_time > q.start_time and p.start_time < q.end_time:
i = p.total_intersection_direction(q)
if i < 0:
n_diff += -i
elif i > 0:
n_same += i
return n_same, n_diff
def n_people(self, t):
"""Returns the number of people at a given time"""
if not self._counts_are_correct:
self._recompute_counts()
self._counts_are_correct = True
if t in self._counts:
return self._counts[t]
elif t < self.start_time:
return 0
else:
index = self._counts.bisect_left(t)
return self._counts.values()[index - 1]
def num_pedestrians(self):
"""Returns the total number of pedestrians in the simulation"""
return len(self.pedestrians)
def get_pedestrians_in_interval(self, start, stop):
"""Returns a list of all the pedestrians who entered in the interval
[start, stop]"""
return list(self.pedestrians.irange_key(start, stop))
def num_entrances_in_interval(self, start, stop):
"""Returns the number of pedestrians who entered in the given interval
of time [start, stop]"""
return len(self.get_pedestrians_in_interval(start, stop))
def get_enter_and_exit_times_in_interval(self, start, stop):
"""Returns the entrance and exit times in a given time interval
as a tuple of lists (entrance_times, exit_times)."""
start_times = []
end_times = []
for p in self.pedestrians:
if start <= p.start_time <= stop:
start_times.append(p.start_time)
if start <= p.end_time <= stop:
end_times.append(p.end_time)
return start_times, end_times
def get_pedestrians_at_time(self, t):
"""Returns a list of all the pedestrians who were there at time t"""
# get all pedestrians who entered at or before time t
entered_before_t = self.pedestrians.irange_key(
min_key=None, max_key=t, inclusive=(True, True))
# Of those, return return the ones who exited after time t
return [p for p in entered_before_t if p.end_time > t]
class TxGraph(object):
"""represents a graph of all transactions
within the current window
Attributes:
median(float) : the current median of the degree of the nodes
highMarker(int) : the latest timestamp seen so far
lowMarker(int) : the earliest timestamp of the window we are
interested in
txMap(dict) : this is a collection of EdgeList's with key being
the timestamp and the value an instance of EdgeList
edgeMap(dict) : this is collection of all Edges within a window
with key being the name of an Edge
nodeMap(dict) : this represents a collection of Nodes with a window
with key being the name of the Node
degreeList(list): list of degrees of noded (sorted)
"""
WINDOW_SIZE = 60
def __init__(self):
self.median = 0
self.highMarker = TxGraph.WINDOW_SIZE
self.lowMarker = 1
self.txMap = SortedDict() #sorted by unix epoch (timestamp)
self.edgeMap = SortedDict() #sorted by edge name
self.nodeMap = SortedDict() #sorted by node name
self.degreeList = SortedList() #sorted by degreeList
def __calculate_median(self, use_existing_list=False):
"""calculates median by adding degrees to a sortedlist
"""
if not use_existing_list:
#lets reconstruct the list
self.degreeList = SortedList()
for node in self.nodeMap.itervalues():
if node.degree > 0:
self.degreeList.add(node.degree)
listLen = len(self.degreeList)
if listLen == 0:
raise Exception("No items in the degreeList")
if listLen == 1:
return self.degreeList[0]/1.0
if (listLen % 2) == 0:
return (self.degreeList[listLen/2] + self.degreeList[(listLen/2) - 1]) / 2.0
return self.degreeList[listLen/2]/1.0
def __get_edgelist(self, tstamp, create=True):
"""returns an instance of EdgeList with matching
timestamp and creates one if needed
"""
edgeList = self.txMap.get(tstamp, None)
if edgeList is None and create is True:
edgeList = EdgeList(tstamp)
self.txMap[tstamp] = edgeList
return edgeList
def __getnode_with_name(self, name, create=True):
"""returns an instance of Node with matching name
and creates one if necessary
Args:
name(str) : name of the edge
create(bool): flag to indicate whether to create a
missing node
"""
node = self.nodeMap.get(name, None)
if node is None and create is True:
node = Node(name)
self.nodeMap[name] = node
return node
def __incr_degree_of_edge_nodes(self, edge):
"""increments the degree of the two nodes
of an edge
"""
src = self.__getnode_with_name(edge.source)
src.incr_degree()
tar = self.__getnode_with_name(edge.target)
tar.incr_degree()
return (src.degree, tar.degree)
def __decr_degree_of_edge_nodes(self, edge):
"""decrements the degree of the two nodes
of an edge
"""
self.__decr_degree_of_node(edge.source)
self.__decr_degree_of_node(edge.target)
def __decr_degree_of_node(self, name):
"""decrements the degree of a node
and removes it from the nodeMap if degree is 0
"""
node = self.__getnode_with_name(name, create=False)
node.decr_degree()
if node.degree == 0:
del self.nodeMap[node.name]
def __remove_edge(self, edge):
"""removes an edge from the graph and updates the
degree of a node. If degree of a node goes to 0, then
remove the node as well
Args:
egde(Edge) : An instance of Edge class
"""
self.__decr_degree_of_edge_nodes(edge)
del self.edgeMap[edge.name]
def __update_tstamp_for_existing_edge(self, edgeName, tstamp):
"""updates the timestamp for an existing edge and moves
the edge to an appropriate EdgeList
Args:
edgeName(str) : name of the edge to be updated
tstamp(int) : unix epoch of the timstamp
"""
currEdge = self.edgeMap[edgeName]
if not currEdge:
return
if tstamp <= currEdge.tstamp:
return #ignore older transactions within the window
#remove the edge from the edgelist with old timestamp
edgeList = self.__get_edgelist(currEdge.tstamp, create=False)
del edgeList.edges[currEdge.name]
#update the tstamp in the edge
currEdge.tstamp = tstamp
#move this edge to the correct edgelist
edgeList = self.__get_edgelist(tstamp)
edgeList.edges[currEdge.name] = currEdge
def __update_tx_window(self):
"""updates the transaction window of the graph
This method is called when a newer transaction out the
window arrives. It does the following:
1. Gets the edgeList's that are below the lowMarker
2. Goes through the edges and deletes them from the edgeMap
3. Update the degree of the nodes
4. Moves the window by deleting the stale edgeLists
"""
tsIter = self.txMap.irange(None, self.lowMarker, inclusive=(True,False))
lastTStamp = None
for tstamp in tsIter:
lastTStamp = tstamp
edgeList = self.txMap[tstamp]
for edge in edgeList.edges.itervalues():
self.__remove_edge(edge)
#lets delete the stale edgelists
if lastTStamp:
lowIdx = self.txMap.index(lastTStamp)
del self.txMap.iloc[:lowIdx+1]
def process_transaction(self, tstamp, source, target):
"""this is the starting point of transaction processing.
We first check whether the tx is within the window.
If it is, then we update the Edge (if it already exists) or
create a new Edge if necessary and update the median.
If the tx is not within the window and is newer, we then
move the window and remove all stale(older) edges and create
a new edge for the newer transaction and finally update the
median
"""
#basic sanity checks
if source is None or target is None:
raise Exception("Invalid node")
if len(source) == 0 or len(target) == 0:
raise Exception("Invalid node")
if source == target:
raise Exception("source and target cannot be the same")
#timestamp of the transaction is old and can be ignored
if tstamp < self.lowMarker:
return
#create a new edge representing this transaction
newEdge = Edge(tstamp, source, target)
if tstamp <= self.highMarker:
if newEdge.name in self.edgeMap:
self.__update_tstamp_for_existing_edge(newEdge.name, tstamp)
#no need to recalculate the median here since degree does not change
return
"""handle new edge
1. find the edgelist with the same timestamp (if not create it)
2. add this edge to the edgelist and edgemap
4. create new Nodes for the edges if needed or update their degrees
5. update the degreeList with the new degrees
6. recalculate the median but use the existing degreeList
"""
edgeList = self.__get_edgelist(tstamp)
edgeList.edges[newEdge.name] = newEdge
self.edgeMap[newEdge.name] = newEdge
"""
this is optimization because most of the degrees of the
nodes hasn't changed and therefore we can reuse the existing list
"""
srcDegree, tarDegree = self.__incr_degree_of_edge_nodes(newEdge)
if srcDegree == 1:
self.degreeList.add(1)
else:
self.degreeList.remove(srcDegree - 1)
self.degreeList.add(srcDegree)
if tarDegree == 1:
self.degreeList.add(1)
else:
self.degreeList.remove(tarDegree - 1)
self.degreeList.add(tarDegree)
self.median = self.__calculate_median(use_existing_list=True)
return
"""this transaction is newer and we need to move the window
1. update the low and high markers of the timestamp window
2. create edgelist with this newer timestamp
2. add the new edge to the edgelist
3. add the new edge to the edgemap
4. create new Nodes of the edges if needed or update their degrees
5. calculate the median (but reconstruct the degreeList)
"""
#this tx is newer and we need to move the window
self.highMarker = tstamp
self.lowMarker = tstamp - TxGraph.WINDOW_SIZE + 1
self.__update_tx_window()
if newEdge.name in self.edgeMap:
self.__update_tstamp_for_existing_edge(newEdge.name, tstamp)
else:
edgeList = self.__get_edgelist(tstamp)
edgeList.edges[newEdge.name] = newEdge
self.edgeMap[newEdge.name] = newEdge
self.__incr_degree_of_edge_nodes(newEdge)
self.median = self.__calculate_median()
class RegionMap(object):
"""
Mostly used in SimAbstractMemory, RegionMap stores a series of mappings between concrete memory address ranges and
memory regions, like stack frames and heap regions.
"""
def __init__(self, is_stack):
"""
Constructor
:param is_stack: Whether this is a region map for stack frames or not. Different strategies apply for stack
regions.
"""
self.is_stack = is_stack
# A sorted list, which maps stack addresses to region IDs
self._address_to_region_id = SortedDict()
# A dict, which maps region IDs to memory address ranges
self._region_id_to_address = { }
#
# Properties
#
def __repr__(self):
return "RegionMap<%s>" % (
"S" if self.is_stack else "H"
)
@property
def is_empty(self):
return len(self._address_to_region_id) == 0
@property
def stack_base(self):
if not self.is_stack:
raise SimRegionMapError('Calling "stack_base" on a non-stack region map.')
return next(self._address_to_region_id.irange(reverse=True))
@property
def region_ids(self):
return self._region_id_to_address.keys()
#
# Public methods
#
@SimStatePlugin.memo
def copy(self, memo): # pylint: disable=unused-argument
r = RegionMap(is_stack=self.is_stack)
# A shallow copy should be enough, since we never modify any RegionDescriptor object in-place
r._address_to_region_id = self._address_to_region_id.copy()
r._region_id_to_address = self._region_id_to_address.copy()
return r
def map(self, absolute_address, region_id, related_function_address=None):
"""
Add a mapping between an absolute address and a region ID. If this is a stack region map, all stack regions
beyond (lower than) this newly added regions will be discarded.
:param absolute_address: An absolute memory address.
:param region_id: ID of the memory region.
:param related_function_address: A related function address, mostly used for stack regions.
"""
if self.is_stack:
# Sanity check
if not region_id.startswith('stack_'):
raise SimRegionMapError('Received a non-stack memory ID "%d" in a stack region map' % region_id)
# Remove all stack regions that are lower than the one to add
while True:
try:
addr = next(self._address_to_region_id.irange(maximum=absolute_address, reverse=True))
descriptor = self._address_to_region_id[addr]
# Remove this mapping
del self._address_to_region_id[addr]
# Remove this region ID from the other mapping
del self._region_id_to_address[descriptor.region_id]
except StopIteration:
break
else:
if absolute_address in self._address_to_region_id:
descriptor = self._address_to_region_id[absolute_address]
# Remove this mapping
del self._address_to_region_id[absolute_address]
del self._region_id_to_address[descriptor.region_id]
# Add this new region mapping
desc = RegionDescriptor(
region_id,
absolute_address,
related_function_address=related_function_address
)
self._address_to_region_id[absolute_address] = desc
self._region_id_to_address[region_id] = desc
def unmap_by_address(self, absolute_address):
"""
Removes a mapping based on its absolute address.
:param absolute_address: An absolute address
"""
desc = self._address_to_region_id[absolute_address]
del self._address_to_region_id[absolute_address]
del self._region_id_to_address[desc.region_id]
def absolutize(self, region_id, relative_address):
"""
Convert a relative address in some memory region to an absolute address.
:param region_id: The memory region ID
:param relative_address: The relative memory offset in that memory region
:return: An absolute address if converted, or an exception is raised when region id does not
exist.
"""
if region_id == 'global':
# The global region always bases 0
return relative_address
if region_id not in self._region_id_to_address:
raise SimRegionMapError('Non-existent region ID "%s"' % region_id)
base_address = self._region_id_to_address[region_id].base_address
return base_address + relative_address
def relativize(self, absolute_address, target_region_id=None):
"""
Convert an absolute address to the memory offset in a memory region.
Note that if an address belongs to heap region is passed in to a stack region map, it will be converted to an
offset included in the closest stack frame, and vice versa for passing a stack address to a heap region.
Therefore you should only pass in address that belongs to the same category (stack or non-stack) of this region
map.
:param absolute_address: An absolute memory address
:return: A tuple of the closest region ID, the relative offset, and the related function
address.
"""
if target_region_id is None:
if self.is_stack:
# Get the base address of the stack frame it belongs to
base_address = next(self._address_to_region_id.irange(minimum=absolute_address, reverse=False))
else:
try:
base_address = next(self._address_to_region_id.irange(maximum=absolute_address, reverse=True))
except StopIteration:
# Not found. It belongs to the global region then.
return 'global', absolute_address, None
descriptor = self._address_to_region_id[base_address]
else:
if target_region_id == 'global':
# Just return the absolute address
return 'global', absolute_address, None
if target_region_id not in self._region_id_to_address:
raise SimRegionMapError('Trying to relativize to a non-existent region "%s"' % target_region_id)
descriptor = self._region_id_to_address[target_region_id]
base_address = descriptor.base_address
return descriptor.region_id, absolute_address - base_address, descriptor.related_function_address
class ProductCollection:
def __init__(self):
self.products = {}
self.products_by_title = {}
self.products_by_price = SortedDict()
self.products_by_price_and_title = {}
self.products_by_price_and_supplier = {}
def add(self, product):
self._add_to_products_by_id(product)
self._add_to_products_by_title(product)
self._add_to_products_by_price(product)
self._add_to_products_by_price_and_title(product)
self._add_to_products_by_price_and_supplier(product)
def remove(self, _id):
if _id not in self.products:
return False
product = self.products[_id]
self._remove_from_products(product)
self._remove_from_products_by_title(product)
self._remove_from_products_by_price(product)
self._remove_from_products_by_price_and_title(product)
self._remove_from_products_by_price_and_supplier(product)
def find_products_in_price_range(self, start_price, end_price):
if start_price < 0 or start_price > end_price:
raise Exception('Invalid price range!')
prices = self.products_by_price.irange(start_price, end_price)
return (product for price in prices
for product in self.products_by_price[price])
def find_products_by_title(self, title: str):
if title not in self.products_by_title:
return []
return (product for product in self.products_by_title[title])
def find_products_by_title_and_price(self, title, price):
if title not in self.products_by_price_and_title or price not in self.products_by_price_and_title[
title]:
return []
return (product
for product in self.products_by_price_and_title[title][price])
def find_products_by_title_and_price_range(self, title, start_price,
end_price):
if title not in self.products_by_price_and_title:
return []
if start_price < 0 or start_price > end_price:
raise Exception('Invalid price range!')
prices = self.products_by_price_and_title[title].irange(
start_price, end_price)
return (product for price in prices
for product in self.products_by_price_and_title[title][price])
def find_products_by_supplier_and_price(self, supplier, price):
if (supplier not in self.products_by_price_and_supplier
or price not in self.products_by_price_and_supplier[supplier]):
return []
return (product for product in
self.products_by_price_and_supplier[supplier][price])
def find_products_by_supplier_and_price_range(self, supplier, start_price,
end_price):
if supplier not in self.products_by_price_and_supplier:
return []
if start_price < 0 or start_price > end_price:
raise Exception('Invalid price range!')
prices = self.products_by_price_and_supplier[supplier].irange(
start_price, end_price)
return (product for price in prices for product in
self.products_by_price_and_supplier[supplier][price])
def _add_to_products_by_id(self, product):
self.products[product.id] = product
def _remove_from_products(self, product):
del self.products[product.id]
def _add_to_products_by_title(self, product):
if product.title not in self.products_by_title:
self.products_by_title[product.title] = SortedSet()
self.products_by_title[product.title].add(product)
def _remove_from_products_by_title(self, product):
self.products_by_title[product.title].remove(product)
def _add_to_products_by_price(self, product):
if product.price not in self.products_by_price:
self.products_by_price[product.price] = SortedSet()
self.products_by_price[product.price].add(product)
def _remove_from_products_by_price(self, product):
self.products_by_price[product.price].remove(product)
def _add_to_products_by_price_and_title(self, product):
if product.title not in self.products_by_price_and_title:
self.products_by_price_and_title[product.title] = SortedDict()
if product.price not in self.products_by_price_and_title[
product.title]:
self.products_by_price_and_title[product.title][
product.price] = SortedSet()
self.products_by_price_and_title[product.title][product.price].add(
product)
def _remove_from_products_by_price_and_title(self, product):
self.products_by_price_and_title[product.title][product.price].remove(
product)
def _add_to_products_by_price_and_supplier(self, product):
if product.supplier not in self.products_by_price_and_supplier:
self.products_by_price_and_supplier[
product.supplier] = SortedDict()
if product.price not in self.products_by_price_and_supplier[
product.supplier]:
self.products_by_price_and_supplier[product.supplier][
product.price] = SortedSet()
self.products_by_price_and_supplier[product.supplier][
product.price].add(product)
def _remove_from_products_by_price_and_supplier(self, product):
self.products_by_price_and_supplier[product.supplier][
product.price].remove(product)
class PiecewiseConstantFunction(Generic[T]):
def __init__(self, initial_value: float = 0) -> None:
""" Initialize the constant function to a particular value
:param initial_value: the starting value for the function
"""
self.breakpoints = SortedDict()
self._initial_value: float = initial_value
def add_breakpoint(self,
xval: XValue[T],
yval: float,
squash: bool = True) -> None:
""" Add a breakpoint to the function and update the value
Let f(x) be the original function, and next_bp be the first breakpoint > xval; after calling
this method, the function will be modified to f'(x) = yval for x \in [xval, next_bp)
:param xval: the x-position of the breakpoint to add/modify
:param yval: the value to set the function to at xval
:param squash: if True and f(xval) = yval before calling this method, the function will remain unchanged
"""
if squash and self.call(xval) == yval:
return
self.breakpoints[xval] = yval
def add_delta(self, xval: XValue[T], delta: float) -> None:
""" Modify the function value for x >= xval
Let f(x) be the original function; After calling this method,
the function will be modified to f'(x) = f(x) + delta for all x >= xval
:param xval: the x-position of the breakpoint to add/modify
:param delta: the amount to shift the function value by at xval
"""
if delta == 0:
return
if xval not in self.breakpoints:
self.breakpoints[xval] = self.call(xval)
for x in self.breakpoints.irange(xval):
self.breakpoints[x] += delta
self.values.cache_clear()
self.integrals.cache_clear()
def call(self, xval: XValue[T]) -> float:
""" Compute the output of the function at a point
:param xval: the x-position to compute
:returns: f(xval)
"""
if len(self.breakpoints) == 0 or xval < self.breakpoints.keys()[0]:
return self._initial_value
else:
lower_index = self.breakpoints.bisect(xval) - 1
return self.breakpoints.values()[lower_index]
def _breakpoint_info(
self, index: Optional[int]
) -> Tuple[Optional[int], Optional[XValue[T]], float]:
""" Helper function for computing breakpoint information
:param index: index of the breakpoint to compute
:returns: (index, breakpoint, value)
* index is the breakpoint index (if it exists), or None if we're off the end
* breakpoint is the x-value of the breakpoint, or None if we're off the end
* value is f(breakpoint), or f(last_breakpoint) if we're off the end
"""
try:
breakpoint, value = self.breakpoints.peekitem(index)
except IndexError:
index = None
breakpoint, value = None, self.breakpoints.values()[-1]
return (index, breakpoint, value)
@lru_cache(maxsize=_LRU_CACHE_SIZE
) # cache results of calls to this function
def values(self, start: XValue[T], stop: XValue[T],
step: XValueDiff[T]) -> 'SortedDict[XValue[T], float]':
""" Compute a sequence of values of the function
This is more efficient than [self.call(xval) for xval in range(start, stop, step)] because each self.call(..)
takes O(log n) time due to the binary tree structure of self._breakpoints. This method can compute the range
of values in linear time in the range, which is significantly faster for large value ranges.
:param start: lower bound of value sequence
:param stop: upper bound of value sequence
:param step: width between points in the sequence
:returns: a SortedDict of the values of the function between start and stop, with the x-distance between
each data-point equal to `step`; like normal "range" functions the right endpoint is not included
"""
step = step or (stop - start)
if len(self.breakpoints) == 0:
num_values = int(math.ceil((stop - start) / step))
return SortedDict([(start + step * i, self._initial_value)
for i in range(num_values)])
curr_xval = start
curr_value = self.call(start)
next_index, next_breakpoint, next_value = self._breakpoint_info(
self.breakpoints.bisect(start))
sequence = SortedDict()
while curr_xval < stop:
sequence[curr_xval] = curr_value
next_xval = min(stop, curr_xval + step)
while next_breakpoint and next_xval >= next_breakpoint:
assert next_index is not None # if next_breakpoint is set, next_index should also be set
curr_value = next_value
next_index, next_breakpoint, next_value = self._breakpoint_info(
next_index + 1)
curr_xval = next_xval
return sequence
@lru_cache(maxsize=_LRU_CACHE_SIZE
) # cache results of calls to this function
def integrals(
self,
start: XValue[T],
stop: XValue[T],
step: XValueDiff[T],
transform: Callable[[XValueDiff[T]], float] = lambda x: cast(float, x),
) -> 'SortedDict[XValue[T], float]':
""" Compute a sequence of integrals of the function
:param start: lower bound of integral sequence
:param stop: upper bound of integral sequence
:param step: width of each "chunk" of the integral sequence
:param transform: function to apply to x-widths before computing the integral
:returns: a SortedDict of the numeric integral values of the function between start and stop;
each integral has a range of size `step`, and the key-value is the left endpoint of the chunk
"""
step = step or (stop - start)
if len(self.breakpoints) == 0:
# If there are no breakpoints, just split up the range into even widths and compute
# (width * self._initial_value) for each chunk.
step_width = transform(step)
range_width = transform(stop - start)
num_full_chunks = int(range_width // step_width)
sequence = SortedDict([(start + step * i,
step_width * self._initial_value)
for i in range(num_full_chunks)])
# If the width does not evenly divide the range, compute the last chunk separately
if range_width % step_width != 0:
sequence[
start + step *
num_full_chunks] = range_width % step_width * self._initial_value
return sequence
# Set up starting loop parameters
curr_xval = start
curr_value = self.call(start)
next_index, next_breakpoint, next_value = self._breakpoint_info(
self.breakpoints.bisect(start))
# Loop through the entire range and compute the integral of each chunk
sequence = SortedDict()
while curr_xval < stop:
orig_xval = curr_xval
next_xval = min(stop, curr_xval + step)
# For each breakpoint in [curr_xval, next_xval), compute the area of that sub-chunk
next_integral: float = 0
while next_breakpoint and next_xval >= next_breakpoint:
assert next_index is not None # if next_breakpoint is set, next_index should also be set
next_integral += transform(next_breakpoint -
curr_xval) * curr_value
curr_xval = next_breakpoint
curr_value = next_value
next_index, next_breakpoint, next_value = self._breakpoint_info(
next_index + 1)
# Handle any remaining width between the last breakpoint and the end of the chunk
next_integral += transform(next_xval - curr_xval) * curr_value
sequence[orig_xval] = next_integral
curr_xval = next_xval
return sequence
def integral(
self,
start: XValue[T],
stop: XValue[T],
transform: Callable[[XValueDiff[T]], float] = lambda x: cast(float, x),
) -> float:
""" Helper function to compute the integral of the whole specified range
:param start: lower bound of the integral
:param stop: upper bound of the integral
:returns: the integral of the function between start and stop
"""
return self.integrals(start, stop, (stop - start),
transform).values()[0]
def __str__(self) -> str:
ret = f'{self._initial_value}, x < {self.breakpoints.keys()[0]}\n'
for xval, yval in self.breakpoints.items():
ret += f'{yval}, x >= {xval}\n'
return ret
def __add__(
self, other: 'PiecewiseConstantFunction[T]'
) -> 'PiecewiseConstantFunction[T]':
new_func: 'PiecewiseConstantFunction[T]' = PiecewiseConstantFunction(
self._initial_value + other._initial_value)
for xval, y0, y1 in _merged_breakpoints(self, other):
new_func.add_breakpoint(xval, y0 + y1)
return new_func
def __sub__(
self, other: 'PiecewiseConstantFunction[T]'
) -> 'PiecewiseConstantFunction[T]':
new_func: 'PiecewiseConstantFunction[T]' = PiecewiseConstantFunction(
self._initial_value - other._initial_value)
for xval, y0, y1 in _merged_breakpoints(self, other):
new_func.add_breakpoint(xval, y0 - y1)
return new_func
def __mul__(
self, other: 'PiecewiseConstantFunction[T]'
) -> 'PiecewiseConstantFunction[T]':
new_func: 'PiecewiseConstantFunction[T]' = PiecewiseConstantFunction(
self._initial_value * other._initial_value)
for xval, y0, y1 in _merged_breakpoints(self, other):
new_func.add_breakpoint(xval, y0 * y1)
return new_func
def __truediv__(
self, other: 'PiecewiseConstantFunction[T]'
) -> 'PiecewiseConstantFunction[T]':
try:
new_func: 'PiecewiseConstantFunction[T]' = PiecewiseConstantFunction(
self._initial_value / other._initial_value)
except ZeroDivisionError:
new_func = PiecewiseConstantFunction()
for xval, y0, y1 in _merged_breakpoints(self, other):
try:
new_func.add_breakpoint(xval, y0 / y1)
except ZeroDivisionError:
new_func.add_breakpoint(xval, 0)
return new_func
class DwarfData(object):
"""
Loads dwarf data from the given file.
Data:
- DIEs: Apart from attributes, it also contains DIE offset as well as
parent DIE offset in order to build a tree.
- LineInfoMap: Instruction address to LineInfoEntry
- LineInfoEntries: Map of entries containing line info
TODO: - Include DW_TAG_formal_parameter?
"""
def __init__(self, path):
self._dwData = None
self._dieTree = None
self._subDies = None
self._subprograms = None
self._inlSubroutines = {}
self._dwLineMap = None
# Read dwarf data.
self._read_data(path)
assert self._validate_data(), "Invalid dwarf data."
# Process dwarf data.
self._build_die_tree()
self._find_subprograms()
self._find_inlined_subroutines()
# Save LineInfoMap as SortedDict
self._dwLineMap = SortedDict({
int(k): int(v)
for k, v in self._dwData['LineInfoMap'].iteritems()
})
def get_dw_lines(self, ranges):
"""
Return dwarf lines that belong to given instruction address ranges
Args
ranges : List of ranges to retrieve dwarf line entries for
Return
dwLines : Dict of DwarfLine entries referenced in given range list
"""
dwLineIndices = []
for r in ranges:
for dwl_i in self._dwLineMap.irange(minimum=r[0], maximum=r[1]):
dwLineIndices.append(self._dwLineMap[dwl_i])
dwLines = {
i: self._dwData['LineInfoEntries'][str(i)]
for i in set(dwLineIndices)
}
return dwLines
def get_line_info(self, ranges):
"""
Returns line info for the given address ranges.
Args:
ranges : list of address ranges
Return:
Returns dict containing the following elements:
- begin : Begin Line
- end : End Line
- max : Min Line
- min : Max Line
where each element is a dict: {l, c, d} where:
- l : line
- c : column
- d : discriminator
* All values are integers.
"""
assert isinstance(ranges, list)
l_info = {'begin': {}, 'end': {}, 'max': {}, 'min': {}}
def get_lcd(dwlIndex):
dwl = self._dwData['LineInfoEntries'].get(str(dwlIndex), None)
if dwl is None:
assert False, "Invalid dwLine index."
return {
'l': int(dwl['LineNumber']),
'c': int(dwl['LineOffset']),
'd': int(dwl['Discriminator'])
}
l_info['begin'] = get_lcd(self._dwLineMap[ranges[0][0]])
l_info['end'] = get_lcd(self._dwLineMap[ranges[-1][-1]])
minLine = None
maxLine = None
for r in ranges:
for dwl_i in self._dwLineMap.irange(minimum=r[0], maximum=r[1]):
dwlIndex = self._dwLineMap[dwl_i]
lcd = get_lcd(dwlIndex)
if minLine is None and maxLine is None:
minLine = lcd
maxLine = lcd
continue
# Max
if lcd['l'] > maxLine['l']:
maxLine = lcd
elif lcd['l'] == maxLine['l']:
if lcd['c'] > maxLine['c']:
maxLine = lcd
elif lcd['c'] == maxLine['c']:
if lcd['d'] > maxLine['d']:
maxLine = lcd
# Same for min...
if lcd['l'] < minLine['l']:
minLine = lcd
elif lcd['l'] == minLine['l']:
if lcd['c'] < minLine['c']:
minLine = lcd
elif lcd['c'] == minLine['c']:
if lcd['d'] < minLine['d']:
minLine = lcd
# Update min/max
l_info['max'] = maxLine
l_info['min'] = minLine
return l_info
def get_subprograms(self):
"""
Get subprogram dict.
Return:
Dict keyed by subprogram low_pc (serving as entry point) with values
of the following form: {name, dieOffset}.
"""
return self._subprograms
def get_subprogram_file(self, dieOffset):
"""
Returns the file associated with the given subprogram DIE.
Args:
dieOffset: Offset of subprogram DIE.
Return:
Tuple (comp_dir, filename).
FIXME:
Return the actual file. The source file table must be exported in JSON
(it is already parsed from the elf object). For now, the file path
of parent CU is returned instead.
"""
assert dieOffset in self._subDies, "Invalid subprogram DIE offset."
# Get parent DIE (CU)
cuDie = list(self._dieTree.predecessors(dieOffset))
assert len(cuDie) == 1, "Invalid DIE tree."
cuDie = cuDie[0]
# Get the source file table associated with this CU. Use DW_AT_decl_file
# index for retrieving the actual file.
# FIXME: Return the actual file.
cuDieAttrs = self._dieTree.nodes[cuDie]['attrs']
return cuDieAttrs['DW_AT_comp_dir'], cuDieAttrs['DW_AT_name']
def get_inlined_subroutines(self, subDieOffset):
"""
Returns a list of inlined subroutines for the given subprogram DIE offset.
Args:
subDieOffset: Valid subprogram DIE offset.
Note:
** high_pc points at first byte after last instruction
Return:
List of inlined subroutines of the form {dieOffset, low_pc, high_pc}.
Returns an empty list if given subprogram does not contain inlined
subroutines.
"""
assert subDieOffset in self._subDies
return self._inlSubroutines.get(subDieOffset, [])
def get_local_variables(self, subDieOffset):
"""
Returns local variables given a subroutine DIE offset. The subroutine DIE
may be a normal subroutine or an abstract instance root entry (part of
self._subDies).
Note:
SP register hardcoded for avr (fbreg28).
Args:
subDieOffset: Offset of subprogram DIE containing variables.
Return:
A dict keyed by variable offset in stack, with values of the form
{name, byteSize}.
"""
def get_locs(chDie):
rx = re.compile(r'\[([^\]]+)\];')
return rx.findall(chDie['attrs']['DW_AT_location'])
def get_ops(loc):
assert loc[-1] == ','
return loc.split(',')[:-1]
assert subDieOffset in self._subDies, \
"Invalid subprogram DIE offset."
localVars = {}
for s in self._dieTree.successors(subDieOffset):
chDie = self._dieTree.nodes[s]
if chDie['tag'] != 'DW_TAG_variable':
continue
if not {'DW_AT_name', 'DW_AT_location', 'DW_AT_type'} <= set(
chDie['attrs'].keys()):
continue
locs = get_locs(chDie)
name = chDie['attrs']['DW_AT_name']
if len(locs) == 1:
ops = get_ops(locs[0])
if len(ops) == 1:
o = ops[0].split(':')
if len(o) == 2:
k = o[0]
v = o[1]
else:
continue
if k != 'DW_OP_breg28':
continue
# Get base type
typeDieOffset = int(chDie['attrs']['DW_AT_type'])
if not self._dieTree.has_node(typeDieOffset):
continue
bs = self._get_type_byte_size(typeDieOffset)
localVars[v] = {'name': name, 'byteSize': bs}
return localVars
def _get_type_byte_size(self, die):
"""Decodes byte size for a DIE with tag DW_TAG_base_type."""
if not self._dieTree.has_node(die):
assert False, "Invalid DIE offset."
die = self._dieTree.nodes[die]
assert die['tag'] == 'DW_TAG_base_type'
assert 'DW_AT_byte_size' in die['attrs'].keys()
return self._decode_dwarf_constant(die['attrs']['DW_AT_byte_size'])[1]
def _decode_dwarf_constant(self, attr):
"""
Decodes a dwarf constant of the form S_{}_U_{}, returning signed, unsigned.
Args:
attr: Attribute (string) of the form S_{}_U_{}.
Return:
Tuple (s, u), signed and unsigned respectively.
"""
m = re.search(r'S_([+\-0-9]+)_U_([+\-0-9]+)', attr)
assert m is not None
s = int(m.group(1))
u = int(m.group(2))
assert u >= 0
return s, u
def _find_inlined_subroutines(self):
"""
Scans subtrees with roots in self._subDies for DIEs with tag
'DW_AT_inlined_subroutine'. Only inlined subroutines which
have low_pc, high_pc are processed here. Nested inlining is not
supported.
Note:
Inlined subroutine DIEs are marked in self._inlSubroutines. This
dict is keyed by subprogram DIE offset, where each value is a list
of inlined subroutines of the form {dieOffset, low_pc, high_pc}.
Return:
Void
"""
def add_inl(die, node, subp):
"""
Add the inlined subroutine to self._inlSubroutines. 'Decode' high_pc
if it represents an offset.
Args:
die: Die offset.
node: Die node is self._dieTree.
subp: Offset of subprogram DIE containing inlined node.
Return:
Void.
"""
assert subp in self._subDies
# Get range
lo_pc = int(node['attrs']['DW_AT_low_pc'])
if node['attrs']['DW_AT_high_pc'][0] == 'S':
# Decode dwarf constant
_, off = self._decode_dwarf_constant(
node['attrs']['DW_AT_high_pc'])
hi_pc = lo_pc + off
else:
hi_pc = int(node['attrs']['DW_AT_high_pc'])
# Update self._inlSubroutines
if subp not in self._inlSubroutines:
self._inlSubroutines[subp] = []
self._inlSubroutines[subp].append({
'dieOffset': die,
'low_pc': lo_pc,
'high_pc': hi_pc
})
def get_surrounding_sub(dnode):
"""Walk up the tree and return next surrounding subroutine
Not necessarily the direct parent. There may be lexical scopes and other stuff.
"""
n = dnode
try:
while n:
n = next(iter(self._dieTree.predecessors(n)))
dd = self._dieTree.nodes[n]
if 'tag' in dd and dd['tag'] in SUBROUTINE:
return n
except StopIteration:
pass
return None
def get_sub_details(doff):
"""Return dict with details of subroutine. If die is inlined, go to abstract origin"""
assert int(doff) in self._dieTree.nodes, "DWARF data incomplete"
# --
dd = self._dieTree.nodes[int(doff)]
assert dd['tag'] in SUBROUTINE, "not a subprogram"
if dd['tag'] == "DW_TAG_inlined_subroutine":
return get_sub_details(dd['attrs']['DW_AT_abstract_origin'])
return dd['attrs']
def test_die(die, subp):
"""
Test if die is an inlined node, add it to self._inlSubroutines.
Note:
- Nested inlining not supported.
- DW_AT_ranges not supported.
Args:
die: Die offset.
subp: Subprogram die offset.
Returns:
Void.
"""
dNode = self._dieTree.nodes[die]
if dNode['tag'] == 'DW_TAG_inlined_subroutine':
inlined_into = int(get_surrounding_sub(die))
abstract_orig = int(dNode['attrs']['DW_AT_abstract_origin'])
assert inlined_into is not None, "Inlined sub without surrounding sub"
sub_into = get_sub_details(inlined_into)
sub_myself = get_sub_details(abstract_orig)
log.info(
"Sub '{}' (die +{:x}) was inlined into '{}' (die +{:x})".
format(sub_myself['DW_AT_name'], die,
sub_into['DW_AT_name'], inlined_into))
if self._dieTree.node[inlined_into][
'tag'] == 'DW_TAG_inlined_subroutine':
raise NotImplementedError(
"nested inlining @die offset {:x}".format(die))
# Check if it contains DW_AT_ranges
if 'DW_AT_ranges' in dNode['attrs']:
raise NotImplementedError(
"Inlined subroutine not in contiguous range.")
assert {'DW_AT_low_pc', 'DW_AT_high_pc'} <= set(dNode['attrs'].keys()), \
"Incorrect inlined subroutine DIE @0d{}".format(die)
# Add inlined subroutine
add_inl(die, dNode, subp)
def find_children(subt, root):
"""Search the subtree rooted by subp recursively."""
for ch in self._dieTree.successors(subt):
test_die(ch, root)
find_children(ch, root)
for s in self._subDies:
find_children(s, s)
if self._inlSubroutines:
log.warning("Support for inlining is experimental!")
def _find_subprograms(self):
"""
Finds subprograms that have debug info, notes them in self._subprograms.
NOTE: Only subprogram DIEs that are direct children of CU DIEs are
processed here (as it should be).
NOTE: Only subprogram DIEs that have low_pc, high_pc are processed here.
DIEs with an DW_AT_ranges are skipped here (non-contiguous case).
To be completed later... (FIXME).
** It is assumed that low_pc represents the entry point of the
subprogram. If this is not the case, then the Executable class will
simply assume that this subroutine does not possess debug info
(mismatch between entry point as symbol address in json data and
the low_pc address here).
-----
NOTE: Subprogram DIEs that are abstract instance root entries are
are not added to self._subprograms, but are noted in self._subDies.
This happens for subprograms that are always inlined, so that there
is no external symbol left.
NOTE: It can happen that a subprogram DIE (DW_TAG_subprogram) which is
not a concrete out-of-line instance (see below) may be missing
a DW_AT_name or DW_AT_low_pc attribute. Example (memmove in maxleaf).
-----
NOTE: DIEs with DW_TAG_subprogram which have an DW_AT_abstract_origin
attribute represent a concrete out-of-line instance of an
inlinable subroutine.
"""
self._subDies = []
self._subprograms = {}
# Walk the tree, only traverse CU's that have DW_AT_comp_dir and
# DW_AT_name attributes.
#
# FIXME: Process all CU's after fixing self.get_subprogram_file()
cuDies = self._dieTree.successors(0)
for cu in cuDies:
cuAttrs = self._dieTree.nodes[cu]['attrs']
if not {'DW_AT_name', 'DW_AT_comp_dir'} <= set(cuAttrs.keys()):
continue
# Find subprogram DIE's that are children of current CU DIE
for s in self._dieTree.successors(cu):
die = self._dieTree.nodes[s]
if die['tag'] != 'DW_TAG_subprogram':
continue
# Check if it has an abstract origin attribute -> concrete
# out of line instance.
if 'DW_AT_abstract_origin' in die['attrs']:
# log.debug("Skipping subprog DIE @0d{}, concrete out-of-line
# instance".format(s))
continue
# Don't process abstract instance root entries here but note
# them in self._subDies.
if 'DW_AT_inline' in die['attrs']:
log.debug(
"Subprog DIE @0d{} is an abstract instance root entry."
.format(s))
self._subDies.append(s)
continue
if not {'DW_AT_low_pc'} <= set(die['attrs'].keys()):
# a declaration
continue
if 'DW_AT_ranges' in die['attrs']:
log.error(
"Skipping subprogram DIE @0d{}, contains DW_AT_ranges."
.format(s))
continue
self._subprograms.update({
int(die['attrs']['DW_AT_low_pc']): {
'name': die['attrs']['DW_AT_name'],
'dieOffset': s
}
})
self._subDies.append(s)
def _build_die_tree(self):
"""
Builds the die tree as nx.DiGraph. Children of the root node are
Compilation Unit DIE's found in .debug_info section. Each node has
the following attributes:
- tag : DW_TAG_*
- attrs : DW_AT_*
Note:
Nodes are keyed by DIE offset, the root node having offset 0.
"""
dieTree = nx.DiGraph()
dieTree.add_node(0, tag='root')
for die in self._dwData['DIEs']:
assert die['Offset'] != 0, "Invalid DIE."
# Add node
dieTree.add_node(die['Offset'],
tag=die['Tag'],
attrs=die['Attributes'])
# Add edge to parent
dieTree.add_edge(die['ParentOffset'], die['Offset'])
# Check if graph is indeed a tree, raise an error if not.
assert nx.is_tree(dieTree), "Invalid DIE tree."
self._dieTree = dieTree
def _read_data(self, path):
with open(path, 'r') as fp:
self._dwData = json.load(fp)
def _validate_data(self):
status = False
while True:
if not {'Type', 'Data'} <= set(self._dwData.keys()):
break
if self._dwData['Type'] != 'DebugInfo':
break
if not {'DIEs', 'LineInfoEntries', 'LineInfoMap'} <= \
set(self._dwData['Data'].keys()):
break
if len(self._dwData['Data']['DIEs']) < 1:
log.error("Empty DIE map.")
break
self._dwData = self._dwData['Data']
status = True
break
return status
class DynamicCHConfig(MemcacheKVConfiguration):
def __init__(self, cache_nodes, db_node, write_mode, c, hash_space):
super().__init__(cache_nodes, db_node, write_mode)
self.c = c
self.hash_space = hash_space
self.key_hash_ring = SortedDict()
self.node_hash_ring = SortedDict()
self.node_hashes = {}
self.key_rates = {}
self.iloads = {}
self.ploads = {}
for node in self.cache_nodes:
self.iloads[node.id] = 0
self.ploads[node.id] = 0
node_hash = node.id * (hash_space // len(cache_nodes))
self.node_hashes[node.id] = node_hash
self.node_hash_ring[node_hash] = node.id
def key_hash_fn(self, key):
return hash(key)
def lookup_node(self, key):
key_hash = self.key_hash_fn(key) % self.hash_space
for (node_hash, node_id) in self.node_hash_ring.items():
if node_hash >= key_hash:
return node_id
return self.node_hash_ring.peekitem(0)[1]
def install_key(self, key):
key_hash = self.key_hash_fn(key) % self.hash_space
keys = self.key_hash_ring.setdefault(key_hash, set())
keys.add(key)
def remove_key(self, key):
key_hash = self.key_hash_fn(key) % self.hash_space
keys = self.key_hash_ring.get(key_hash, set())
keys.discard(key)
def search_migration_keys(self, node_id, starting_hash, agg_pload,
target_pload, migration_keys):
mapped_key_hashes = list(
self.key_hash_ring.irange(minimum=0,
maximum=starting_hash,
reverse=True))
new_node_hash = None
for key_hash in mapped_key_hashes:
if key_hash in self.node_hash_ring:
assert self.node_hash_ring[key_hash] == node_id
prev_hash = (key_hash - 1) % self.hash_space
if prev_hash in self.node_hash_ring:
# Never collide two nodes on the hash ring!
assert self.node_hash_ring[prev_hash] != node_id
new_node_hash = key_hash
break
for key in self.key_hash_ring[key_hash]:
agg_pload += self.key_rates[key].rate()
if agg_pload >= target_pload:
new_node_hash = prev_hash
migration_keys.append(key)
if new_node_hash is not None:
break
return (agg_pload, new_node_hash)
def rehash_node(self, node_id, target_pload):
node_hash = self.node_hashes[node_id]
migration_dst = None
# Find next node in the hash ring to migrate to
for (next_node_hash, next_node_id) in self.node_hash_ring.items():
if next_node_hash > node_hash:
migration_dst = next_node_id
break
if migration_dst is None:
migration_dst = self.node_hash_ring.peekitem(0)[1]
assert migration_dst != node_id
# Find set of keys to migrate, and the new node hash
migration_keys = []
(agg_pload,
new_node_hash) = self.search_migration_keys(node_id, node_hash, 0,
target_pload,
migration_keys)
if new_node_hash is None:
# Search beyond hash 0
(agg_pload, new_node_hash) = self.search_migration_keys(
node_id, self.hash_space - 1, agg_pload, target_pload,
migration_keys)
assert new_node_hash is not None
if new_node_hash == node_hash:
# We cannot find a new hash. This is due to
# the next hash on the ring already occupied
# by another node.
assert len(migration_keys) == 0
return None
# Update node hash
self.node_hashes[node_id] = new_node_hash
self.node_hash_ring.pop(node_hash)
self.node_hash_ring[new_node_hash] = node_id
# Update pload
self.ploads[node_id] -= agg_pload
self.ploads[migration_dst] += agg_pload
return [
MemcacheKVRequest.MigrationRequest(
keys=migration_keys, dst=self.cache_nodes[migration_dst])
]
def key_to_nodes(self, key, op_type):
if op_type == kv.Operation.Type.PUT or op_type == kv.Operation.Type.DEL:
if op_type == kv.Operation.Type.PUT:
self.install_key(key)
elif op_type == kv.Operation.Type.DEL:
self.remove_key(key)
node_id = self.lookup_node(key)
return MappedNodes([self.cache_nodes[node_id]], None)
else:
node_id = self.lookup_node(key)
total_iload = sum(self.iloads.values())
expected_iload = (self.c * total_iload) / len(self.cache_nodes)
total_pload = sum(self.ploads.values())
expected_pload = (self.c * total_pload) / len(self.cache_nodes)
if self.iloads[node_id] <= expected_iload or self.ploads[
node_id] <= expected_pload:
return MappedNodes([self.cache_nodes[node_id]], None)
pload_diff = self.ploads[node_id] - expected_pload
migration_requests = self.rehash_node(node_id, pload_diff)
return MappedNodes([self.cache_nodes[node_id]], migration_requests)
def report_op_send(self, node, op, time):
self.iloads[node.id] += 1
node_id = self.lookup_node(op.key)
key_rate = self.key_rates.setdefault(op.key, KeyRate())
old_rate = key_rate.rate()
key_rate.count += 1
key_rate.time = time
self.ploads[node_id] += (key_rate.rate() - old_rate)
def report_op_receive(self, node):
self.iloads[node.id] -= 1
class RegionMap(object):
"""
Mostly used in SimAbstractMemory, RegionMap stores a series of mappings between concrete memory address ranges and
memory regions, like stack frames and heap regions.
"""
def __init__(self, is_stack):
"""
Constructor
:param is_stack: Whether this is a region map for stack frames or not. Different strategies apply for stack
regions.
"""
self.is_stack = is_stack
# A sorted list, which maps stack addresses to region IDs
self._address_to_region_id = SortedDict()
# A dict, which maps region IDs to memory address ranges
self._region_id_to_address = {}
#
# Properties
#
def __repr__(self):
return "RegionMap<%s>" % ("S" if self.is_stack else "H")
@property
def is_empty(self):
return len(self._address_to_region_id) == 0
@property
def stack_base(self):
if not self.is_stack:
raise SimRegionMapError(
'Calling "stack_base" on a non-stack region map.')
return next(self._address_to_region_id.irange(reverse=True))
@property
def region_ids(self):
return self._region_id_to_address.keys()
#
# Public methods
#
@SimStatePlugin.memo
def copy(self, memo): # pylint: disable=unused-argument
r = RegionMap(is_stack=self.is_stack)
# A shallow copy should be enough, since we never modify any RegionDescriptor object in-place
r._address_to_region_id = self._address_to_region_id.copy()
r._region_id_to_address = self._region_id_to_address.copy()
return r
def map(self, absolute_address, region_id, related_function_address=None):
"""
Add a mapping between an absolute address and a region ID. If this is a stack region map, all stack regions
beyond (lower than) this newly added regions will be discarded.
:param absolute_address: An absolute memory address.
:param region_id: ID of the memory region.
:param related_function_address: A related function address, mostly used for stack regions.
"""
if self.is_stack:
# Sanity check
if not region_id.startswith('stack_'):
raise SimRegionMapError(
'Received a non-stack memory ID "%d" in a stack region map'
% region_id)
# Remove all stack regions that are lower than the one to add
while True:
try:
addr = next(
self._address_to_region_id.irange(
maximum=absolute_address, reverse=True))
descriptor = self._address_to_region_id[addr]
# Remove this mapping
del self._address_to_region_id[addr]
# Remove this region ID from the other mapping
del self._region_id_to_address[descriptor.region_id]
except StopIteration:
break
else:
if absolute_address in self._address_to_region_id:
descriptor = self._address_to_region_id[absolute_address]
# Remove this mapping
del self._address_to_region_id[absolute_address]
del self._region_id_to_address[descriptor.region_id]
# Add this new region mapping
desc = RegionDescriptor(
region_id,
absolute_address,
related_function_address=related_function_address)
self._address_to_region_id[absolute_address] = desc
self._region_id_to_address[region_id] = desc
def unmap_by_address(self, absolute_address):
"""
Removes a mapping based on its absolute address.
:param absolute_address: An absolute address
"""
desc = self._address_to_region_id[absolute_address]
del self._address_to_region_id[absolute_address]
del self._region_id_to_address[desc.region_id]
def absolutize(self, region_id, relative_address):
"""
Convert a relative address in some memory region to an absolute address.
:param region_id: The memory region ID
:param relative_address: The relative memory offset in that memory region
:return: An absolute address if converted, or an exception is raised when region id does not
exist.
"""
if region_id == 'global':
# The global region always bases 0
return relative_address
if region_id not in self._region_id_to_address:
raise SimRegionMapError('Non-existent region ID "%s"' % region_id)
base_address = self._region_id_to_address[region_id].base_address
return base_address + relative_address
def relativize(self, absolute_address, target_region_id=None):
"""
Convert an absolute address to the memory offset in a memory region.
Note that if an address belongs to heap region is passed in to a stack region map, it will be converted to an
offset included in the closest stack frame, and vice versa for passing a stack address to a heap region.
Therefore you should only pass in address that belongs to the same category (stack or non-stack) of this region
map.
:param absolute_address: An absolute memory address
:return: A tuple of the closest region ID, the relative offset, and the related function
address.
"""
if target_region_id is None:
if self.is_stack:
# Get the base address of the stack frame it belongs to
base_address = next(
self._address_to_region_id.irange(minimum=absolute_address,
reverse=False))
else:
try:
base_address = next(
self._address_to_region_id.irange(
maximum=absolute_address, reverse=True))
except StopIteration:
# Not found. It belongs to the global region then.
return 'global', absolute_address, None
descriptor = self._address_to_region_id[base_address]
else:
if target_region_id == 'global':
# Just return the absolute address
return 'global', absolute_address, None
if target_region_id not in self._region_id_to_address:
raise SimRegionMapError(
'Trying to relativize to a non-existent region "%s"' %
target_region_id)
descriptor = self._region_id_to_address[target_region_id]
base_address = descriptor.base_address
return descriptor.region_id, absolute_address - base_address, descriptor.related_function_address
class CFBlanket(Analysis):
"""
A Control-Flow Blanket is a representation for storing all instructions, data entries, and bytes of a full program.
"""
def __init__(self, cfg=None):
self._blanket = SortedDict()
self._ffi = cffi.FFI()
if cfg is not None:
self._from_cfg(cfg)
else:
_l.debug("CFG is not specified. Initialize CFBlanket from the knowledge base.")
for func in self.kb.functions.values():
self.add_function(func)
def floor_addr(self, addr):
try:
return next(self._blanket.irange(maximum=addr, reverse=True))
except StopIteration:
raise KeyError(addr)
def floor_item(self, addr):
key = self.floor_addr(addr)
return key, self._blanket[key]
def floor_items(self, addr=None):
if addr is None:
start_addr = None
else:
try:
start_addr = next(self._blanket.irange(maximum=addr, reverse=True))
except StopIteration:
start_addr = addr
for key in self._blanket.irange(minimum=start_addr):
yield key, self._blanket[key]
def ceiling_addr(self, addr):
try:
return next(self._blanket.irange(minimum=addr, reverse=False))
except StopIteration:
raise KeyError(addr)
def ceiling_item(self, addr):
key = self.ceiling_addr(addr)
return key, self._blanket[key]
def __getitem__(self, addr):
return self._blanket[addr]
def add_obj(self, addr, obj):
"""
Adds an object `obj` to the blanket at the specified address `addr`
"""
self._blanket[addr] = obj
def add_function(self, func):
"""
Add a function `func` and all blocks of this function to the blanket.
"""
for block in func.blocks:
self.add_obj(block.addr, block)
def dbg_repr(self):
"""
The debugging representation of this CFBlanket.
:return: The debugging representation of this CFBlanket.
:rtype: str
"""
output = [ ]
for obj in self.project.loader.all_objects:
for section in obj.sections:
if section.memsize == 0:
continue
min_addr, max_addr = section.min_addr, section.max_addr
output.append("### Object %s" % repr(section))
output.append("### Range %#x-%#x" % (min_addr, max_addr))
pos = min_addr
while pos < max_addr:
try:
addr, thing = self.floor_item(pos)
output.append("%#x: %s" % (addr, repr(thing)))
if thing.size == 0: pos += 1
else: pos += thing.size
except KeyError:
pos += 1
output.append("")
return "\n".join(output)
def _from_cfg(self, cfg):
"""
Initialize CFBlanket from a CFG instance.
:param cfg: A CFG instance.
:return: None
"""
# Let's first add all functions first
for func in cfg.kb.functions.values():
self.add_function(func)
self._mark_unknowns()
def _mark_unknowns(self):
"""
Mark all unmapped regions.
:return: None
"""
for obj in self.project.loader.all_objects:
if isinstance(obj, cle.ELF):
# sections?
if obj.sections:
for section in obj.sections:
if not section.memsize or not section.vaddr:
continue
min_addr, max_addr = section.min_addr, section.max_addr
self._mark_unknowns_core(min_addr, max_addr, obj=obj, section=section)
elif obj.segments:
for segment in obj.segments:
if not segment.memsize:
continue
min_addr, max_addr = segment.min_addr, segment.max_addr
self._mark_unknowns_core(min_addr, max_addr, obj=obj, segment=segment)
else:
# is it empty?
_l.warning("Empty ELF object %s.", repr(obj))
elif isinstance(obj, cle.PE):
if obj.sections:
for section in obj.sections:
if not section.memsize:
continue
min_addr, max_addr = section.min_addr, section.max_addr
self._mark_unknowns_core(min_addr, max_addr, obj=obj, section=section)
else:
# is it empty?
_l.warning("Empty PE object %s.", repr(obj))
else:
min_addr, max_addr = obj.min_addr, obj.max_addr
self._mark_unknowns_core(min_addr, max_addr, obj=obj)
def _mark_unknowns_core(self, min_addr, max_addr, obj=None, segment=None, section=None):
try:
addr = self.floor_addr(min_addr)
if addr < min_addr:
raise KeyError
except KeyError:
# there is no other lower address
try:
next_addr = self.ceiling_addr(min_addr)
if next_addr >= max_addr:
raise KeyError
except KeyError:
next_addr = max_addr
size = next_addr - min_addr
if obj is None or isinstance(obj, cle.ExternObject):
bytes_ = None
else:
try:
_l.debug("Loading bytes from object %s, section %s, segmeng %s, addresss %#x.",
obj, section, segment, min_addr)
bytes_ = self.project.loader.memory.load(min_addr, size)
except KeyError:
# The address does not exist
bytes_ = None
self.add_obj(min_addr,
Unknown(min_addr, size, bytes_=bytes_, object_=obj, segment=segment, section=section)
)
addr = min_addr
while addr < max_addr:
last_addr, last_item = self.floor_item(addr)
if last_addr < min_addr:
# impossible
raise Exception('Impossible')
if last_item.size == 0:
# Make sure everything has a non-zero size
last_item_size = 1
else:
last_item_size = last_item.size
end_addr = last_addr + last_item_size
if end_addr < max_addr:
try:
next_addr = self.ceiling_addr(end_addr)
except KeyError:
next_addr = max_addr
if next_addr > end_addr:
# there is a gap
size = next_addr - end_addr
if obj is None or isinstance(obj, cle.ExternObject):
bytes_ = None
else:
try:
_l.debug("Loading bytes from object %s, section %s, segmeng %s, addresss %#x.",
obj, section, segment, next_addr)
bytes_ = self.project.loader.memory.load(next_addr, size)
except KeyError:
# The address does not exist
bytes_ = None
self.add_obj(end_addr,
Unknown(end_addr, size, bytes_=bytes_, object_=obj, segment=segment, section=section)
)
addr = next_addr
else:
addr = max_addr
class KeyedRegion(object):
"""
KeyedRegion keeps a mapping between stack offsets and all variables covering that offset. It assumes no variable in
this region overlap with another variable in this region.
Registers and function frames can all be viewed as a keyed region.
"""
def __init__(self, tree=None):
self._storage = SortedDict() if tree is None else tree
def _get_container(self, offset):
try:
base_offset = next(
self._storage.irange(maximum=offset, reverse=True))
except StopIteration:
return offset, None
else:
container = self._storage[base_offset]
if container.includes(offset):
return base_offset, container
return offset, None
def __contains__(self, offset):
"""
Test if there is at least one varaible covering the given offset.
:param offset:
:return:
"""
return self._get_container(offset)[1] is not None
def __len__(self):
return len(self._storage)
def __iter__(self):
return self._storage.itervalues()
def __eq__(self, other):
if set(self._storage.keys()) != set(other._storage.keys()):
return False
for k, v in self._storage.iteritems():
if v != other._storage[k]:
return False
return True
def copy(self):
if not self._storage:
return KeyedRegion()
kr = KeyedRegion()
for key, ro in self._storage.iteritems():
kr._storage[key] = ro.copy()
return kr
def merge(self, other, make_phi_func=None):
"""
Merge another KeyedRegion into this KeyedRegion.
:param KeyedRegion other: The other instance to merge with.
:return: None
"""
# TODO: is the current solution not optimal enough?
for _, item in other._storage.iteritems(): # type: RegionObject
for loc_and_var in item.objects:
self.__store(loc_and_var,
overwrite=False,
make_phi_func=make_phi_func)
return self
def dbg_repr(self):
"""
Get a debugging representation of this keyed region.
:return: A string of debugging output.
"""
keys = self._storage.keys()
offset_to_vars = {}
for key in sorted(keys):
ro = self._storage[key]
variables = [obj.variable for obj in ro.objects]
offset_to_vars[key] = variables
s = []
for offset, variables in offset_to_vars.iteritems():
s.append("Offset %#x: %s" % (offset, variables))
return "\n".join(s)
def add_variable(self, start, variable):
"""
Add a variable to this region at the given offset.
:param int start:
:param SimVariable variable:
:return: None
"""
self._store(start, variable, overwrite=False)
def set_variable(self, start, variable):
"""
Add a variable to this region at the given offset, and remove all other variables that are fully covered by
this variable.
:param int start:
:param SimVariable variable:
:return: None
"""
self._store(start, variable, overwrite=True)
def get_base_addr(self, addr):
"""
Get the base offset (the key we are using to index variables covering the given offset) of a specific offset.
:param int addr:
:return:
:rtype: int or None
"""
base_addr, container = self._get_container(addr)
if container is None:
return None
else:
return base_addr
def get_variables_by_offset(self, start):
"""
Find variables covering the given region offset.
:param int start:
:return: A list of stack variables.
:rtype: set
"""
base_addr, container = self._get_container(start)
if container is None:
return []
else:
return container.variables
#
# Private methods
#
def _store(self, start, variable, overwrite=False):
"""
Store a variable into the storage.
:param int start: The beginning address of the variable.
:param variable: The variable to store.
:param bool overwrite: Whether existing variables should be overwritten or not.
:return: None
"""
loc_and_var = LocationAndVariable(start, variable)
self.__store(loc_and_var, overwrite=overwrite)
def __store(self, loc_and_var, overwrite=False, make_phi_func=None):
"""
Store a variable into the storage.
:param LocationAndVariable loc_and_var: The descriptor describing start address and the variable.
:param bool overwrite: Whether existing variables should be overwritten or not.
:return: None
"""
start = loc_and_var.start
variable = loc_and_var.variable
variable_size = variable.size if variable.size is not None else 1
end = start + variable_size
# region items in the middle
overlapping_items = list(self._storage.irange(start, end - 1))
# is there a region item that begins before the start and overlaps with this variable?
floor_key, floor_item = self._get_container(start)
if floor_item is not None and floor_key not in overlapping_items:
# insert it into the beginningq
overlapping_items.insert(0, (floor_key, self._storage[floor_key]))
# scan through the entire list of region items, split existing regions and insert new regions as needed
to_update = {start: RegionObject(start, variable_size, {loc_and_var})}
last_end = start
for floor_key in overlapping_items:
item = self._storage[floor_key]
if item.start < start:
# we need to break this item into two
a, b = item.split(start)
if overwrite:
b.set_object(loc_and_var)
else:
self._add_object_or_make_phi(b,
loc_and_var,
make_phi_func=make_phi_func)
to_update[a.start] = a
to_update[b.start] = b
last_end = b.end
elif item.start > last_end:
# there is a gap between the last item and the current item
# fill in the gap
new_item = RegionObject(last_end, item.start - last_end,
{loc_and_var})
to_update[new_item.start] = new_item
last_end = new_item.end
elif item.end > end:
# we need to split this item into two
a, b = item.split(end)
if overwrite:
a.set_object(loc_and_var)
else:
self._add_object_or_make_phi(a,
loc_and_var,
make_phi_func=make_phi_func)
to_update[a.start] = a
to_update[b.start] = b
last_end = b.end
else:
if overwrite:
item.set_object(loc_and_var)
else:
self._add_object_or_make_phi(item,
loc_and_var,
make_phi_func=make_phi_func)
to_update[loc_and_var.start] = item
self._storage.update(to_update)
def _is_overlapping(self, start, variable):
if variable.size is not None:
# make sure this variable does not overlap with any other variable
end = start + variable.size
try:
prev_offset = next(
self._storage.irange(maximum=end - 1, reverse=True))
except StopIteration:
prev_offset = None
if prev_offset is not None:
if start <= prev_offset < end:
return True
prev_item = self._storage[prev_offset][0]
prev_item_size = prev_item.size if prev_item.size is not None else 1
if start < prev_offset + prev_item_size < end:
return True
else:
try:
prev_offset = next(
self._storage.irange(maximum=start, reverse=True))
except StopIteration:
prev_offset = None
if prev_offset is not None:
prev_item = self._storage[prev_offset][0]
prev_item_size = prev_item.size if prev_item.size is not None else 1
if prev_offset <= start < prev_offset + prev_item_size:
return True
return False
def _add_object_or_make_phi(self, item, loc_and_var, make_phi_func=None): #pylint:disable=no-self-use
if not make_phi_func or len({loc_and_var.variable}
| item.variables) == 1:
item.add_object(loc_and_var)
else:
# make a phi node
item.set_object(
LocationAndVariable(
loc_and_var.start,
make_phi_func(loc_and_var.variable, *item.variables)))
class KeyedRegion:
"""
KeyedRegion keeps a mapping between stack offsets and all objects covering that offset. It assumes no variable in
this region overlap with another variable in this region.
Registers and function frames can all be viewed as a keyed region.
"""
__slots__ = ('_storage', '_object_mapping', '_phi_node_contains' )
def __init__(self, tree=None, phi_node_contains=None):
self._storage = SortedDict() if tree is None else tree
self._object_mapping = weakref.WeakValueDictionary()
self._phi_node_contains = phi_node_contains
def __getstate__(self):
return self._storage, dict(self._object_mapping), self._phi_node_contains
def __setstate__(self, s):
self._storage, om, self._phi_node_contains = s
self._object_mapping = weakref.WeakValueDictionary(om)
def _get_container(self, offset):
try:
base_offset = next(self._storage.irange(maximum=offset, reverse=True))
except StopIteration:
return offset, None
else:
container = self._storage[base_offset]
if container.includes(offset):
return base_offset, container
return offset, None
def __contains__(self, offset):
"""
Test if there is at least one variable covering the given offset.
:param offset:
:return:
"""
if type(offset) is not int:
raise TypeError("KeyedRegion only accepts concrete offsets.")
return self._get_container(offset)[1] is not None
def __len__(self):
return len(self._storage)
def __iter__(self):
return iter(self._storage.values())
def __eq__(self, other):
if set(self._storage.keys()) != set(other._storage.keys()):
return False
for k, v in self._storage.items():
if v != other._storage[k]:
return False
return True
def copy(self):
if not self._storage:
return KeyedRegion(phi_node_contains=self._phi_node_contains)
kr = KeyedRegion(phi_node_contains=self._phi_node_contains)
for key, ro in self._storage.items():
kr._storage[key] = ro.copy()
kr._object_mapping = self._object_mapping.copy()
return kr
def merge(self, other, replacements=None):
"""
Merge another KeyedRegion into this KeyedRegion.
:param KeyedRegion other: The other instance to merge with.
:return: None
"""
# TODO: is the current solution not optimal enough?
for _, item in other._storage.items(): # type: RegionObject
for so in item.stored_objects: # type: StoredObject
if replacements and so.obj in replacements:
so = StoredObject(so.start, replacements[so.obj], so.size)
self._object_mapping[so.obj_id] = so
self.__store(so, overwrite=False)
return self
def replace(self, replacements):
"""
Replace variables with other variables.
:param dict replacements: A dict of variable replacements.
:return: self
"""
for old_var, new_var in replacements.items():
old_var_id = id(old_var)
if old_var_id in self._object_mapping:
# FIXME: we need to check if old_var still exists in the storage
old_so = self._object_mapping[old_var_id] # type: StoredObject
self._store(old_so.start, new_var, old_so.size, overwrite=True)
return self
def dbg_repr(self):
"""
Get a debugging representation of this keyed region.
:return: A string of debugging output.
"""
keys = self._storage.keys()
offset_to_vars = { }
for key in sorted(keys):
ro = self._storage[key]
variables = [ obj.obj for obj in ro.stored_objects ]
offset_to_vars[key] = variables
s = [ ]
for offset, variables in offset_to_vars.items():
s.append("Offset %#x: %s" % (offset, variables))
return "\n".join(s)
def add_variable(self, start, variable):
"""
Add a variable to this region at the given offset.
:param int start:
:param SimVariable variable:
:return: None
"""
size = variable.size if variable.size is not None else 1
self.add_object(start, variable, size)
def add_object(self, start, obj, object_size):
"""
Add/Store an object to this region at the given offset.
:param start:
:param obj:
:param int object_size: Size of the object
:return:
"""
self._store(start, obj, object_size, overwrite=False)
def set_variable(self, start, variable):
"""
Add a variable to this region at the given offset, and remove all other variables that are fully covered by
this variable.
:param int start:
:param SimVariable variable:
:return: None
"""
size = variable.size if variable.size is not None else 1
self.set_object(start, variable, size)
def set_object(self, start, obj, object_size):
"""
Add an object to this region at the given offset, and remove all other objects that are fully covered by this
object.
:param start:
:param obj:
:param object_size:
:return:
"""
self._store(start, obj, object_size, overwrite=True)
def get_base_addr(self, addr):
"""
Get the base offset (the key we are using to index objects covering the given offset) of a specific offset.
:param int addr:
:return:
:rtype: int or None
"""
base_addr, container = self._get_container(addr)
if container is None:
return None
else:
return base_addr
def get_variables_by_offset(self, start):
"""
Find variables covering the given region offset.
:param int start:
:return: A list of stack variables.
:rtype: set
"""
_, container = self._get_container(start)
if container is None:
return []
else:
return container.internal_objects
def get_objects_by_offset(self, start):
"""
Find objects covering the given region offset.
:param start:
:return:
"""
_, container = self._get_container(start)
if container is None:
return set()
else:
return container.internal_objects
#
# Private methods
#
def _store(self, start, obj, size, overwrite=False):
"""
Store a variable into the storage.
:param int start: The beginning address of the variable.
:param obj: The object to store.
:param int size: Size of the object to store.
:param bool overwrite: Whether existing objects should be overwritten or not.
:return: None
"""
stored_object = StoredObject(start, obj, size)
self._object_mapping[stored_object.obj_id] = stored_object
self.__store(stored_object, overwrite=overwrite)
def __store(self, stored_object, overwrite=False):
"""
Store a variable into the storage.
:param StoredObject stored_object: The descriptor describing start address and the variable.
:param bool overwrite: Whether existing objects should be overwritten or not. True to make a strong update,
False to make a weak update.
:return: None
"""
start = stored_object.start
object_size = stored_object.size
end = start + object_size
# region items in the middle
overlapping_items = list(self._storage.irange(start, end-1))
# is there a region item that begins before the start and overlaps with this variable?
floor_key, floor_item = self._get_container(start)
if floor_item is not None and floor_key not in overlapping_items:
# insert it into the beginning
overlapping_items.insert(0, floor_key)
# scan through the entire list of region items, split existing regions and insert new regions as needed
to_update = {start: RegionObject(start, object_size, {stored_object})}
last_end = start
for floor_key in overlapping_items:
item = self._storage[floor_key]
if item.start < start:
# we need to break this item into two
a, b = item.split(start)
if overwrite:
b.set_object(stored_object)
else:
self._add_object_with_check(b, stored_object)
to_update[a.start] = a
to_update[b.start] = b
last_end = b.end
elif item.start > last_end:
# there is a gap between the last item and the current item
# fill in the gap
new_item = RegionObject(last_end, item.start - last_end, {stored_object})
to_update[new_item.start] = new_item
last_end = new_item.end
elif item.end > end:
# we need to split this item into two
a, b = item.split(end)
if overwrite:
a.set_object(stored_object)
else:
self._add_object_with_check(a, stored_object)
to_update[a.start] = a
to_update[b.start] = b
last_end = b.end
else:
if overwrite:
item.set_object(stored_object)
else:
self._add_object_with_check(item, stored_object)
to_update[item.start] = item
self._storage.update(to_update)
def _is_overlapping(self, start, variable):
if variable.size is not None:
# make sure this variable does not overlap with any other variable
end = start + variable.size
try:
prev_offset = next(self._storage.irange(maximum=end-1, reverse=True))
except StopIteration:
prev_offset = None
if prev_offset is not None:
if start <= prev_offset < end:
return True
prev_item = self._storage[prev_offset][0]
prev_item_size = prev_item.size if prev_item.size is not None else 1
if start < prev_offset + prev_item_size < end:
return True
else:
try:
prev_offset = next(self._storage.irange(maximum=start, reverse=True))
except StopIteration:
prev_offset = None
if prev_offset is not None:
prev_item = self._storage[prev_offset][0]
prev_item_size = prev_item.size if prev_item.size is not None else 1
if prev_offset <= start < prev_offset + prev_item_size:
return True
return False
def _add_object_with_check(self, item, stored_object):
if len({stored_object.obj} | item.internal_objects) > 1:
if self._phi_node_contains is not None:
# check if `item` is a phi node that contains stored_object.obj
for so in item.internal_objects:
if self._phi_node_contains(so, stored_object.obj):
# yes! so we want to skip this object
return
# check if `stored_object.obj` is a phi node that contains item.internal_objects
if all(self._phi_node_contains(stored_object.obj, o) for o in item.internal_objects):
# yes!
item.set_object(stored_object)
return
l.warning("Overlapping objects %s.", str({stored_object.obj} | item.internal_objects))
# import ipdb; ipdb.set_trace()
item.add_object(stored_object)
class CFBlanket(Analysis):
"""
A Control-Flow Blanket is a representation for storing all instructions, data entries, and bytes of a full program.
"""
def __init__(self, cfg=None):
self._blanket = SortedDict()
self._regions = []
self._init_regions()
if cfg is not None:
self._from_cfg(cfg)
else:
_l.debug(
"CFG is not specified. Initialize CFBlanket from the knowledge base."
)
for func in self.kb.functions.values():
self.add_function(func)
def _init_regions(self):
for obj in self.project.loader.all_objects:
if isinstance(obj, cle.MetaELF):
if obj.sections:
# Enumerate sections in an ELF file
for section in obj.sections:
if section.occupies_memory:
mr = MemoryRegion(section.vaddr, section.memsize,
'TODO', obj, section)
self._regions.append(mr)
else:
raise NotImplementedError()
else:
mr = MemoryRegion(obj.min_addr,
obj.size if hasattr(obj, 'size') else 80,
'TODO', obj, None)
self._regions.append(mr)
# Sort them just in case
self._regions = list(sorted(self._regions, key=lambda x: x.addr))
@property
def regions(self):
"""
Return all memory regions.
"""
return self._regions
def floor_addr(self, addr):
try:
return next(self._blanket.irange(maximum=addr, reverse=True))
except StopIteration:
raise KeyError(addr)
def floor_item(self, addr):
key = self.floor_addr(addr)
return key, self._blanket[key]
def floor_items(self, addr=None, reverse=False):
if addr is None:
start_addr = None
else:
try:
start_addr = next(
self._blanket.irange(maximum=addr, reverse=True))
except StopIteration:
start_addr = addr
for key in self._blanket.irange(minimum=start_addr, reverse=reverse):
yield key, self._blanket[key]
def ceiling_addr(self, addr):
try:
return next(self._blanket.irange(minimum=addr))
except StopIteration:
raise KeyError(addr)
def ceiling_item(self, addr):
key = self.ceiling_addr(addr)
return key, self._blanket[key]
def ceiling_items(self, addr=None, reverse=False, include_first=True):
if addr is None:
start_addr = None
else:
try:
start_addr = next(self._blanket.irange(minimum=addr))
except StopIteration:
start_addr = addr
for key in self._blanket.irange(
maximum=start_addr if include_first else start_addr - 1,
reverse=reverse):
yield key, self._blanket[key]
def __getitem__(self, addr):
return self._blanket[addr]
def add_obj(self, addr, obj):
"""
Adds an object `obj` to the blanket at the specified address `addr`
"""
self._blanket[addr] = obj
def add_function(self, func):
"""
Add a function `func` and all blocks of this function to the blanket.
"""
for block in func.blocks:
self.add_obj(block.addr, block)
def dbg_repr(self):
"""
The debugging representation of this CFBlanket.
:return: The debugging representation of this CFBlanket.
:rtype: str
"""
output = []
for obj in self.project.loader.all_objects:
for section in obj.sections:
if section.memsize == 0:
continue
min_addr, max_addr = section.min_addr, section.max_addr
output.append("### Object %s" % repr(section))
output.append("### Range %#x-%#x" % (min_addr, max_addr))
pos = min_addr
while pos < max_addr:
try:
addr, thing = self.floor_item(pos)
output.append("%#x: %s" % (addr, repr(thing)))
if thing.size == 0: pos += 1
else: pos += thing.size
except KeyError:
pos += 1
output.append("")
return "\n".join(output)
def _from_cfg(self, cfg):
"""
Initialize CFBlanket from a CFG instance.
:param cfg: A CFG instance.
:return: None
"""
# Let's first add all functions first
for func in cfg.kb.functions.values():
self.add_function(func)
self._mark_unknowns()
def _mark_unknowns(self):
"""
Mark all unmapped regions.
:return: None
"""
for obj in self.project.loader.all_objects:
if isinstance(obj, cle.ELF):
# sections?
if obj.sections:
for section in obj.sections:
if not section.memsize or not section.vaddr:
continue
min_addr, max_addr = section.min_addr, section.max_addr
self._mark_unknowns_core(min_addr,
max_addr + 1,
obj=obj,
section=section)
elif obj.segments:
for segment in obj.segments:
if not segment.memsize:
continue
min_addr, max_addr = segment.min_addr, segment.max_addr
self._mark_unknowns_core(min_addr,
max_addr + 1,
obj=obj,
segment=segment)
else:
# is it empty?
_l.warning("Empty ELF object %s.", repr(obj))
elif isinstance(obj, cle.PE):
if obj.sections:
for section in obj.sections:
if not section.memsize:
continue
min_addr, max_addr = section.min_addr, section.max_addr
self._mark_unknowns_core(min_addr,
max_addr + 1,
obj=obj,
section=section)
else:
# is it empty?
_l.warning("Empty PE object %s.", repr(obj))
else:
min_addr, max_addr = obj.min_addr, obj.max_addr
self._mark_unknowns_core(min_addr, max_addr + 1, obj=obj)
def _mark_unknowns_core(self,
min_addr,
max_addr,
obj=None,
segment=None,
section=None):
# The region should be [min_addr, max_addr)
try:
addr = self.floor_addr(min_addr)
if addr < min_addr:
raise KeyError
except KeyError:
# there is no other lower address
try:
next_addr = self.ceiling_addr(min_addr)
if next_addr >= max_addr:
raise KeyError
except KeyError:
next_addr = max_addr
size = next_addr - min_addr
if obj is None or isinstance(obj, cle.ExternObject):
bytes_ = None
else:
try:
_l.debug(
"Loading bytes from object %s, section %s, segmeng %s, addresss %#x.",
obj, section, segment, min_addr)
bytes_ = self.project.loader.memory.load(min_addr, size)
except KeyError:
# The address does not exist
bytes_ = None
self.add_obj(
min_addr,
Unknown(min_addr,
size,
bytes_=bytes_,
object_=obj,
segment=segment,
section=section))
addr = min_addr
while addr < max_addr:
last_addr, last_item = self.floor_item(addr)
if last_addr < min_addr:
# impossible
raise Exception('Impossible')
if last_item.size == 0:
# Make sure everything has a non-zero size
last_item_size = 1
else:
last_item_size = last_item.size
end_addr = last_addr + last_item_size
if end_addr < max_addr:
try:
next_addr = self.ceiling_addr(end_addr)
except KeyError:
next_addr = max_addr
if next_addr > end_addr:
# there is a gap
size = next_addr - end_addr
if obj is None or isinstance(obj, cle.ExternObject):
bytes_ = None
else:
try:
_l.debug(
"Loading bytes from object %s, section %s, segmeng %s, addresss %#x.",
obj, section, segment, next_addr)
bytes_ = self.project.loader.memory.load(
next_addr, size)
except KeyError:
# The address does not exist
bytes_ = None
self.add_obj(
end_addr,
Unknown(end_addr,
size,
bytes_=bytes_,
object_=obj,
segment=segment,
section=section))
addr = next_addr
else:
addr = max_addr
class CFBlanket(Analysis):
"""
A Control-Flow Blanket is a representation for storing all instructions, data entries, and bytes of a full program.
Region types:
- section
- segment
- extern
- tls
- kernel
"""
def __init__(self, exclude_region_types=None):
self._blanket = SortedDict()
self._regions = []
self._exclude_region_types = set(
) if not exclude_region_types else exclude_region_types
self._init_regions()
# initialize
for func in self.kb.functions.values():
self.add_function(func)
self._mark_memory_data()
self._mark_unknowns()
def _init_regions(self):
for obj in self.project.loader.all_objects:
if isinstance(obj, cle.MetaELF):
if obj.sections:
if "section" not in self._exclude_region_types:
# Enumerate sections in an ELF file
for section in obj.sections:
if section.occupies_memory:
mr = MemoryRegion(section.vaddr,
section.memsize, 'section',
obj, section)
self._regions.append(mr)
else:
raise NotImplementedError(
"Currently ELFs without sections are not supported. Please implement or "
"complain on GitHub.")
elif isinstance(obj, cle.PE):
if obj.sections:
if "section" not in self._exclude_region_types:
for section in obj.sections:
mr = MemoryRegion(section.vaddr, section.memsize,
'section', obj, section)
self._regions.append(mr)
else:
raise NotImplementedError(
"Currently PEs without sections are not supported. Please report to "
"GitHub and provide an example binary.")
elif isinstance(obj, KernelObject):
if "kernel" not in self._exclude_region_types:
size = obj.max_addr - obj.min_addr
mr = MemoryRegion(obj.min_addr, size, "kernel", obj, None)
self._regions.append(mr)
elif isinstance(obj, ExternObject):
if "extern" not in self._exclude_region_types:
size = obj.max_addr - obj.min_addr
mr = MemoryRegion(obj.min_addr, size, "extern", obj, None)
self._regions.append(mr)
elif isinstance(obj, ELFTLSObject):
if "tls" not in self._exclude_region_types:
size = obj.max_addr - obj.min_addr
mr = MemoryRegion(obj.min_addr, size, "tls", obj, None)
self._regions.append(mr)
else:
if hasattr(obj, "size"):
size = obj.size
else:
size = obj.max_addr - obj.min_addr
type_ = "TODO"
mr = MemoryRegion(obj.min_addr, size, type_, obj, obj)
self._regions.append(mr)
# Sort them just in case
self._regions = list(sorted(self._regions, key=lambda x: x.addr))
@property
def regions(self):
"""
Return all memory regions.
"""
return self._regions
def floor_addr(self, addr):
try:
return next(self._blanket.irange(maximum=addr, reverse=True))
except StopIteration:
raise KeyError(addr)
def floor_item(self, addr):
key = self.floor_addr(addr)
return key, self._blanket[key]
def floor_items(self, addr=None, reverse=False):
if addr is None:
start_addr = None
else:
try:
start_addr = next(
self._blanket.irange(maximum=addr, reverse=True))
except StopIteration:
start_addr = addr
for key in self._blanket.irange(minimum=start_addr, reverse=reverse):
yield key, self._blanket[key]
def ceiling_addr(self, addr):
try:
return next(self._blanket.irange(minimum=addr))
except StopIteration:
raise KeyError(addr)
def ceiling_item(self, addr):
key = self.ceiling_addr(addr)
return key, self._blanket[key]
def ceiling_items(self, addr=None, reverse=False, include_first=True):
if addr is None:
start_addr = None
else:
try:
start_addr = next(self._blanket.irange(minimum=addr))
except StopIteration:
start_addr = addr
for key in self._blanket.irange(
maximum=start_addr if include_first else start_addr - 1,
reverse=reverse):
yield key, self._blanket[key]
def __getitem__(self, addr):
return self._blanket[addr]
def add_obj(self, addr, obj):
"""
Adds an object `obj` to the blanket at the specified address `addr`
"""
self._blanket[addr] = obj
def add_function(self, func):
"""
Add a function `func` and all blocks of this function to the blanket.
"""
for block in func.blocks:
self.add_obj(block.addr, block)
def dbg_repr(self):
"""
The debugging representation of this CFBlanket.
:return: The debugging representation of this CFBlanket.
:rtype: str
"""
output = []
for obj in self.project.loader.all_objects:
for section in obj.sections:
if section.memsize == 0:
continue
min_addr, max_addr = section.min_addr, section.max_addr
output.append("### Object %s" % repr(section))
output.append("### Range %#x-%#x" % (min_addr, max_addr))
pos = min_addr
while pos < max_addr:
try:
addr, thing = self.floor_item(pos)
output.append("%#x: %s" % (addr, repr(thing)))
if thing.size == 0: pos += 1
else: pos += thing.size
except KeyError:
pos += 1
output.append("")
return "\n".join(output)
def _mark_memory_data(self):
"""
Mark all memory data.
:return: None
"""
if 'CFGFast' not in self.kb.cfgs:
return
cfg_model = self.kb.cfgs['CFGFast']
for addr, memory_data in cfg_model.memory_data.items():
memory_data: MemoryData
if memory_data.sort == MemoryDataSort.CodeReference:
# skip Code Reference
continue
self.add_obj(addr, memory_data)
def _mark_unknowns(self):
"""
Mark all unmapped regions.
:return: None
"""
for obj in self.project.loader.all_objects:
if isinstance(obj, cle.ELF):
# sections?
if obj.sections and "section" not in self._exclude_region_types:
for section in obj.sections:
if not section.memsize or not section.vaddr:
continue
min_addr, max_addr = section.min_addr, section.max_addr
self._mark_unknowns_core(min_addr,
max_addr + 1,
obj=obj,
section=section)
elif obj.segments and "segment" not in self._exclude_region_types:
for segment in obj.segments:
if not segment.memsize:
continue
min_addr, max_addr = segment.min_addr, segment.max_addr
self._mark_unknowns_core(min_addr,
max_addr + 1,
obj=obj,
segment=segment)
else:
# is it empty?
_l.warning("Empty ELF object %s.", repr(obj))
elif isinstance(obj, cle.PE):
if obj.sections:
for section in obj.sections:
if not section.memsize:
continue
min_addr, max_addr = section.min_addr, section.max_addr
self._mark_unknowns_core(min_addr,
max_addr + 1,
obj=obj,
section=section)
else:
# is it empty?
_l.warning("Empty PE object %s.", repr(obj))
elif isinstance(obj, ELFTLSObject):
if "tls" in self._exclude_region_types:
# Skip them for now
pass
else:
min_addr, max_addr = obj.min_addr, obj.max_addr
self._mark_unknowns_core(min_addr, max_addr + 1, obj=obj)
elif isinstance(obj, KernelObject):
if "kernel" in self._exclude_region_types:
# skip
pass
else:
min_addr, max_addr = obj.min_addr, obj.max_addr
self._mark_unknowns_core(min_addr, max_addr + 1, obj=obj)
else:
min_addr, max_addr = obj.min_addr, obj.max_addr
self._mark_unknowns_core(min_addr, max_addr + 1, obj=obj)
def _mark_unknowns_core(self,
min_addr,
max_addr,
obj=None,
segment=None,
section=None):
# The region should be [min_addr, max_addr)
try:
addr = self.floor_addr(min_addr)
if addr < min_addr:
raise KeyError
except KeyError:
# there is no other lower address
try:
next_addr = self.ceiling_addr(min_addr)
if next_addr >= max_addr:
raise KeyError
except KeyError:
next_addr = max_addr
size = next_addr - min_addr
if obj is None or isinstance(obj, cle.ExternObject):
bytes_ = None
else:
try:
_l.debug(
"Loading bytes from object %s, section %s, segmeng %s, addresss %#x.",
obj, section, segment, min_addr)
bytes_ = self.project.loader.memory.load(min_addr, size)
except KeyError:
# The address does not exist
bytes_ = None
self.add_obj(
min_addr,
Unknown(min_addr,
size,
bytes_=bytes_,
object_=obj,
segment=segment,
section=section))
addr = min_addr
while addr < max_addr:
last_addr, last_item = self.floor_item(addr)
if last_addr < min_addr:
# impossible
raise Exception('Impossible')
if last_item.size == 0:
# Make sure everything has a non-zero size
last_item_size = 1
else:
last_item_size = last_item.size
end_addr = last_addr + last_item_size
if end_addr < max_addr:
try:
next_addr = self.ceiling_addr(end_addr)
except KeyError:
next_addr = max_addr
if next_addr > end_addr:
# there is a gap
size = next_addr - end_addr
if obj is None or isinstance(obj, cle.ExternObject):
bytes_ = None
else:
try:
_l.debug(
"Loading bytes from object %s, section %s, segmeng %s, addresss %#x.",
obj, section, segment, next_addr)
bytes_ = self.project.loader.memory.load(
next_addr, size)
except KeyError:
# The address does not exist
bytes_ = None
self.add_obj(
end_addr,
Unknown(end_addr,
size,
bytes_=bytes_,
object_=obj,
segment=segment,
section=section))
addr = next_addr
else:
addr = max_addr
class TreePage(BasePage):
"""
Page object, implemented with a sorted dict. Who knows what's underneath!
"""
def __init__(self, *args, **kwargs):
storage = kwargs.pop("storage", None)
super(TreePage, self).__init__(*args, **kwargs)
self._storage = SortedDict() if storage is None else storage
def keys(self):
if len(self._storage) == 0:
return set()
else:
return set.union(*(set(range(*self._resolve_range(mo))) for mo in self._storage.itervalues()))
def replace_mo(self, state, old_mo, new_mo):
start, end = self._resolve_range(old_mo)
for key in self._storage.irange(start, end-1):
val = self._storage[key]
if val is old_mo:
#assert new_mo.includes(a)
self._storage[key] = new_mo
def store_overwrite(self, state, new_mo, start, end):
# iterate over each item we might overwrite
# track our mutations separately since we're in the process of iterating
deletes = []
updates = { start: new_mo }
for key in self._storage.irange(maximum=end-1, reverse=True):
old_mo = self._storage[key]
# make sure we aren't overwriting all of an item that overlaps the end boundary
if end < self._page_addr + self._page_size and end not in updates and old_mo.includes(end):
updates[end] = old_mo
# we can't set a minimum on the range because we need to do the above for
# the first object before start too
if key < start:
break
# delete any key that falls within the range
deletes.append(key)
#assert all(m.includes(i) for i,m in updates.items())
# perform mutations
for key in deletes:
del self._storage[key]
self._storage.update(updates)
def store_underwrite(self, state, new_mo, start, end):
# track the point that we need to write up to
last_missing = end - 1
# track also updates since we can't update while iterating
updates = {}
for key in self._storage.irange(maximum=end-1, reverse=True):
mo = self._storage[key]
# if the mo stops
if mo.base <= last_missing and not mo.includes(last_missing):
updates[max(mo.last_addr+1, start)] = new_mo
last_missing = mo.base - 1
# we can't set a minimum on the range because we need to do the above for
# the first object before start too
if last_missing < start:
break
# if there are no memory objects <= start, we won't have filled start yet
if last_missing >= start:
updates[start] = new_mo
#assert all(m.includes(i) for i,m in updates.items())
self._storage.update(updates)
def load_mo(self, state, page_idx):
"""
Loads a memory object from memory.
:param page_idx: the index into the page
:returns: a tuple of the object
"""
try:
key = next(self._storage.irange(maximum=page_idx, reverse=True))
except StopIteration:
return None
else:
return self._storage[key]
def load_slice(self, state, start, end):
"""
Return the memory objects overlapping with the provided slice.
:param start: the start address
:param end: the end address (non-inclusive)
:returns: tuples of (starting_addr, memory_object)
"""
keys = list(self._storage.irange(start, end-1))
if not keys or keys[0] != start:
try:
key = next(self._storage.irange(maximum=start, reverse=True))
except StopIteration:
pass
else:
if self._storage[key].includes(start):
items.insert(0, key)
return [(key, self._storage[key]) for key in keys]
def _copy_args(self):
return { 'storage': self._storage.copy() }
class QLinearDisassembly(QDisassemblyBaseControl, QAbstractScrollArea):
OBJECT_PADDING = 0
def __init__(self, workspace, disasm_view, parent=None):
QDisassemblyBaseControl.__init__(self, workspace, disasm_view, QAbstractScrollArea)
QAbstractScrollArea.__init__(self, parent=parent)
self.setHorizontalScrollBarPolicy(Qt.ScrollBarAlwaysOn)
self.setVerticalScrollBarPolicy(Qt.ScrollBarAlwaysOn)
self.horizontalScrollBar().setSingleStep(Conf.disasm_font_width)
self.verticalScrollBar().setSingleStep(16)
# self.setTransformationAnchor(QGraphicsView.NoAnchor)
# self.setResizeAnchor(QGraphicsView.NoAnchor)
# self.setAlignment(Qt.AlignLeft)
self._viewer = None # type: QLinearDisassemblyView
self._line_height = Conf.disasm_font_height
self._offset_to_region = SortedDict()
self._addr_to_region_offset = SortedDict()
# Offset (in bytes) into the entire blanket view
self._offset = 0
# The maximum offset (in bytes) of the blanket view
self._max_offset = None
# The first line that is rendered of the first object in self.objects. Start from 0.
self._start_line_in_object = 0
self._disasms = { }
self.objects = [ ]
self.verticalScrollBar().actionTriggered.connect(self._on_vertical_scroll_bar_triggered)
self._init_widgets()
def reload(self):
self.initialize()
#
# Properties
#
@property
def offset(self):
return self._offset
@offset.setter
def offset(self, v):
self._offset = v
@property
def max_offset(self):
if self._max_offset is None:
self._max_offset = self._calculate_max_offset()
return self._max_offset
@property
def cfg(self):
return self.workspace.instance.cfg
@property
def cfb(self):
return self.workspace.instance.cfb
@property
def scene(self):
return self._viewer._scene
#
# Events
#
def resizeEvent(self, event):
old_height = event.oldSize().height()
new_height = event.size().height()
self._viewer._scene.setSceneRect(QRectF(0, 0, event.size().width(), new_height))
if new_height > old_height:
# we probably need more objects generated
curr_offset = self._offset
self._offset = None # force a re-generation of objects
self.prepare_objects(curr_offset, start_line=self._start_line_in_object)
self.redraw()
super().resizeEvent(event)
def wheelEvent(self, event):
"""
:param QWheelEvent event:
:return:
"""
delta = event.delta()
if delta < 0:
# scroll down by some lines
lines = min(int(-delta // self._line_height), 3)
self.prepare_objects(self.offset, start_line=self._start_line_in_object + lines)
elif delta > 0:
# Scroll up by some lines
lines = min(int(delta // self._line_height), 3)
self.prepare_objects(self.offset, start_line=self._start_line_in_object - lines)
self.verticalScrollBar().setValue(self.offset * self._line_height)
event.accept()
self.viewport().update()
def _on_vertical_scroll_bar_triggered(self, action):
if action == QAbstractSlider.SliderSingleStepAdd:
# scroll down by one line
self.prepare_objects(self.offset, start_line=self._start_line_in_object + 1)
self.viewport().update()
elif action == QAbstractSlider.SliderSingleStepSub:
# Scroll up by one line
self.prepare_objects(self.offset, start_line=self._start_line_in_object - 1)
self.viewport().update()
elif action == QAbstractSlider.SliderPageStepAdd:
# Scroll down by one page
lines_per_page = int(self.height() // self._line_height)
self.prepare_objects(self.offset, start_line=self._start_line_in_object
+ lines_per_page)
self.viewport().update()
elif action == QAbstractSlider.SliderPageStepSub:
# Scroll up by one page
lines_per_page = int(self.height() // self._line_height)
self.prepare_objects(self.offset,
start_line=self._start_line_in_object - lines_per_page)
self.viewport().update()
elif action == QAbstractSlider.SliderMove:
# Setting a new offset
new_offset = int(self.verticalScrollBar().value() // self._line_height)
self.prepare_objects(new_offset)
self.viewport().update()
#
# Public methods
#
def redraw(self):
if self._viewer is not None:
self._viewer.redraw()
def refresh(self):
self._update_size()
self.redraw()
def initialize(self):
if self.cfb is None:
return
self._addr_to_region_offset.clear()
self._offset_to_region.clear()
self._disasms.clear()
self._offset = None
self._max_offset = None
self._start_line_in_object = 0
# enumerate memory regions
byte_offset = 0
for mr in self.cfb.regions: # type: MemoryRegion
if mr.type in {'tls', 'kernel'}:
# Skip TLS objects and kernel objects
continue
self._addr_to_region_offset[mr.addr] = byte_offset
self._offset_to_region[byte_offset] = mr
byte_offset += mr.size
self._update_size()
def goto_function(self, func):
if func.addr not in self._block_addr_map:
_l.error('Unable to find entry block for function %s', func)
view_height = self.viewport().height()
desired_center_y = self._block_addr_map[func.addr].pos().y()
_l.debug('Going to function at 0x%x by scrolling to %s', func.addr, desired_center_y)
self.verticalScrollBar().setValue(desired_center_y - (view_height / 3))
def show_instruction(self, insn_addr, insn_pos=None, centering=False, use_block_pos=False):
"""
:param insn_addr:
:param QGraphicsItem item:
:param centering:
:param use_block_pos:
:return:
"""
if insn_pos is not None:
# check if item is already visible in the viewport
viewport = self._viewer.viewport()
rect = self._viewer.mapToScene(QRect(0, 0, viewport.width(), viewport.height())).boundingRect()
if rect.contains(insn_pos):
return
self.navigate_to_addr(insn_addr)
def navigate_to_addr(self, addr):
if not self._addr_to_region_offset:
return
try:
floor_region_addr = next(self._addr_to_region_offset.irange(maximum=addr, reverse=True))
except StopIteration:
floor_region_addr = next(self._addr_to_region_offset.irange())
floor_region_offset = self._addr_to_region_offset[floor_region_addr]
offset_into_region = addr - floor_region_addr
self.navigate_to(floor_region_offset + offset_into_region)
def navigate_to(self, offset):
self.verticalScrollBar().setValue(offset * self._line_height)
self.prepare_objects(offset, start_line=0)
#
# Private methods
#
def _init_widgets(self):
self._viewer = QLinearDisassemblyView(self)
layout = QHBoxLayout()
layout.addWidget(self._viewer)
layout.setContentsMargins(0, 0, 0, 0)
self.setLayout(layout)
def _update_size(self):
# ask all objects to update their sizes
for obj in self.objects:
obj.clear_cache()
obj.refresh()
# update vertical scrollbar
self.verticalScrollBar().setRange(0, self.max_offset * self._line_height - self.height() // 2)
offset = 0 if self.offset is None else self.offset
self.verticalScrollBar().setValue(offset * self._line_height)
def clear_objects(self):
self.objects.clear()
self._offset = None
def prepare_objects(self, offset, start_line=0):
"""
Prepare objects to print based on offset and start_line. Update self.objects, self._offset, and
self._start_line_in_object.
:param int offset: Beginning offset (in bytes) to display in the linear viewer.
:param int start_line: The first line into the first object to display in the linear viewer.
:return: None
"""
if offset is None:
offset = 0
if offset == self._offset and start_line == self._start_line_in_object:
return
# Convert the offset to memory region
base_offset, mr = self._region_from_offset(offset) # type: int, MemoryRegion
if mr is None:
return
addr = self._addr_from_offset(mr, base_offset, offset)
_l.debug("Address %#x, offset %d, start_line %d.", addr, offset, start_line)
self._insaddr_to_block.clear()
if start_line < 0:
# Which object are we currently displaying at the top of the disassembly view?
try:
top_obj_addr = self.cfb.floor_addr(addr=addr)
except KeyError:
top_obj_addr = addr
# Reverse-iterate until we have enough lines to compensate start_line
for obj_addr, obj in self.cfb.ceiling_items(addr=top_obj_addr, reverse=True, include_first=False):
qobject = self._obj_to_paintable(obj_addr, obj)
if qobject is None:
continue
object_lines = int(qobject.height // self._line_height)
_l.debug("Compensating negative start_line: object %s, object_lines %d.", obj, object_lines)
start_line += object_lines
if start_line >= 0:
addr = obj_addr
# Update offset
new_region_addr = next(self._addr_to_region_offset.irange(maximum=addr, reverse=True))
new_region_offset = self._addr_to_region_offset[new_region_addr]
offset = (addr - new_region_addr) + new_region_offset
break
else:
# umm we don't have enough objects to compensate the negative start_line
start_line = 0
# update addr and offset to their minimal values
addr = next(self._addr_to_region_offset.irange())
offset = self._addr_to_region_offset[addr]
_l.debug("After adjustment: Address %#x, offset %d, start_line %d.", addr, offset, start_line)
scene = self.scene
# remove existing objects
for obj in self.objects:
scene.removeItem(obj)
self.objects = [ ]
viewable_lines = int(self.height() // self._line_height)
lines = 0
start_line_in_object = 0
# Load a page of objects
x = 80
y = -start_line * self._line_height
for obj_addr, obj in self.cfb.floor_items(addr=addr):
qobject = self._obj_to_paintable(obj_addr, obj)
_l.debug("Converted %s to %s at %x.", obj, qobject, obj_addr)
if qobject is None:
# Conversion failed
continue
if isinstance(qobject, QLinearBlock):
for insn_addr in qobject.addr_to_insns.keys():
self._insaddr_to_block[insn_addr] = qobject
# qobject.setCacheMode(QGraphicsItem.DeviceCoordinateCache)
object_lines = int(qobject.height // self._line_height)
if start_line >= object_lines:
# this object should be skipped. ignore it
start_line -= object_lines
# adjust the offset as well
if obj_addr <= addr < obj_addr + obj.size:
offset += obj_addr + obj.size - addr
else:
offset += obj.size
_l.debug("Skipping object %s (size %d). New offset: %d.", obj, obj.size, offset)
y = -start_line * self._line_height
else:
if start_line > 0:
_l.debug("First object to paint: %s (size %d). Current offset %d. Start printing from line %d. "
"Y pos %d.", obj, obj.size, offset, start_line, y)
# this is the first object to paint
start_line_in_object = start_line
start_line = 0
lines += object_lines - start_line_in_object
else:
lines += object_lines
self.objects.append(qobject)
qobject.setPos(x, y)
scene.addItem(qobject)
y += qobject.height + self.OBJECT_PADDING
if lines > viewable_lines:
break
_l.debug("Final offset %d, start_line_in_object %d.", offset, start_line_in_object)
# Update properties
self._offset = offset
self._start_line_in_object = start_line_in_object
def _obj_to_paintable(self, obj_addr, obj):
if isinstance(obj, Block):
cfg_node = self.cfg.get_any_node(obj_addr, force_fastpath=True)
func_addr = cfg_node.function_address
if self.workspace.instance.kb.functions.contains_addr(func_addr):
func = self.workspace.instance.kb.functions[func_addr]
disasm = self._get_disasm(func)
qobject = QLinearBlock(self.workspace, func_addr, self.disasm_view, disasm,
self.disasm_view.infodock, obj.addr, [obj], {}, None, container=self._viewer,
)
else:
# TODO: Get disassembly even if the function does not exist
_l.warning("Function %s does not exist, and we cannot get disassembly for block %s.",
func_addr, obj)
qobject = None
elif isinstance(obj, MemoryData):
qobject = QMemoryDataBlock(self.workspace, self.disasm_view.infodock, obj_addr, obj, parent=None,
container=self._viewer)
elif isinstance(obj, Unknown):
qobject = QUnknownBlock(self.workspace, obj_addr, obj.bytes, container=self._viewer)
else:
qobject = None
return qobject
def _calculate_max_offset(self):
try:
max_off = next(self._offset_to_region.irange(reverse=True))
mr = self._offset_to_region[max_off] # type: MemoryRegion
return max_off + mr.size
except StopIteration:
return 0
def _region_from_offset(self, offset):
try:
off = next(self._offset_to_region.irange(maximum=offset, reverse=True))
return off, self._offset_to_region[off]
except StopIteration:
return None, None
def _addr_from_offset(self, mr, base_offset, offset):
return mr.addr + (offset - base_offset)
def _get_disasm(self, func):
"""
:param func:
:return:
"""
if func.addr not in self._disasms:
self._disasms[func.addr] = self.workspace.instance.project.analyses.Disassembly(function=func)
return self._disasms[func.addr]
class UltraPage(MemoryObjectMixin, PageBase):
SUPPORTS_CONCRETE_LOAD = True
def __init__(self, memory=None, init_zero=False, **kwargs):
super().__init__(**kwargs)
if memory is not None:
self.concrete_data = bytearray(memory.page_size)
if init_zero:
self.symbolic_bitmap = bytearray(memory.page_size)
else:
self.symbolic_bitmap = bytearray(b'\1' * memory.page_size)
else:
self.concrete_data = None
self.symbolic_bitmap = None
self.symbolic_data = SortedDict()
@classmethod
def new_from_shared(cls, data, memory=None, **kwargs):
o = cls(**kwargs)
o.concrete_data = data
o.symbolic_bitmap = bytearray(memory.page_size)
o.refcount = 2
return o
def copy(self, memo):
o = super().copy(memo)
o.concrete_data = bytearray(self.concrete_data)
o.symbolic_bitmap = bytearray(self.symbolic_bitmap)
o.symbolic_data = SortedDict(self.symbolic_data)
return o
def load(self,
addr,
size=None,
page_addr=None,
endness=None,
memory=None,
cooperate=False,
**kwargs):
concrete_run = []
symbolic_run = ...
last_run = None
result = []
def cycle(end):
if last_run is symbolic_run and symbolic_run is None:
fill(end)
elif last_run is concrete_run:
new_ast = claripy.BVV(concrete_run, (end - result[-1][0]) *
memory.state.arch.byte_width)
new_obj = SimMemoryObject(new_ast, result[-1][0], endness)
result[-1] = (result[-1][0], new_obj)
def fill(end):
global_end_addr = end
global_start_addr = result[-1][0]
size = global_end_addr - global_start_addr
new_ast = self._default_value(
global_start_addr,
size,
name='%s_%x' % (memory.id, global_start_addr),
key=(self.category, global_start_addr),
memory=memory,
**kwargs)
new_item = SimMemoryObject(new_ast,
global_start_addr,
endness=endness)
self.symbolic_data[global_start_addr - page_addr] = new_item
result[-1] = (global_start_addr, new_item)
for subaddr in range(addr, addr + size):
realaddr = subaddr + page_addr
if self.symbolic_bitmap[subaddr]:
cur_val = self._get_object(subaddr, page_addr, memory=memory)
if cur_val is last_run and last_run is symbolic_run:
pass
else:
cycle(realaddr)
last_run = symbolic_run = cur_val
result.append((realaddr, cur_val))
else:
cur_val = self.concrete_data[subaddr]
if last_run is concrete_run:
if endness == 'Iend_LE':
last_run = concrete_run = concrete_run | (
cur_val << (memory.state.arch.byte_width *
(realaddr - result[-1][0])))
else:
last_run = concrete_run = (
concrete_run <<
memory.state.arch.byte_width) | cur_val
result[-1] = (result[-1][0], concrete_run)
else:
cycle(realaddr)
last_run = concrete_run = cur_val
result.append((realaddr, cur_val))
cycle(page_addr + addr + size)
if not cooperate:
result = self._force_load_cooperation(result,
size,
endness,
page_addr=page_addr,
memory=memory,
**kwargs)
return result
def store(self,
addr,
data: Union[int, SimMemoryObject],
size: int = None,
endness=None,
memory=None,
page_addr=None,
cooperate=False,
**kwargs):
if not cooperate:
data = self._force_store_cooperation(addr,
data,
size,
endness,
page_addr=page_addr,
memory=memory,
**kwargs)
if size >= memory.page_size - addr:
size = memory.page_size - addr
if type(data) is not int:
if data.object.op == 'BVV':
# trim the unnecessary leading bytes if there are any
full_bits = len(data.object)
start = (page_addr + addr - data.base) & (
(1 << memory.state.arch.bits) - 1)
if start >= data.base + data.length:
raise SimMemoryError("Not enough bytes to store.")
start_bits = full_bits - start * memory.state.arch.byte_width - 1
# trim the overflowing bytes if there are any
end_bits = start_bits + 1 - size * memory.state.arch.byte_width
if start_bits != full_bits or end_bits != 0:
if endness == 'Iend_LE':
start_bits, end_bits = len(
data.object) - end_bits - 1, len(
data.object) - start_bits - 1
obj = data.object[start_bits:end_bits]
data = obj.args[0]
if type(data) is int or data.object.op == 'BVV':
# mark range as not symbolic
self.symbolic_bitmap[addr:addr + size] = b'\0' * size
# store
arange = range(addr, addr + size)
if type(data) is int:
ival = data
else: # data.object.op == 'BVV'
ival = data.object.args[0]
if endness == 'Iend_BE':
arange = reversed(arange)
assert memory.state.arch.byte_width == 8
# TODO: Make UltraPage support architectures with greater byte_widths (but are still multiples of 8)
for subaddr in arange:
self.concrete_data[subaddr] = ival & 0xff
ival >>= 8
else:
# mark range as symbolic
self.symbolic_bitmap[addr:addr + size] = b'\1' * size
# set ending object
try:
endpiece = next(
self.symbolic_data.irange(maximum=addr + size,
reverse=True))
except StopIteration:
pass
else:
if endpiece != addr + size:
self.symbolic_data[addr +
size] = self.symbolic_data[endpiece]
# clear range
for midpiece in self.symbolic_data.irange(maximum=addr + size - 1,
minimum=addr,
reverse=True):
del self.symbolic_data[midpiece]
# set.
self.symbolic_data[addr] = data
def merge(self,
others: List['UltraPage'],
merge_conditions,
common_ancestor=None,
page_addr: int = None,
memory=None):
all_pages = [self] + others
merged_to = None
merged_objects = set()
merged_offsets = set()
changed_bytes: Set[int] = set()
for o in others:
changed_bytes |= self.changed_bytes(o, page_addr=page_addr)
for b in sorted(changed_bytes):
if merged_to is not None and not b >= merged_to:
l.info("merged_to = %d ... already merged byte 0x%x",
merged_to, b)
continue
l.debug("... on byte 0x%x", b)
memory_objects: List[Tuple[SimMemoryObject, Any]] = []
concretes: List[Tuple[int, Any]] = []
unconstrained_in: List[Tuple['UltraPage', Any]] = []
our_mo: Optional[SimMemoryObject] = None
# first get a list of all memory objects at that location, and
# all memories that don't have those bytes
for pg, fv in zip(all_pages, merge_conditions):
if pg.symbolic_bitmap[b]:
mo = pg._get_object(b, page_addr)
if mo is not None:
l.info("... MO present in %s", fv)
memory_objects.append((mo, fv))
if pg is self:
our_mo = mo
else:
l.info("... not present in %s", fv)
unconstrained_in.append((pg, fv))
else:
# concrete data
concretes.append((pg.concrete_data[b], fv))
# fast path: no memory objects, no unconstrained positions, and only one concrete value
if not memory_objects and not unconstrained_in and len(
set(cv for cv, _ in concretes)) == 1:
cv = concretes[0][0]
self.store(b, cv, size=1, cooperate=True, page_addr=page_addr)
continue
# convert all concrete values into memory objects
for cv, fv in concretes:
mo = SimMemoryObject(claripy.BVV(cv, size=8), page_addr + b,
'Iend_LE')
memory_objects.append((mo, fv))
mos = set(mo for mo, _ in memory_objects)
mo_bases = set(mo.base for mo, _ in memory_objects)
mo_lengths = set(mo.length for mo, _ in memory_objects)
if not unconstrained_in and not (mos - merged_objects):
continue
# first, optimize the case where we are dealing with the same-sized memory objects
if len(mo_bases) == 1 and len(
mo_lengths) == 1 and not unconstrained_in:
to_merge = [(mo.object, fv) for mo, fv in memory_objects]
# Update `merged_to`
mo_base = list(mo_bases)[0]
merged_to = mo_base + list(mo_lengths)[0]
merged_val = self._merge_values(to_merge,
memory_objects[0][0].length,
memory=memory)
if merged_val is None:
continue
if our_mo is None:
# this object does not exist in the current page. do the store
new_object = SimMemoryObject(merged_val, page_addr + b,
memory_objects[0][0].endness)
self.store(b,
new_object,
size=list(mo_lengths)[0],
cooperate=True,
memory=memory)
merged_objects.add(new_object)
else:
# do the replacement
new_object = self._replace_memory_object(our_mo,
merged_val,
memory=memory)
merged_objects.add(new_object)
merged_objects.update(mos)
merged_offsets.add(b)
else:
# get the size that we can merge easily. This is the minimum of
# the size of all memory objects and unallocated spaces.
min_size = min([
mo.length - (page_addr + b - mo.base)
for mo, _ in memory_objects
])
for um, _ in unconstrained_in:
for i in range(0, min_size):
if um._contains(b + i, page_addr):
min_size = i
break
merged_to = b + min_size
l.info("... determined minimum size of %d", min_size)
# Now, we have the minimum size. We'll extract/create expressions of that
# size and merge them
extracted = [(mo.bytes_at(page_addr + b, min_size), fv)
for mo, fv in memory_objects
] if min_size != 0 else []
created = [
(self._default_value(None,
min_size,
name="merge_uc_%s_%x" % (uc.id, b),
memory=memory), fv)
for uc, fv in unconstrained_in
]
to_merge = extracted + created
merged_val = self._merge_values(to_merge,
min_size,
memory=memory)
if merged_val is None:
continue
self.store(b,
merged_val,
size=len(merged_val) //
memory.state.arch.byte_width,
inspect=False,
page_addr=page_addr,
memory=memory) # do not convert endianness again
merged_offsets.add(b)
return merged_offsets
def concrete_load(self, addr, size, **kwargs):
if type(self.concrete_data) is bytearray:
return memoryview(
self.concrete_data)[addr:addr + size], memoryview(
self.symbolic_bitmap)[addr:addr + size]
else:
return self.concrete_data[addr:addr + size], memoryview(
self.symbolic_bitmap)[addr:addr + size]
def changed_bytes(self, other, page_addr=None) -> Set[int]:
changes = set()
for addr in range(len(self.symbolic_bitmap)):
if self.symbolic_bitmap[addr] != other.symbolic_bitmap[addr]:
changes.add(addr)
elif self.symbolic_bitmap[addr] == 0:
if self.concrete_data[addr] != other.concrete_data[addr]:
changes.add(addr)
else:
try:
aself = next(
self.symbolic_data.irange(maximum=addr, reverse=True))
except StopIteration:
aself = None
try:
aother = next(
other.symbolic_data.irange(maximum=addr, reverse=True))
except StopIteration:
aother = None
if aself is None and aother is None:
pass
elif aself is None and aother is not None:
oobj = other.symbolic_data[aother]
if oobj.includes(addr + page_addr):
changes.add(addr)
elif aother is None and aself is not None:
aobj = self.symbolic_data[aself]
if aobj.includes(addr + page_addr):
changes.add(addr)
else:
real_addr = page_addr + addr
aobj = self.symbolic_data[aself]
oobj = other.symbolic_data[aother]
acont = aobj.includes(real_addr)
ocont = oobj.includes(real_addr)
if acont != ocont:
changes.add(addr)
elif acont is False:
pass
else:
abyte = aobj.bytes_at(real_addr, 1)
obyte = oobj.bytes_at(real_addr, 1)
if abyte is not obyte:
changes.add(addr)
return changes
def _contains(self, start: int, page_addr: int):
if not self.symbolic_bitmap[start]:
# concrete data
return True
else:
# symbolic data or does not exist
return self._get_object(start, page_addr) is not None
def _get_object(self,
start: int,
page_addr: int,
memory=None) -> Optional[SimMemoryObject]:
try:
place = next(self.symbolic_data.irange(maximum=start,
reverse=True))
except StopIteration:
return None
else:
obj = self.symbolic_data[place]
if obj.includes(start + page_addr):
return obj
elif memory is not None and obj.includes(
start + page_addr + (1 << memory.state.arch.bits)):
return obj
else:
return None
def replace_all_with_offsets(self,
offsets: Iterable[int],
old: claripy.ast.BV,
new: claripy.ast.BV,
memory=None):
memory_objects = set()
for offset in sorted(list(offsets)):
try:
a = next(
self.symbolic_data.irange(maximum=offset, reverse=True))
except StopIteration:
a = None
if a is None:
continue
aobj = self.symbolic_data[a]
memory_objects.add(aobj)
replaced_objects_cache = {}
for mo in memory_objects:
replaced_object = None
if mo.object in replaced_objects_cache:
if mo.object is not replaced_objects_cache[mo.object]:
replaced_object = replaced_objects_cache[mo.object]
else:
replaced_object = mo.object.replace(old, new)
replaced_objects_cache[mo.object] = replaced_object
if mo.object is replaced_object:
# The replace does not really occur
replaced_object = None
if replaced_object is not None:
self._replace_memory_object(mo, replaced_object, memory=memory)
def _replace_memory_object(self,
old: SimMemoryObject,
new_content: claripy.Bits,
memory=None):
"""
Replaces the memory object `old` with a new memory object containing `new_content`.
:param old: A SimMemoryObject (i.e., one from :func:`memory_objects_for_hash()` or :func:`
memory_objects_for_name()`).
:param new_content: The content (claripy expression) for the new memory object.
:returns: the new memory object
"""
if (old.object.size() if not old.is_bytes else len(old.object) *
self.state.arch.byte_width) != new_content.size():
raise SimMemoryError(
"memory objects can only be replaced by the same length content"
)
new = SimMemoryObject(new_content,
old.base,
old.endness,
byte_width=old._byte_width)
for k in list(self.symbolic_data):
if self.symbolic_data[k] is old:
self.symbolic_data[k] = new
if isinstance(new.object, claripy.ast.BV):
for b in range(old.base, old.base + old.length):
self._update_mappings(b, old.object, new.object, memory=memory)
return new
class MemoryIndex(object):
''' An ad-hoc memory index file for when an on-file index file does not
exist. '''
def __init__(self, tbl, colind):
self.__idx = SortedDict(Key)
init_idx_with(self, tbl, colind)
def add(self, rowid, key):
if key not in self.__idx:
val = set()
self.__idx[key] = val
else:
val = self.__idx[key]
val.add(rowid)
def clear(self):
self.__idx.clear()
def search(self, key, inequality):
if inequality == '!=':
for k in self.__idx:
if compare(k, key, '!='):
yield from self.__idx[k]
elif inequality in ('=', '=='):
if key in self.__idx:
yield from self.__idx[key]
else:
opers = {
'<=': {
'minimum': None,
'maximum': key
},
'<': {
'minimum': None,
'maximum': key,
'inclusive': (False, False)
},
'>=': {
'minimum': key,
'maximum': None
},
'>': {
'minimum': key,
'maximum': None,
'inclusive': (False, False)
}
}
for k in self.__idx.irange(**opers[inequality]):
yield from self.__idx[k]
def modify(self, old_rowid, new_rowid, key):
if key not in self.__idx:
return False
s = self.__idx[key]
if rowid not in s:
return False
s.remove(old_rowid)
s.add(new_rowid)
return True
def delete(self, rowid, key):
if key not in self.__idx:
return False
s = self.__idx[key]
if rowid not in s:
return False
s.remove(rowid)
return True
class QLinearViewer(QWidget):
def __init__(self, workspace, disasm_view, parent=None):
super(QLinearViewer, self).__init__(parent)
self.workspace = workspace
self.disasm_view = disasm_view
self.objects = [] # Objects that will be painted
self.cfg = None
self.cfb = None
self._offset_to_region = SortedDict()
self._addr_to_region_offset = SortedDict()
# Offset (in bytes) into the entire blanket view
self._offset = 0
# The maximum offset (in bytes) of the blanket view
self._max_offset = None
# The first line that is rendered of the first object in self.objects. Start from 0.
self._start_line_in_object = 0
self._linear_view = None # type: QLinearGraphicsView
self._disasms = {}
self._init_widgets()
#
# Properties
#
@property
def offset(self):
return self._offset
@offset.setter
def offset(self, v):
self._offset = v
@property
def start_line_in_object(self):
return self._start_line_in_object
@property
def max_offset(self):
if self._max_offset is None:
self._max_offset = self._calculate_max_offset()
return self._max_offset
#
# Proxy properties
#
@property
def selected_operands(self):
return self._linear_view.selected_operands
@property
def selected_insns(self):
return self._linear_view.selected_insns
#
# Public methods
#
def initialize(self):
if self.cfb is None:
return
self._addr_to_region_offset.clear()
self._offset_to_region.clear()
self._disasms.clear()
self._offset = 0
self._max_offset = None
self._start_line_in_object = 0
# enumerate memory regions
byte_offset = 0
for mr in self.cfb.regions: # type:MemoryRegion
self._addr_to_region_offset[mr.addr] = byte_offset
self._offset_to_region[byte_offset] = mr
byte_offset += mr.size
def navigate_to_addr(self, addr):
if not self._addr_to_region_offset:
return
try:
floor_region_addr = next(
self._addr_to_region_offset.irange(maximum=addr, reverse=True))
except StopIteration:
floor_region_addr = next(self._addr_to_region_offset.irange())
floor_region_offset = self._addr_to_region_offset[floor_region_addr]
offset_into_region = addr - floor_region_addr
self.navigate_to(floor_region_offset + offset_into_region)
def refresh(self):
self._linear_view.refresh()
def navigate_to(self, offset):
self._linear_view.navigate_to(int(offset))
self.prepare_objects(offset)
self._linear_view.refresh()
def prepare_objects(self, offset, start_line=0):
"""
Prepare objects to print based on offset and start_line. Update self.objects, self._offset, and
self._start_line_in_object.
:param int offset: Beginning offset (in bytes) to display in the linear viewer.
:param int start_line: The first line into the first object to display in the linear viewer.
:return: None
"""
if offset == self._offset and start_line == self._start_line_in_object:
return
# Convert the offset to memory region
base_offset, mr = self._region_from_offset(
offset) # type: int,MemoryRegion
if mr is None:
return
addr = self._addr_from_offset(mr, base_offset, offset)
_l.debug("Address %#x, offset %d, start_line %d.", addr, offset,
start_line)
if start_line < 0:
# Which object are we currently displaying at the top of the disassembly view?
try:
top_obj_addr = self.cfb.floor_addr(addr=addr)
except KeyError:
top_obj_addr = addr
# Reverse-iterate until we have enough lines to compensate start_line
for obj_addr, obj in self.cfb.ceiling_items(addr=top_obj_addr,
reverse=True,
include_first=False):
qobject = self._obj_to_paintable(obj_addr, obj)
if qobject is None:
continue
object_lines = int(qobject.height //
self._linear_view.line_height())
_l.debug(
"Compensating negative start_line: object %s, object_lines %d.",
obj, object_lines)
start_line += object_lines
if start_line >= 0:
addr = obj_addr
# Update offset
new_region_addr = next(
self._addr_to_region_offset.irange(maximum=addr,
reverse=True))
new_region_offset = self._addr_to_region_offset[
new_region_addr]
offset = (addr - new_region_addr) + new_region_offset
break
else:
# umm we don't have enough objects to compensate the negative start_line
start_line = 0
# update addr and offset to their minimal values
addr = next(self._addr_to_region_offset.irange())
offset = self._addr_to_region_offset[addr]
_l.debug("After adjustment: Address %#x, offset %d, start_line %d.",
addr, offset, start_line)
self.objects = []
viewable_lines = int(self._linear_view.height() //
self._linear_view.line_height())
lines = 0
start_line_in_object = 0
# Load a page of objects
for obj_addr, obj in self.cfb.floor_items(addr=addr):
qobject = self._obj_to_paintable(obj_addr, obj)
if qobject is None:
# Conversion failed
continue
if isinstance(qobject, QBlock):
for insn_addr in qobject.addr_to_insns.keys():
self._linear_view._add_insn_addr_block_mapping(
insn_addr, qobject)
object_lines = int(qobject.height //
self._linear_view.line_height())
if start_line >= object_lines:
# this object should be skipped. ignore it
start_line -= object_lines
# adjust the offset as well
if obj_addr <= addr < obj_addr + obj.size:
offset += obj_addr + obj.size - addr
else:
offset += obj.size
_l.debug("Skipping object %s (size %d). New offset: %d.", obj,
obj.size, offset)
else:
if start_line > 0:
_l.debug(
"First object to paint: %s (size %d). Current offset %d.",
obj, obj.size, offset)
# this is the first object to paint
start_line_in_object = start_line
start_line = 0
lines += object_lines - start_line_in_object
else:
lines += object_lines
self.objects.append(qobject)
if lines > viewable_lines:
break
_l.debug("Final offset %d, start_line_in_object %d.", offset,
start_line_in_object)
# Update properties
self._offset = offset
self._start_line_in_object = start_line_in_object
#
# Private methods
#
def _init_widgets(self):
self._linear_view = QLinearGraphicsView(self, self.disasm_view)
layout = QHBoxLayout()
layout.addWidget(self._linear_view)
layout.setContentsMargins(0, 0, 0, 0)
self.setLayout(layout)
# Setup proxy methods
self.update_label = self._linear_view.update_label
self.select_instruction = self._linear_view.select_instruction
self.unselect_instruction = self._linear_view.unselect_instruction
self.unselect_all_instructions = self._linear_view.unselect_all_instructions
self.select_operand = self._linear_view.select_operand
self.unselect_operand = self._linear_view.unselect_operand
self.unselect_all_operands = self._linear_view.unselect_all_operands
self.show_selected = self._linear_view.show_selected
self.show_instruction = self._linear_view.show_instruction
def _obj_to_paintable(self, obj_addr, obj):
if isinstance(obj, Block):
cfg_node = self.cfg.get_any_node(obj.addr, force_fastpath=True)
if cfg_node is not None:
func_addr = cfg_node.function_address
func = self.cfg.kb.functions[func_addr] # FIXME: Resiliency
disasm = self._get_disasm(func)
qobject = QBlock(
self.workspace,
func_addr,
self.disasm_view,
disasm,
self.disasm_view.infodock,
obj.addr,
[obj],
{},
mode='linear',
)
else:
# TODO: This should be displayed as a function thunk
_l.error(
"QLinearViewer: Unexpected result: CFGNode %#x is not found in CFG."
"Display it as a QUnknownBlock.", obj.addr)
qobject = QUnknownBlock(self.workspace, obj_addr, obj.bytes)
elif isinstance(obj, Unknown):
qobject = QUnknownBlock(self.workspace, obj_addr, obj.bytes)
else:
qobject = None
return qobject
def _calculate_max_offset(self):
try:
max_off = next(self._offset_to_region.irange(reverse=True))
mr = self._offset_to_region[max_off] # type: MemoryRegion
return max_off + mr.size
except StopIteration:
return 0
def _region_from_offset(self, offset):
try:
off = next(
self._offset_to_region.irange(maximum=offset, reverse=True))
return off, self._offset_to_region[off]
except StopIteration:
return None, None
def _addr_from_offset(self, mr, base_offset, offset):
return mr.addr + (offset - base_offset)
def _get_disasm(self, func):
"""
:param func:
:return:
"""
if func.addr not in self._disasms:
self._disasms[
func.
addr] = self.workspace.instance.project.analyses.Disassembly(
function=func)
return self._disasms[func.addr]
class KeyedRegion(object):
"""
KeyedRegion keeps a mapping between stack offsets and all objects covering that offset. It assumes no variable in
this region overlap with another variable in this region.
Registers and function frames can all be viewed as a keyed region.
"""
def __init__(self, tree=None):
self._storage = SortedDict() if tree is None else tree
def _get_container(self, offset):
try:
base_offset = next(self._storage.irange(maximum=offset, reverse=True))
except StopIteration:
return offset, None
else:
container = self._storage[base_offset]
if container.includes(offset):
return base_offset, container
return offset, None
def __contains__(self, offset):
"""
Test if there is at least one varaible covering the given offset.
:param offset:
:return:
"""
return self._get_container(offset)[1] is not None
def __len__(self):
return len(self._storage)
def __iter__(self):
return iter(self._storage.values())
def __eq__(self, other):
if set(self._storage.keys()) != set(other._storage.keys()):
return False
for k, v in self._storage.items():
if v != other._storage[k]:
return False
return True
def copy(self):
if not self._storage:
return KeyedRegion()
kr = KeyedRegion()
for key, ro in self._storage.items():
kr._storage[key] = ro.copy()
return kr
def merge(self, other, make_phi_func=None):
"""
Merge another KeyedRegion into this KeyedRegion.
:param KeyedRegion other: The other instance to merge with.
:return: None
"""
# TODO: is the current solution not optimal enough?
for _, item in other._storage.items(): # type: RegionObject
for loc_and_var in item.stored_objects:
self.__store(loc_and_var, overwrite=False, make_phi_func=make_phi_func)
return self
def dbg_repr(self):
"""
Get a debugging representation of this keyed region.
:return: A string of debugging output.
"""
keys = self._storage.keys()
offset_to_vars = { }
for key in sorted(keys):
ro = self._storage[key]
variables = [ obj.obj for obj in ro.stored_objects ]
offset_to_vars[key] = variables
s = [ ]
for offset, variables in offset_to_vars.items():
s.append("Offset %#x: %s" % (offset, variables))
return "\n".join(s)
def add_variable(self, start, variable):
"""
Add a variable to this region at the given offset.
:param int start:
:param SimVariable variable:
:return: None
"""
size = variable.size if variable.size is not None else 1
self.add_object(start, variable, size)
def add_object(self, start, obj, object_size):
"""
Add/Store an object to this region at the given offset.
:param start:
:param obj:
:param int object_size: Size of the object
:return:
"""
self._store(start, obj, object_size, overwrite=False)
def set_variable(self, start, variable):
"""
Add a variable to this region at the given offset, and remove all other variables that are fully covered by
this variable.
:param int start:
:param SimVariable variable:
:return: None
"""
size = variable.size if variable.size is not None else 1
self.set_object(start, variable, size)
def set_object(self, start, obj, object_size):
"""
Add an object to this region at the given offset, and remove all other objects that are fully covered by this
object.
:param start:
:param obj:
:param object_size:
:return:
"""
self._store(start, obj, object_size, overwrite=True)
def get_base_addr(self, addr):
"""
Get the base offset (the key we are using to index objects covering the given offset) of a specific offset.
:param int addr:
:return:
:rtype: int or None
"""
base_addr, container = self._get_container(addr)
if container is None:
return None
else:
return base_addr
def get_variables_by_offset(self, start):
"""
Find variables covering the given region offset.
:param int start:
:return: A list of stack variables.
:rtype: set
"""
_, container = self._get_container(start)
if container is None:
return []
else:
return container.internal_objects
def get_objects_by_offset(self, start):
"""
Find objects covering the given region offset.
:param start:
:return:
"""
_, container = self._get_container(start)
if container is None:
return set()
else:
return container.internal_objects
#
# Private methods
#
def _store(self, start, obj, size, overwrite=False):
"""
Store a variable into the storage.
:param int start: The beginning address of the variable.
:param obj: The object to store.
:param int size: Size of the object to store.
:param bool overwrite: Whether existing objects should be overwritten or not.
:return: None
"""
stored_object = StoredObject(start, obj, size)
self.__store(stored_object, overwrite=overwrite)
def __store(self, stored_object, overwrite=False, make_phi_func=None):
"""
Store a variable into the storage.
:param StoredObject stored_object: The descriptor describing start address and the variable.
:param bool overwrite: Whether existing objects should be overwritten or not.
:return: None
"""
start = stored_object.start
object_size = stored_object.size
end = start + object_size
# region items in the middle
overlapping_items = list(self._storage.irange(start, end-1))
# is there a region item that begins before the start and overlaps with this variable?
floor_key, floor_item = self._get_container(start)
if floor_item is not None and floor_key not in overlapping_items:
# insert it into the beginning
overlapping_items.insert(0, floor_key)
# scan through the entire list of region items, split existing regions and insert new regions as needed
to_update = {start: RegionObject(start, object_size, {stored_object})}
last_end = start
for floor_key in overlapping_items:
item = self._storage[floor_key]
if item.start < start:
# we need to break this item into two
a, b = item.split(start)
if overwrite:
b.set_object(stored_object)
else:
self._add_object_or_make_phi(b, stored_object, make_phi_func=make_phi_func)
to_update[a.start] = a
to_update[b.start] = b
last_end = b.end
elif item.start > last_end:
# there is a gap between the last item and the current item
# fill in the gap
new_item = RegionObject(last_end, item.start - last_end, {stored_object})
to_update[new_item.start] = new_item
last_end = new_item.end
elif item.end > end:
# we need to split this item into two
a, b = item.split(end)
if overwrite:
a.set_object(stored_object)
else:
self._add_object_or_make_phi(a, stored_object, make_phi_func=make_phi_func)
to_update[a.start] = a
to_update[b.start] = b
last_end = b.end
else:
if overwrite:
item.set_object(stored_object)
else:
self._add_object_or_make_phi(item, stored_object, make_phi_func=make_phi_func)
to_update[item.start] = item
self._storage.update(to_update)
def _is_overlapping(self, start, variable):
if variable.size is not None:
# make sure this variable does not overlap with any other variable
end = start + variable.size
try:
prev_offset = next(self._storage.irange(maximum=end-1, reverse=True))
except StopIteration:
prev_offset = None
if prev_offset is not None:
if start <= prev_offset < end:
return True
prev_item = self._storage[prev_offset][0]
prev_item_size = prev_item.size if prev_item.size is not None else 1
if start < prev_offset + prev_item_size < end:
return True
else:
try:
prev_offset = next(self._storage.irange(maximum=start, reverse=True))
except StopIteration:
prev_offset = None
if prev_offset is not None:
prev_item = self._storage[prev_offset][0]
prev_item_size = prev_item.size if prev_item.size is not None else 1
if prev_offset <= start < prev_offset + prev_item_size:
return True
return False
def _add_object_or_make_phi(self, item, stored_object, make_phi_func=None): #pylint:disable=no-self-use
if not make_phi_func or len({stored_object.obj} | item.internal_objects) == 1:
item.add_object(stored_object)
else:
# make a phi node
item.set_object(StoredObject(stored_object.start,
make_phi_func(stored_object.obj, *item.internal_objects),
stored_object.size,
)
)
class PatchManager(KnowledgeBasePlugin):
"""
A placeholder-style implementation for a binary patch manager. This class should be significantly changed in the
future when all data about loaded binary objects are loaded into angr knowledge base from CLE. As of now, it only
stores byte-level replacements. Other angr components may choose to use or not use information provided by this
manager. In other words, it is not transparent.
Patches should not overlap, but it's user's responsibility to check for and avoid overlapping patches.
"""
def __init__(self, kb):
super().__init__()
self._patches = SortedDict()
self._kb = kb
def add_patch(self, addr, new_bytes):
self._patches[addr] = Patch(addr, new_bytes)
def remove_patch(self, addr):
if addr in self._patches:
del self._patches[addr]
def patch_addrs(self):
return self._patches.keys()
def get_patch(self, addr):
"""
Get patch at the given address.
:param int addr: The address of the patch.
:return: The patch if there is one starting at the address, or None if there isn't any.
:rtype: Patch or None
"""
return self._patches.get(addr, None)
def get_all_patches(self, addr, size):
"""
Retrieve all patches that cover a region specified by [addr, addr+size).
:param int addr: The address of the beginning of the region.
:param int size: Size of the region.
:return: A list of patches.
:rtype: list
"""
patches = []
for patch_addr in self._patches.irange(maximum=addr + size - 1,
reverse=True):
p = self._patches[patch_addr]
if self.overlap(p.addr, p.addr + len(p), addr, addr + size):
patches.append(p)
else:
break
return patches[::-1]
def keys(self):
return self._patches.keys()
def items(self):
return self._patches.items()
def values(self):
return self._patches.values()
def copy(self):
o = PatchManager(self._kb)
o._patches = self._patches.copy()
@staticmethod
def overlap(a0, a1, b0, b1):
return a0 <= b0 < a1 or a0 <= b1 < a1 or b0 <= a0 < b1
class LevelTrace(object):
""" Traces the level of some entity across a time span """
def __init__(self, trace=None):
""" Creates a new level trace, possibly copying from an existing object. """
if trace is None:
self._trace = SortedDict()
elif isinstance(trace, LevelTrace):
self._trace = SortedDict(trace._trace)
else:
self._trace = SortedDict(trace)
# Make sure trace is terminated (returns to 0)
if len(self._trace) > 0 and self._trace[self._trace.keys()[-1]] != 0:
raise ValueError(
"Trace not terminated - ends with {}:{}!".format(
self._trace.keys()[-1], self._trace[self._trace.keys()[-1]])
)
def __repr__(self):
items = ', '.join(["{!r}: {!r}".format(k, v)
for k, v in self._trace.items()])
return "LevelTrace({{{}}})".format(items)
def __eq__(self, other):
return self._trace == other._trace
def __neg__(self):
return self.map(operator.neg)
def __sub__(self, other):
return self.zip_with(other, operator.sub)
def __add__(self, other):
return self.zip_with(other, operator.add)
def start(self):
""" Returns first non-null point in trace """
if len(self._trace) == 0:
return 0
return self._trace.keys()[0]
def end(self):
""" Returns first point in trace that is null and only followed by nulls """
if len(self._trace) == 0:
return 0
return self._trace.keys()[-1]
def length(self):
if len(self._trace) == 0:
return 0
return self.end() - self.start()
def get(self, time):
ix = self._trace.bisect_right(time) - 1
if ix < 0:
return 0
else:
(_, lvl) = self._trace.peekitem(ix)
return lvl
def map(self, fn):
return LevelTrace({t: fn(v) for t, v in self._trace.items()})
def map_key(self, fn):
return LevelTrace(dict(fn(t, v) for t, v in self._trace.items()))
def shift(self, time):
return self.map_key(lambda t, v: (t + time, v))
def __getitem__(self, where):
# For non-slices defaults to get
if not isinstance(where, slice):
return self.get(where)
if where.step is not None:
raise ValueError("Stepping meaningless for LevelTrace!")
# Limit
res = LevelTrace(self)
if where.start is not None and where.start > res.start():
res.set(res.start(), where.start, 0)
if where.stop is not None and where.stop < res.end():
res.set(where.stop, res.end(), 0)
# Shift, if necessary
if where.start is not None:
res = res.shift(-where.start)
return res
def set(self, start, end, level):
""" Sets the level for some time range
:param start: Start of range
:param end: End of range
:aram amount: Level to set
"""
# Check errors, no-ops
if start >= end:
return
# Determine levels at start (and before start)
start_ix = self._trace.bisect_right(start) - 1
prev_lvl = lvl = 0
if start_ix >= 0:
(t, lvl) = self._trace.peekitem(start_ix)
# If we have no entry exactly at our start point, the
# level was constant at this point before
if start > t:
prev_lvl = lvl
# Otherwise look up previous level. Default 0 (see above)
elif start_ix > 0:
(_, prev_lvl) = self._trace.peekitem(start_ix-1)
# Prepare start
if prev_lvl == level:
if start in self._trace:
del self._trace[start]
else:
self._trace[start] = level
# Remove all in-between states
for time in list(self._trace.irange(start, end, inclusive=(False, False))):
lvl = self._trace[time]
del self._trace[time]
# Add or remove end, if necessary
if end not in self._trace:
if lvl != level:
self._trace[end] = lvl
elif level == self._trace[end]:
del self._trace[end]
def add(self, start, end, amount):
""" Increases the level for some time range
:param start: Start of range
:param end: End of range
:aram amount: Amount to add to level
"""
# Check errors, no-ops
if start > end:
raise ValueError("End needs to be after start!")
if start == end or amount == 0:
return
# Determine levels at start (and before start)
start_ix = self._trace.bisect_right(start) - 1
prev_lvl = lvl = 0
if start_ix >= 0:
(t, lvl) = self._trace.peekitem(start_ix)
# If we have no entry exactly at our start point, the
# level was constant at this point before
if start > t:
prev_lvl = lvl
# Otherwise look up previous level. Default 0 (see above)
elif start_ix > 0:
(_, prev_lvl) = self._trace.peekitem(start_ix-1)
# Prepare start
if prev_lvl == lvl + amount:
del self._trace[start]
else:
self._trace[start] = lvl + amount
# Update all in-between states
for time in self._trace.irange(start, end, inclusive=(False, False)):
lvl = self._trace[time]
self._trace[time] = lvl + amount
# Add or remove end, if necessary
if end not in self._trace:
self._trace[end] = lvl
elif lvl + amount == self._trace[end]:
del self._trace[end]
def __delitem__(self, where):
# Cannot set single values
if not isinstance(where, slice):
raise ValueError("Cannot set level for single point, pass an interval!")
if where.step is not None:
raise ValueError("Stepping meaningless for LevelTrace!")
# Set range to zero
start = (where.start if where.start is not None else self.start())
end = (where.stop if where.stop is not None else self.end())
self.set(start, end, 0)
def __setitem__(self, where, value):
# Cannot set single values
if not isinstance(where, slice):
raise ValueError("Cannot set level for single point, pass an interval!")
if where.step is not None:
raise ValueError("Stepping meaningless for LevelTrace!")
# Setting a level trace?
if isinstance(value, LevelTrace):
# Remove existing data
del self[where]
if where.start is not None:
if value.start() < 0:
raise ValueError("Level trace starts before 0!")
value = value.shift(where.start)
if where.stop is not None:
if value.end() > where.stop:
raise ValueError("Level trace to set is larger than slice!")
self._trace = (self + value)._trace
else:
# Otherwise set constant value
start = (where.start if where.start is not None else self.start())
end = (where.stop if where.stop is not None else self.end())
self.set(start, end, value)
def foldl1(self, start, end, fn):
"""
Does a left-fold over the levels present in the given range. Seeds
with level at start.
"""
if start > end:
raise ValueError("End needs to be after start!")
val = self.get(start)
start_ix = self._trace.bisect_right(start)
end_ix = self._trace.bisect_left(end)
for lvl in self._trace.values()[start_ix:end_ix]:
val = fn(val, lvl)
return val
def minimum(self, start, end):
""" Returns the lowest level in the given range """
return self.foldl1(start, end, min)
def maximum(self, start, end):
""" Returns the highest level in the given range """
return self.foldl1(start, end, max)
def foldl_time(self, start, end, val, fn):
"""
Does a left-fold over the levels present in the given range,
also passing how long the level was held. Seed passed.
"""
if start > end:
raise ValueError("End needs to be after start!")
last_time = start
last_lvl = self.get(start)
start_ix = self._trace.bisect_right(start)
end_ix = self._trace.bisect_left(end)
for time, lvl in self._trace.items()[start_ix:end_ix]:
val = fn(val, time-last_time, last_lvl)
last_time = time
last_lvl = lvl
return fn(val, end-last_time, last_lvl)
def integrate(self, start, end):
""" Returns the integral over a range (sum below level curve) """
return self.foldl_time(start, end, 0,
lambda v, time, lvl: v + time * lvl)
def average(self, start, end):
""" Returns the average level over a given range """
return self.integrate(start, end) / (end - start)
def find_above(self, time, level):
"""Returns the first time larger or equal to the given start time
where the level is at least the specified value.
"""
if self.get(time) >= level:
return time
ix = self._trace.bisect_right(time)
for t, lvl in self._trace.items()[ix:]:
if lvl >= level:
return t
return None
def find_below(self, time, level):
"""Returns the first time larger or equal to the given start time
where the level is less or equal the specified value.
"""
if self.get(time) <= level:
return time
ix = self._trace.bisect_right(time)
for t, lvl in self._trace.items()[ix:]:
if lvl <= level:
return t
return None
def find_below_backward(self, time, level):
"""Returns the last time smaller or equal to the given time where
there exists a region to the left where the level is below the
given value.
"""
last = time
ix = self._trace.bisect_right(time)-1
if ix >= 0:
for t, lvl in self._trace.items()[ix::-1]:
if lvl <= level and time > t:
return last
last = t
if level >= 0:
return last
return None
def find_above_backward(self, time, level):
"""Returns the last time smaller or equal to the given time where
there exists a region to the left where the level is below the
given value.
"""
last = time
ix = self._trace.bisect_right(time)-1
if ix >= 0:
for t, lvl in self._trace.items()[ix::-1]:
if lvl >= level and time > t:
return last
last = t
if level <= 0:
return last
return None
def find_period_below(self, start, end, target, length):
"""Returns a period where the level is below the target for a certain
length of time, within a given start and end time"""
if start > end:
raise ValueError("End needs to be after start!")
if length < 0:
raise ValueError("Period length must be larger than zero!")
period_start = (start if self.get(start) <= target else None)
start_ix = self._trace.bisect_right(start)
end_ix = self._trace.bisect_left(end)
for time, lvl in self._trace.items()[start_ix:end_ix]:
# Period long enough?
if period_start is not None:
if time >= period_start + length:
return period_start
# Not enough space until end?
elif time + length > end:
return None
# Above target? Reset period
if lvl > target:
period_start = None
else:
if period_start is None:
period_start = time
# Possible at end?
if period_start is not None and period_start+length <= end:
return period_start
# Nothing found
return None
def zip_with(self, other, fn):
# Simple cases
if len(self._trace) == 0:
return other.map(lambda x: fn(0, x))
if len(other._trace) == 0:
return self.map(lambda x: fn(x, 0))
# Read first item from both sides
left = self._trace.items()
right = other._trace.items()
left_ix = 0
right_ix = 0
left_val = 0
right_val = 0
last_val = 0
trace = SortedDict()
# Go through pairs
while left_ix < len(left) and right_ix < len(right):
# Next items
lt,lv = left[left_ix]
rt,rv = right[right_ix]
# Determine what to do
if lt < rt:
v = fn(lv, right_val)
if v != last_val:
last_val = trace[lt] = v
left_val = lv
left_ix += 1
elif lt > rt:
v = fn(left_val, rv)
if v != last_val:
last_val = trace[rt] = v
right_val = rv
right_ix += 1
else:
v = fn(lv, rv)
if v != last_val:
last_val = trace[lt] = v
left_val = lv
left_ix += 1
right_val = rv
right_ix += 1
# Handle left-overs
while left_ix < len(left):
lt,lv = left[left_ix]
v = fn(lv, right_val)
if v != last_val:
last_val = trace[lt] = v
left_ix += 1
while right_ix < len(right):
rt,rv = right[right_ix]
v = fn(left_val, rv)
if v != last_val:
last_val = trace[rt] = v
right_ix += 1
return LevelTrace(trace)
