def test_intervaldict_iterators():
d = I.IntervalDict([(I.closedopen(0, 1), 0), (I.closedopen(1, 3), 1), (I.singleton(3), 2)])
assert d.keys() == [I.closedopen(0, 1), I.closedopen(1, 3), I.singleton(3)]
assert d.domain() == I.closed(0, 3)
assert d.values() == [0, 1, 2]
assert d.items() == list(zip(d.keys(), d.values()))
assert list(d) == d.keys()
# Iterators on empty
assert I.IntervalDict().values() == []
assert I.IntervalDict().items() == []
assert I.IntervalDict().keys() == []
assert I.IntervalDict().domain() == I.empty()
def read_memory_fast(s, module_list):
module_list = sorted(module_list)
# Get list of available address ranges for LSASS
pslist = s.plugins.pslist(proc_regex='lsass.exe')
task = next(pslist.filter_processes())
addr_space = task.get_process_address_space()
max_memory = s.GetParameter("highest_usermode_address")
mem_list = sorted([(run.start, run.end)
for run in addr_space.get_address_ranges(end=max_memory)
])
# Enumerate modules, find "holes" that need zero filling
filling = I.empty()
for a, mod_start, size in module_list:
d = I.IntervalDict()
mod = I.closedopen(mod_start, mod_start + size)
d[mod] = 'fill'
# What parts of the module are available?
for start, end in mem_list:
mem = I.closedopen(start, end)
if mem & mod != I.empty():
d[mem] = 'mem'
if start > mod_start + size:
break
filling |= d.find('fill')
# What to read, what to zero fill
operations = []
for x in list(filling):
operations.append((x.lower, x.upper, 'pad'))
for start, end in mem_list:
operations.append((start, end, 'mem'))
# Read & fill
memoryinfo_list = []
memory64_list = []
for start, end, op in sorted(operations):
size = end - start
mi = dict(BaseAddress=start,
AllocationBase=0,
AllocationProtect=0,
RegionSize=size,
Protect=0)
mi['State'] = 0
mi['Type'] = 0
memoryinfo_list.append(mi)
if op == 'fill':
#data = b'\x00'*size
# Attempt to read these anyway. Rekall will fill with zeros if the read fails
data = addr_space.read(start, size)
else:
data = addr_space.read(start, size)
memory64_list.append((start, size, data))
return memoryinfo_list, memory64_list
def find_intervals(genome1, genome2, all_match, strip):
# NOTE: There might be trouble with RSIs that are the delimited at front
# and back by the same marker.
# Only strip genomes (to find RSIs) if instructed in arguments
stripped1 = genome1
stripped2 = genome2
if strip:
stripped1 = strip_genome_unique(stripped1)
stripped2 = strip_genome_unique(stripped2)
# Index one genome, loop over the other.
index = Genome_pairs_index()
index.index_pairs(stripped1)
ints = intervals.IntervalDict()
for key, chrom in stripped2.chromosomes.iteritems():
for i in xrange(len(chrom) - 1):
pair2 = (chrom[i].id, chrom[i + 1].id)
if markers.are_siblings(*pair2):
continue
full_ids2 = [ marker.id for marker in
genome2.chromosomes[ key ] \
[ chrom[i].index : chrom[i+1].index+1 ] ]
pairs = index.query(*pair2)
# Find all pairs in the index with a full match (or all match)
matches = []
match_ids = []
for pair1 in pairs:
# do we have a match between pair1 and pair2?
full_ids1 = [ marker.id for marker in
genome1.chromosomes[ pair1[0].chromosome ] \
[ pair1[1].index : pair1[2].index+1 ] ]
cmp_result, _ = compare_marker_intervals(
full_ids1, full_ids2, all_match)
if cmp_result:
matches.append(pair1)
match_ids = full_ids1
if matches:
# Add a new interval to ints if not already there, and add the
# loci from matches to it. If the interval is already there, it
# must already have the loci from matches.
if not full_ids2 in ints:
ints.add(
intervals.Interval(
id=''.join(full_ids2),
marker_ids=full_ids2,
loci=[l for l, _, _ in matches],
order=intervals.Order(1),
weight=1,
comment='',
))
# Add the newly found locus in genome 2 to the interval.
interval = ints[match_ids]
locus2 = markers.Locus(
species=genome2.species,
chromosome=key,
start=chrom[i].locus.start,
end=chrom[i + 1].locus.end,
orientation=1,
comment='',
)
if interval.marker_ids == list(reversed(full_ids2)):
locus2.orientation *= -1
interval.loci.append(locus2)
# Filter the ints to distinguish repeat spanning intervals.
adjs = intervals.IntervalDict()
RSIs = intervals.IntervalDict()
for interval in ints.itervalues():
if len(interval.marker_ids) == 2:
adjs.add(interval)
else:
RSIs.add(interval)
return adjs, RSIs
def test_intervaldict_other_methods():
# Containment
d = I.IntervalDict([(I.closed(0, 3), 0)])
assert 0 in d
assert -1 not in d
assert I.closed(-2, -1) not in d
assert I.closed(1, 2) in d
assert I.closed(1, 4) not in d
# Repr
assert repr(d) == '{' + repr(I.closed(0, 3)) + ': 0}'
# pop
t = d.pop(2)
assert t == 0
t = d.pop(4, 1)
assert t == 1
with pytest.raises(KeyError):
d.pop(4)
# pop intervals
d = I.IntervalDict([(I.closed(0, 3), 0)])
t = d.pop(I.closed(0, 1))
assert t.items() == [(I.closed(0, 1), 0)]
assert d.items() == [(I.openclosed(1, 3), 0)]
t = d.pop(I.closed(0, 2), 1)
assert t.items() == [(I.closed(0, 1), 1), (I.openclosed(1, 2), 0)]
assert d.items() == [(I.openclosed(2, 3), 0)]
# popitem
d = I.IntervalDict([(I.closed(0, 3), 0)])
t = d.popitem()
assert t.items() == [(I.closed(0, 3), 0)]
assert len(d) == 0
with pytest.raises(KeyError):
I.IntervalDict().popitem()
# clear
d = I.IntervalDict([(I.closed(0, 3), 0)])
d.clear()
assert d == I.IntervalDict()
d.clear()
assert d == I.IntervalDict()
# copy
d = I.IntervalDict([(I.closed(0, 3), 0)])
assert d.copy() == d
# find
assert d.find(-1) == I.empty()
assert d.find(0) == I.closed(0, 3)
# init & update & eq
d = I.IntervalDict({I.closed(0, 2): 0, I.closed(4, 5): 1})
assert d == I.IntervalDict([(I.closed(0, 2), 0), (I.closed(4, 5), 1)])
a, b = d.copy(), d.copy()
a.update({I.closed(-1, 1): 2})
b.update([[I.closed(-1, 1), 2]])
assert a != d
assert a == b
assert a != 1
assert a.items() == [(I.closed(-1, 1), 2), (I.openclosed(1, 2), 0), (I.closed(4, 5), 1)]
assert I.IntervalDict([(0, 0), (1, 1)]) == I.IntervalDict([(1, 1), (0, 0)])
def test_intervaldict_get_set():
d = I.IntervalDict()
# Single value
d[I.closed(0, 2)] = 0
assert len(d) == 1
assert d[2] == 0
assert d.get(2) == 0
with pytest.raises(KeyError):
d[3]
assert d.get(3) is None
# Intervals
d = I.IntervalDict([(I.closed(0, 2), 0)])
assert d[I.open(-I.inf, I.inf)].items() == [(I.closed(0, 2), 0)]
assert d[I.closed(0, 2)].items() == [(I.closed(0, 2), 0)]
assert d[I.closed(-1, 0)].items() == [(I.singleton(0), 0)]
assert d[I.closed(-2, -1)].items() == []
assert d.get(I.closed(0, 2)).items() == [(I.closed(0, 2), 0)]
assert d.get(I.closed(-2, -1)).items() == [(I.closed(-2, -1), None)]
assert d.get(I.closed(-1, 0)).items() == [(I.closedopen(-1, 0), None), (I.singleton(0), 0)]
d[I.closed(1, 3)] = 1
assert d.items() == [(I.closedopen(0, 1), 0), (I.closed(1, 3), 1)]
assert len(d) == 2
assert d[0] == 0
assert d.get(0, -1) == 0
assert d[1] == 1
assert d.get(1, -1) == 1
assert d[3] == 1
assert d.get(3, -1) == 1
with pytest.raises(KeyError):
d[4]
assert d.get(4, -1) == -1
# Set values
d = I.IntervalDict([(I.closed(0, 2), 0)])
d[3] = 2
assert d.items() == [(I.closed(0, 2), 0), (I.singleton(3), 2)]
d[3] = 3
assert d.items() == [(I.closed(0, 2), 0), (I.singleton(3), 3)]
d[I.closed(0, 2)] = 1
assert d.items() == [(I.closed(0, 2), 1), (I.singleton(3), 3)]
d[I.closed(-1, 1)] = 2
assert d.items() == [(I.closed(-1, 1), 2), (I.openclosed(1, 2), 1), (I.singleton(3), 3)]
d = I.IntervalDict([(I.closed(0, 2), 0)])
d[I.closed(-1, 4)] = 1
assert d.items() == [(I.closed(-1, 4), 1)]
d[I.closed(5, 6)] = 1
assert d.items() == [(I.closed(-1, 4) | I.closed(5, 6), 1)]
# Delete
d = I.IntervalDict([(I.closed(0, 2), 0)])
del d[1]
with pytest.raises(KeyError):
d[1]
with pytest.raises(KeyError):
del d[3]
d = I.IntervalDict([(I.closed(0, 2), 0)])
del d[I.closed(-1, 1)]
assert d.items() == [(I.openclosed(1, 2), 0)]
del d[I.closed(-10, -9)]
assert d.items() == [(I.openclosed(1, 2), 0)]
del d[I.empty()]
assert d.items() == [(I.openclosed(1, 2), 0)]
# setdefault
d = I.IntervalDict([(I.closed(0, 2), 0)])
assert d.setdefault(-1, default=0) == 0
assert d[-1] == 0
assert d.setdefault(0, default=1) == 0
assert d[0] == 0
d = I.IntervalDict([(I.closed(0, 2), 0)])
t = d.setdefault(I.closed(-2, -1), -1)
assert t.items() == [(I.closed(-2, -1), -1)]
assert d.items() == [(I.closed(-2, -1), -1), (I.closed(0, 2), 0)]
d = I.IntervalDict([(I.closed(0, 2), 0)])
t = d.setdefault(I.closed(-1, 1), 2)
assert t.items() == [(I.closedopen(-1, 0), 2), (I.closed(0, 1), 0)]
assert d.items() == [(I.closedopen(-1, 0), 2), (I.closed(0, 2), 0)]
def test_intervaldict_combine():
add = lambda x, y: x + y
assert I.IntervalDict().combine(I.IntervalDict(), add) == I.IntervalDict()
d = I.IntervalDict([(I.closed(0, 3), 0)])
assert I.IntervalDict().combine(d, add) == d
assert d.combine(I.IntervalDict(), add) == d
d1 = I.IntervalDict([(I.closed(1, 3) | I.closed(5, 7), 1)])
d2 = I.IntervalDict([(I.closed(2, 4) | I.closed(6, 8), 2)])
assert d1.combine(d2, add) == d2.combine(d1, add)
assert d1.combine(d2, add) == I.IntervalDict([
(I.closedopen(1, 2) | I.closedopen(5, 6), 1),
(I.closed(2, 3) | I.closed(6, 7), 3),
(I.openclosed(3, 4) | I.openclosed(7, 8), 2),
])
d1 = I.IntervalDict({
I.closed(0, 1): 2,
I.closed(3, 4): 2
})
d2 = I.IntervalDict({
I.closed(1, 3): 3,
I.closed(4, 5): 1
})
assert d1.combine(d2, add) == d2.combine(d1, add)
assert d1.combine(d2, add) == I.IntervalDict({
I.closedopen(0, 1): 2,
I.singleton(1): 5,
I.open(1, 3): 3,
I.singleton(3): 5,
I.open(3, 4): 2,
I.singleton(4): 3,
I.openclosed(4, 5): 1,
})