class RangeCache(object):
"""
RangeCache is a data structure that tracks a finite set of
ranges (a range is a 2-tuple consisting of a numeric start
and numeric length). New ranges can be added via the `push`
method, and if such a call causes the capacity to be exceeded,
then the "oldest" range is removed. The `get` method implements
an efficient lookup for a single value that may be found within
one of the ranges.
"""
def __init__(self, capacity,
start_key=lambda o: o[0],
length_key=lambda o: o[1]):
"""
@param key: A function that fetches the range start from an item.
"""
super(RangeCache, self).__init__()
self._ranges = SortedCollection(key=start_key)
self._lru = BoundedLRUQueue(capacity, key=start_key)
self._start_key = start_key
self._length_key = length_key
def push(self, o):
"""
Add a range to the cache.
If `key` is not provided to the constructor, then
`o` should be a 3-tuple:
- range start (numeric)
- range length (numeric)
- range item (object)
"""
self._ranges.insert(o)
popped = self._lru.push(o)
if popped is not None:
self._ranges.remove(popped)
def touch(self, o):
self._lru.touch(o)
def get(self, value):
"""
Search for the numeric `value` within the ranges
tracked by this cache.
@raise ValueError: if the value is not found in the range cache.
"""
hit = self._ranges.find_le(value)
if value < self._start_key(hit) + self._length_key(hit):
return hit
raise ValueError("%s not found in range cache" % value)
@staticmethod
def test():
q = RangeCache(2)
x = None
try: x = q.get(0)
except ValueError: pass
assert x is None
x = None
try: x = q.get(1)
except ValueError: pass
assert x is None
q.push((1, 1, [0]))
x = None
try: x = q.get(0)
except ValueError: pass
assert x is None
assert q.get(1) == (1, 1, [0])
assert q.get(1.99) == (1, 1, [0])
x = None
try: x = q.get(2.01)
except ValueError: pass
assert x is None
q.push((3, 1, [1]))
assert q.get(1) == (1, 1, [0])
assert q.get(3) == (3, 1, [1])
q.push((5, 1, [2]))
x = None
try: x = q.get(1)
except ValueError: pass
assert x is None
assert q.get(3) == (3, 1, [1])
assert q.get(5) == (5, 1, [2])
q.touch((3, 1, [1]))
q.push((7, 1, [3]))
assert q.get(3) == (3, 1, [1])
assert q.get(7) == (7, 1, [3])
x = None
try: x = q.get(5)
except ValueError: pass
assert x is None
return True
class RangeSet(object):
# TODO: currently doesn't handle the null range set very well. Should
# introduce a NULL static singelton somehow.
def __init__(self, ranges=tuple()):
# Sort by the start of every range:
self._ranges = SortedCollection(ranges, itemgetter(0))
if ranges:
self._consolidate()
self.begin = self.start = self._ranges[0][0]
self.end = self.stop = self._ranges[-1][1]
self.span = self.end - self.begin + 1
self.coverage = sum(end - begin + 1 for (begin, end) in self._ranges)
else:
self.begin = self.start = self.end = self.stop = None
self.span = self.coverage = 0
def __len__(self):
return len(self._ranges)
def __iter__(self):
return iter(self._ranges)
def __getitem__(self, key):
return self._ranges[key]
def __contains__(self, pos):
try:
begin, end = self._ranges.find_le(pos)
return pos >= begin and pos <= end
except ValueError:
return False
def __add__(self, other):
return RangeSet(list(self) + list(other))
def __or__(self, other):
return self + other
def __and__(self, other):
leftmost = min(self.start, other.start)
rightmost = max(self.stop, other.stop)
return (self.complement(leftmost, rightmost) | other.complement(leftmost, rightmost)).complement(leftmost, rightmost)
def __sub__(self, other):
return self & other.complement(self.start, self.stop)
def __eq__(self, other):
return len(self) == len(other) and all(b1 == b2 and e1 == e2 for (b1, e1), (b2, e2) in zip(self, other))
def __str__(self):
return str(list(self._ranges))
def _consolidate(self):
new_ranges = SortedCollection(key=itemgetter(0))
prev_begin, prev_end = self._ranges[0]
for begin, end in self._ranges[1:]:
if prev_end >= begin - 1:
# Consolidate the previous and current ranges:
prev_end = max(prev_end, end)
else:
# Add the previous range, and continue with the current range
# as the seed for the next iteration:
new_ranges.insert((prev_begin, prev_end))
prev_begin = begin
prev_end = end
new_ranges.insert((prev_begin, prev_end))
self._ranges = new_ranges
def complement(self, begin, end):
if not self:
return RangeSet([(begin, end)])
inter_complement = [(e1+1, b2-1) for (b1, e1), (b2, e2) in zip(self._ranges, self._ranges[1:])]
if begin < self.start:
inter_complement.append((begin, self.start - 1))
if end > self.end:
inter_complement.append((self.end + 1, end))
return RangeSet(inter_complement)
def intersects(self, begin, end):
# TODO: can be optimized
return bool(self & RangeSet([(begin, end)]))
def cut_to(self, begin, end):
# TODO: can be optimized
return self & RangeSet([(begin, end)])
class RangeSet(object):
# TODO: currently doesn't handle the null range set very well. Should
# introduce a NULL static singelton somehow.
def __init__(self, ranges=tuple()):
# Sort by the start of every range:
self._ranges = SortedCollection(ranges, itemgetter(0))
if ranges:
self._consolidate()
self.begin = self.start = self._ranges[0][0]
self.end = self.stop = self._ranges[-1][1]
self.span = self.end - self.begin + 1
self.coverage = sum(end - begin + 1
for (begin, end) in self._ranges)
else:
self.begin = self.start = self.end = self.stop = None
self.span = self.coverage = 0
def __len__(self):
return len(self._ranges)
def __iter__(self):
return iter(self._ranges)
def __getitem__(self, key):
return self._ranges[key]
def __contains__(self, pos):
try:
begin, end = self._ranges.find_le(pos)
return pos >= begin and pos <= end
except ValueError:
return False
def __add__(self, other):
return RangeSet(list(self) + list(other))
def __or__(self, other):
return self + other
def __and__(self, other):
leftmost = min(self.start, other.start)
rightmost = max(self.stop, other.stop)
return (self.complement(leftmost, rightmost)
| other.complement(leftmost, rightmost)).complement(
leftmost, rightmost)
def __sub__(self, other):
return self & other.complement(self.start, self.stop)
def __eq__(self, other):
return len(self) == len(other) and all(
b1 == b2 and e1 == e2 for (b1, e1), (b2, e2) in zip(self, other))
def __str__(self):
return str(list(self._ranges))
def _consolidate(self):
new_ranges = SortedCollection(key=itemgetter(0))
prev_begin, prev_end = self._ranges[0]
for begin, end in self._ranges[1:]:
if prev_end >= begin - 1:
# Consolidate the previous and current ranges:
prev_end = max(prev_end, end)
else:
# Add the previous range, and continue with the current range
# as the seed for the next iteration:
new_ranges.insert((prev_begin, prev_end))
prev_begin = begin
prev_end = end
new_ranges.insert((prev_begin, prev_end))
self._ranges = new_ranges
def complement(self, begin, end):
if not self:
return RangeSet([(begin, end)])
inter_complement = [
(e1 + 1, b2 - 1)
for (b1, e1), (b2, e2) in zip(self._ranges, self._ranges[1:])
]
if begin < self.start:
inter_complement.append((begin, self.start - 1))
if end > self.end:
inter_complement.append((self.end + 1, end))
return RangeSet(inter_complement)
def intersects(self, begin, end):
# TODO: can be optimized
return bool(self & RangeSet([(begin, end)]))
def cut_to(self, begin, end):
# TODO: can be optimized
return self & RangeSet([(begin, end)])
class RangeCache(object):
"""
RangeCache is a data structure that tracks a finite set of
ranges (a range is a 2-tuple consisting of a numeric start
and numeric length). New ranges can be added via the `push`
method, and if such a call causes the capacity to be exceeded,
then the "oldest" range is removed. The `get` method implements
an efficient lookup for a single value that may be found within
one of the ranges.
"""
def __init__(self,
capacity,
start_key=lambda o: o[0],
length_key=lambda o: o[1]):
"""
@param key: A function that fetches the range start from an item.
"""
super(RangeCache, self).__init__()
self._ranges = SortedCollection(key=start_key)
self._lru = BoundedLRUQueue(capacity, key=start_key)
self._start_key = start_key
self._length_key = length_key
def push(self, o):
"""
Add a range to the cache.
If `key` is not provided to the constructor, then
`o` should be a 3-tuple:
- range start (numeric)
- range length (numeric)
- range item (object)
"""
self._ranges.insert(o)
popped = self._lru.push(o)
if popped is not None:
self._ranges.remove(popped)
def touch(self, o):
self._lru.touch(o)
def get(self, value):
"""
Search for the numeric `value` within the ranges
tracked by this cache.
@raise ValueError: if the value is not found in the range cache.
"""
hit = self._ranges.find_le(value)
if value < self._start_key(hit) + self._length_key(hit):
return hit
raise ValueError("%s not found in range cache" % value)
@staticmethod
def test():
q = RangeCache(2)
x = None
try:
x = q.get(0)
except ValueError:
pass
assert x is None
x = None
try:
x = q.get(1)
except ValueError:
pass
assert x is None
q.push((1, 1, [0]))
x = None
try:
x = q.get(0)
except ValueError:
pass
assert x is None
assert q.get(1) == (1, 1, [0])
assert q.get(1.99) == (1, 1, [0])
x = None
try:
x = q.get(2.01)
except ValueError:
pass
assert x is None
q.push((3, 1, [1]))
assert q.get(1) == (1, 1, [0])
assert q.get(3) == (3, 1, [1])
q.push((5, 1, [2]))
x = None
try:
x = q.get(1)
except ValueError:
pass
assert x is None
assert q.get(3) == (3, 1, [1])
assert q.get(5) == (5, 1, [2])
q.touch((3, 1, [1]))
q.push((7, 1, [3]))
assert q.get(3) == (3, 1, [1])
assert q.get(7) == (7, 1, [3])
x = None
try:
x = q.get(5)
except ValueError:
pass
assert x is None
return True
class PatternFrameSchedule(FrameScheduleBase):
'''
Schedules of this type assume that time and frequency are separated out into
a rectangular grid of time/frequency tiles. Time slots need not be adjacent.
Channels in frequency must be adjacent. Time slots cannot overlap, and extend across
all channels. Each pattern of time/frequency/owner allocations is mapped to an index.
The action space stored in _action_space must include at least one time/frequency
allocation pattern or this schedule will not work
'''
stateTup = namedtuple('stateTup', 'time_ref frame_num_ref first_frame_num action_ind epoch_num')
LinkTuple = namedtuple('LinkTuple', 'owner linktype')
varTup = namedtuple('varTup', ('frame_offset tx_time valid tx_gain gains slot_bw schedule_seq max_scheds rf_freq'))
PatternTuple = namedtuple("PatternTuple", 'owner len offset type bb_freq')
gains = None
slot_bw = None
schedule_seq = None
max_scheds = None
# this variable will be tricky: It'll be modified as a class level variable, and won't
# be sent across during pickle/unpickle operations but will rely on the classes on
# remote machines being configured properly
_action_space = None
sync_space = None
num_actions = None
def __init__(self, tx_time=None, frame_offset=None, time_ref=None,
first_frame_num=None, frame_num_ref=None, valid=None,
tx_gain=None, max_schedules=2, action_ind=None,
rf_freq=None, slot_bw=0.0, epoch_num=None):
if tx_time is not None:
self.tx_time = time_spec_t(tx_time)
else:
self.tx_time = None
self.frame_offset = frame_offset
self.valid = valid
# this is the list of schedule states this schedule object knows about.
# The schedules are ordered by first_frame_num
self.schedule_seq = SortedCollection(key=itemgetter(2))
self.max_scheds = max_schedules
# use a default dict so slots with no initialized gain will use the default tx
# gain
self.tx_gain = tx_gain
self.gains = defaultdict(self.constant_factory(self.tx_gain))
# set default values for all controllable parameters. These are what will be used
# if the action space doesn't specify a value
self.rf_freq = rf_freq
self.slot_bw = slot_bw
first_state = (time_ref, frame_num_ref, first_frame_num, action_ind, epoch_num)
# only add the initial state if all the necessary params are defined
if all( v is not None for v in first_state):
self.add_schedule(*first_state)
@staticmethod
def constant_factory(value):
return itertools.repeat(value).next
def add_schedule(self, time_ref, frame_num_ref, first_frame_num, action_ind, epoch_num=None):
'''
Add a schedule to the end of the schedule queue, and if the queue is over
capacity, pop off the oldest element
'''
self.schedule_seq.insert((time_spec_t(time_ref).to_tuple(), frame_num_ref, first_frame_num, action_ind, epoch_num))
if len(self.schedule_seq) > self.max_scheds:
# find the first element in the list when sorted by frame number
self.schedule_seq.remove(self.schedule_seq[0])
def compute_frame(self, frame_num=None):
'''
Given a frame number, produce an individual frame configuration
'''
if frame_num is None:
sched = self.stateTup(*self.schedule_seq[0])
else:
try:
sched_tup = self.schedule_seq.find_le(frame_num)
except ValueError:
sched_tup = self.schedule_seq[0]
sched = self.stateTup(*sched_tup)
#print "Frame num is %i, action ind is %i"%(frame_num, sched.action_ind)
#print "schedule sequence is %s"%self.schedule_seq
action = self._action_space[sched.action_ind]
# if "pattern" not in action:
# # TODO: Make a better exception for when there aren't any patterns
# raise KeyError("Expected at least one pattern object in the action space")
# else:
frame_len = action["frame_len"]
frame_delta = frame_num - sched.frame_num_ref
t0 = time_spec_t(sched.time_ref) + frame_len*frame_delta
frame_config = {"frame_len":frame_len,
"t0":t0,
"t0_frame_num":frame_num,
"first_frame_num":sched.first_frame_num,
"valid":self.valid,
"epoch_num":sched.epoch_num,
}
# get all the parameters needed for computing each slot in frame_config
if "rf_freq" in action:
rf_freq = action["rf_freq"]
else:
rf_freq = self.rf_freq
# get the list of gains per slot
act_slots = action["slots"]
gains = [ self.gains[(s.owner, s.type)] for s in act_slots]
slots = [SlotParamTuple(owner=s.owner, len=s.len, offset=s.offset, type=s.type,
rf_freq=rf_freq, bb_freq=s.bb_freq, bw=self.slot_bw,
tx_gain=gain) for gain, s in zip(gains, act_slots)]
frame_config["slots"] = slots
for s in slots:
if s.type == "beacon":
pass
#print ("frame at time %s beacon slot at offset %f fr freq %f and "
# +"channel %f")%(frame_config["t0"], s.offset, s.rf_freq, s.bb_freq)
return frame_config
def store_current_config(self):
'''
No longer needed
'''
pass
def store_tx_gain(self, owner, linktype, gain):
'''
Update the gain setting for the current and next schedules by owner and link type
'''
self.gains[(owner, linktype)] = gain
def get_unique_links(self, frame_num):
'''
Return a list of unique owner-link type tuples
'''
# get the schedule in effect for frame_num
try:
sched = self.stateTup(*self.schedule_seq.find_le(frame_num))
except ValueError:
# didn't find any frames less than or equal frame number, so return an empty
# list
return list()
action = self._action_space[sched.action_ind]
unique_links = set()
# add links from the 'next' frame config
for s in action["pattern"]["slots"]:
unique_links.add(self.LinkTuple(s.owner, s.type))
return list(unique_links)
def get_uplink_gain(self, owner):
'''
Get the uplink gain associated with an owner
'''
uplink_gain = self.gains[(owner, "uplink")]
return uplink_gain
@property
def time_ref(self):
return time_spec_t(self.schedule_seq[-1][0])
@time_ref.setter
def time_ref(self, value):
# ignore values here until redesign makes this unnecessary
pass
def __getstate__(self):
'''
load all the instance variables into a namedtuple and then return that as
a plain tuple to cut down on the size of the pickled object
'''
try:
inst_vars = self.__dict__.copy()
inst_vars["schedule_seq"] = list(inst_vars["schedule_seq"])
inst_vars["gains"] = dict(inst_vars["gains"])
temp_tup = self.varTup(**inst_vars)
except TypeError:
found_fields = inst_vars.keys()
expected_fields = self.varTup._fields
raise TypeError(("The beacon class does not support adding or removing " +
"variables when pickling. " +
"Found %s, expected %s" % (found_fields, expected_fields)))
return tuple(temp_tup)
def __setstate__(self,b):
'''
load b, which will be a plain tuple, into a namedtuple and then convert that to
this instance's __dict__ attribute
'''
try:
temp_tup = self.varTup(*b)
self.__dict__.update(temp_tup._asdict())
self.schedule_seq = SortedCollection(temp_tup.schedule_seq,
key=itemgetter(2))
self.gains = defaultdict(self.constant_factory(self.tx_gain))
self.gains.update(temp_tup.gains)
except TypeError:
raise TypeError(("The beacon class does not support adding or removing " +
"variables when pickling"))
def __cmp__(self, other):
simp_vals_equal = all([ self.__dict__[key] == val for key,val
in other.__dict__.iteritems()
if (key != "gains") and (key != "schedule_seq")])
gains_equal = dict(self.__dict__["gains"]) == dict(other.__dict__["gains"])
seq_equal = list(self.__dict__["schedule_seq"]) == list(other.__dict__["schedule_seq"])
return all([simp_vals_equal, gains_equal, seq_equal])
def __eq__(self, other):
simp_vals_equal = all([ self.__dict__[key] == val for key,val
in other.__dict__.iteritems()
if (key != "gains") and (key != "schedule_seq")])
gains_equal = dict(self.__dict__["gains"]) == dict(other.__dict__["gains"])
seq_equal = list(self.__dict__["schedule_seq"]) == list(other.__dict__["schedule_seq"])
return all([simp_vals_equal, gains_equal, seq_equal])
def __repr__(self):
s = ["PatternFrameSchedule(",
"frame_offset=%r"%self.frame_offset,
", tx_time=%r"%self.tx_time,
", valid=%r"%self.valid,
", tx_gain=%r"%self.tx_gain,
", gains=%r"%dict(self.gains),
", slot_bw=%r"%self.slot_bw,
", schedule_seq=%r"%list(self.schedule_seq),
", max_scheds=%r"%self.max_scheds,
", rf_freq=%r"%self.rf_freq,
")"]
repr_str = ''.join(s)
return repr_str
@staticmethod
def check_types(slot, fields):
# convert each entry to the correct type
types_valid = True
failed_fields = []
#print "converting row: %s" % row
for field_name in slot._fields:
if not isinstance(getattr(slot, field_name), fields[field_name]):
wrong_type = type(getattr(slot, field_name))
failed_fields.append((field_name, fields[field_name], wrong_type))
types_valid = False
return types_valid, failed_fields
@staticmethod
def load_pattern_set_from_file(pattern_file, set_name, fs):
dev_log = logging.getLogger('developer')
# sanitize path name and pull apart path from base file name
abs_pattern_file = os.path.expandvars(os.path.expanduser(pattern_file))
abs_pattern_file = os.path.abspath(abs_pattern_file)
abs_pattern_dir = os.path.dirname(abs_pattern_file)
pattern_basename = os.path.basename(abs_pattern_file)
if os.path.isdir(abs_pattern_dir):
sys.path.append(abs_pattern_dir)
else:
dev_log.error("pattern directory does not exist: %s",abs_pattern_dir)
return False
try:
sanitized_pattern_file = os.path.splitext(pattern_basename)[0]
group_module = __import__(sanitized_pattern_file)
dev_log.info("using pattern sets from %s", group_module.__file__)
pattern_set = getattr(group_module, set_name)
except ImportError:
dev_log.error("Could not import %s from directory %s",
pattern_basename, abs_pattern_dir)
raise ImportError
except AttributeError:
dev_log.error("Pattern set %s not found in file %s",
set_name, group_module.__file__)
raise AttributeError
slot_fields = dict([("owner",int),
("len",float),
("offset",float),
("type",str),
("rf_freq",float),
("bb_freq",int),
("bw",float),
("tx_gain",float),])
all_rf_freqs_found = True
for m, frame in enumerate(pattern_set):
# check that the pattern set includes rf_frequency for each action
if "rf_freq_ind" not in frame:
dev_log.warning("RF frequency index not specified in action number %i in Pattern set %s in file %s",
m, set_name, group_module.__file__)
all_rf_freqs_found = False
for n, slot in enumerate(frame["slots"]):
types_valid, failed_fields = PatternFrameSchedule.check_types(slot, slot_fields)
if not types_valid:
for failure in failed_fields:
dev_log.warning("Field %s in Slot %i in frame index %i failed field type validation. Type was %s but should be %s",
failure[0], n, m, failure[2], failure[1])
# log an error and raise an exception if there's a missing rf frequency field
if not all_rf_freqs_found:
dev_log.error("At least one action in the pattern file was missing an rf_freq_ind field")
raise AttributeError
# sort slots by order of offset
for m, frame in enumerate(pattern_set):
frame["slots"].sort(key=lambda slot: slot.offset)
frame_len_rounded = round(frame["frame_len"]*fs)/fs
#
# enforce slot/frame boundaries occur at integer samples
#
# check that frame len is at an integer sample
if frame["frame_len"] != frame_len_rounded:
dev_log.warn("rounding frame len from %.15f to %.15f", frame["frame_len"],
frame_len_rounded)
pattern_set[m]["frame_len"] = frame_len_rounded
try:
# do a limited amount of error checking
for ind, frame in enumerate(pattern_set):
for num, slot in enumerate(frame["slots"]):
offset_rounded = round(slot.offset*fs)/fs
len_rounded = round(slot.len*fs)/fs
if slot.offset != offset_rounded:
dev_log.warn("rounding frame %d slot %d offset from %.15f to %.15f",
ind, num, slot.offset,offset_rounded)
if slot.len != len_rounded:
dev_log.warn("rounding frame %d slot %d len from %.15f to %.15f",
ind, num, slot.len, len_rounded)
# more precision fun
end_of_slot = round( (offset_rounded + len_rounded)*fs)/fs
if end_of_slot > frame["frame_len"]:
raise InvalidFrameError(("slot %d with offset %f and len %f extends past " +
"the end of the frame, len %f") % (num, slot.offset,
slot.len, frame["frame_len"]))
pattern_set[ind]["slots"][num] = slot._replace(offset=offset_rounded,
len=len_rounded)
except InvalidFrameError, err:
dev_log.error("Invalid Frame: %s", err)
raise
# self.__class__.pattern_set = deepcopy(pattern_set)
return pattern_set
