class RangeCache(object): """ RangeCache is a data structure that tracks a finite set of ranges (a range is a 2-tuple consisting of a numeric start and numeric length). New ranges can be added via the `push` method, and if such a call causes the capacity to be exceeded, then the "oldest" range is removed. The `get` method implements an efficient lookup for a single value that may be found within one of the ranges. """ def __init__(self, capacity, start_key=lambda o: o[0], length_key=lambda o: o[1]): """ @param key: A function that fetches the range start from an item. """ super(RangeCache, self).__init__() self._ranges = SortedCollection(key=start_key) self._lru = BoundedLRUQueue(capacity, key=start_key) self._start_key = start_key self._length_key = length_key def push(self, o): """ Add a range to the cache. If `key` is not provided to the constructor, then `o` should be a 3-tuple: - range start (numeric) - range length (numeric) - range item (object) """ self._ranges.insert(o) popped = self._lru.push(o) if popped is not None: self._ranges.remove(popped) def touch(self, o): self._lru.touch(o) def get(self, value): """ Search for the numeric `value` within the ranges tracked by this cache. @raise ValueError: if the value is not found in the range cache. """ hit = self._ranges.find_le(value) if value < self._start_key(hit) + self._length_key(hit): return hit raise ValueError("%s not found in range cache" % value) @staticmethod def test(): q = RangeCache(2) x = None try: x = q.get(0) except ValueError: pass assert x is None x = None try: x = q.get(1) except ValueError: pass assert x is None q.push((1, 1, [0])) x = None try: x = q.get(0) except ValueError: pass assert x is None assert q.get(1) == (1, 1, [0]) assert q.get(1.99) == (1, 1, [0]) x = None try: x = q.get(2.01) except ValueError: pass assert x is None q.push((3, 1, [1])) assert q.get(1) == (1, 1, [0]) assert q.get(3) == (3, 1, [1]) q.push((5, 1, [2])) x = None try: x = q.get(1) except ValueError: pass assert x is None assert q.get(3) == (3, 1, [1]) assert q.get(5) == (5, 1, [2]) q.touch((3, 1, [1])) q.push((7, 1, [3])) assert q.get(3) == (3, 1, [1]) assert q.get(7) == (7, 1, [3]) x = None try: x = q.get(5) except ValueError: pass assert x is None return True
class RangeSet(object): # TODO: currently doesn't handle the null range set very well. Should # introduce a NULL static singelton somehow. def __init__(self, ranges=tuple()): # Sort by the start of every range: self._ranges = SortedCollection(ranges, itemgetter(0)) if ranges: self._consolidate() self.begin = self.start = self._ranges[0][0] self.end = self.stop = self._ranges[-1][1] self.span = self.end - self.begin + 1 self.coverage = sum(end - begin + 1 for (begin, end) in self._ranges) else: self.begin = self.start = self.end = self.stop = None self.span = self.coverage = 0 def __len__(self): return len(self._ranges) def __iter__(self): return iter(self._ranges) def __getitem__(self, key): return self._ranges[key] def __contains__(self, pos): try: begin, end = self._ranges.find_le(pos) return pos >= begin and pos <= end except ValueError: return False def __add__(self, other): return RangeSet(list(self) + list(other)) def __or__(self, other): return self + other def __and__(self, other): leftmost = min(self.start, other.start) rightmost = max(self.stop, other.stop) return (self.complement(leftmost, rightmost) | other.complement(leftmost, rightmost)).complement(leftmost, rightmost) def __sub__(self, other): return self & other.complement(self.start, self.stop) def __eq__(self, other): return len(self) == len(other) and all(b1 == b2 and e1 == e2 for (b1, e1), (b2, e2) in zip(self, other)) def __str__(self): return str(list(self._ranges)) def _consolidate(self): new_ranges = SortedCollection(key=itemgetter(0)) prev_begin, prev_end = self._ranges[0] for begin, end in self._ranges[1:]: if prev_end >= begin - 1: # Consolidate the previous and current ranges: prev_end = max(prev_end, end) else: # Add the previous range, and continue with the current range # as the seed for the next iteration: new_ranges.insert((prev_begin, prev_end)) prev_begin = begin prev_end = end new_ranges.insert((prev_begin, prev_end)) self._ranges = new_ranges def complement(self, begin, end): if not self: return RangeSet([(begin, end)]) inter_complement = [(e1+1, b2-1) for (b1, e1), (b2, e2) in zip(self._ranges, self._ranges[1:])] if begin < self.start: inter_complement.append((begin, self.start - 1)) if end > self.end: inter_complement.append((self.end + 1, end)) return RangeSet(inter_complement) def intersects(self, begin, end): # TODO: can be optimized return bool(self & RangeSet([(begin, end)])) def cut_to(self, begin, end): # TODO: can be optimized return self & RangeSet([(begin, end)])
class RangeSet(object): # TODO: currently doesn't handle the null range set very well. Should # introduce a NULL static singelton somehow. def __init__(self, ranges=tuple()): # Sort by the start of every range: self._ranges = SortedCollection(ranges, itemgetter(0)) if ranges: self._consolidate() self.begin = self.start = self._ranges[0][0] self.end = self.stop = self._ranges[-1][1] self.span = self.end - self.begin + 1 self.coverage = sum(end - begin + 1 for (begin, end) in self._ranges) else: self.begin = self.start = self.end = self.stop = None self.span = self.coverage = 0 def __len__(self): return len(self._ranges) def __iter__(self): return iter(self._ranges) def __getitem__(self, key): return self._ranges[key] def __contains__(self, pos): try: begin, end = self._ranges.find_le(pos) return pos >= begin and pos <= end except ValueError: return False def __add__(self, other): return RangeSet(list(self) + list(other)) def __or__(self, other): return self + other def __and__(self, other): leftmost = min(self.start, other.start) rightmost = max(self.stop, other.stop) return (self.complement(leftmost, rightmost) | other.complement(leftmost, rightmost)).complement( leftmost, rightmost) def __sub__(self, other): return self & other.complement(self.start, self.stop) def __eq__(self, other): return len(self) == len(other) and all( b1 == b2 and e1 == e2 for (b1, e1), (b2, e2) in zip(self, other)) def __str__(self): return str(list(self._ranges)) def _consolidate(self): new_ranges = SortedCollection(key=itemgetter(0)) prev_begin, prev_end = self._ranges[0] for begin, end in self._ranges[1:]: if prev_end >= begin - 1: # Consolidate the previous and current ranges: prev_end = max(prev_end, end) else: # Add the previous range, and continue with the current range # as the seed for the next iteration: new_ranges.insert((prev_begin, prev_end)) prev_begin = begin prev_end = end new_ranges.insert((prev_begin, prev_end)) self._ranges = new_ranges def complement(self, begin, end): if not self: return RangeSet([(begin, end)]) inter_complement = [ (e1 + 1, b2 - 1) for (b1, e1), (b2, e2) in zip(self._ranges, self._ranges[1:]) ] if begin < self.start: inter_complement.append((begin, self.start - 1)) if end > self.end: inter_complement.append((self.end + 1, end)) return RangeSet(inter_complement) def intersects(self, begin, end): # TODO: can be optimized return bool(self & RangeSet([(begin, end)])) def cut_to(self, begin, end): # TODO: can be optimized return self & RangeSet([(begin, end)])
class PatternFrameSchedule(FrameScheduleBase): ''' Schedules of this type assume that time and frequency are separated out into a rectangular grid of time/frequency tiles. Time slots need not be adjacent. Channels in frequency must be adjacent. Time slots cannot overlap, and extend across all channels. Each pattern of time/frequency/owner allocations is mapped to an index. The action space stored in _action_space must include at least one time/frequency allocation pattern or this schedule will not work ''' stateTup = namedtuple('stateTup', 'time_ref frame_num_ref first_frame_num action_ind epoch_num') LinkTuple = namedtuple('LinkTuple', 'owner linktype') varTup = namedtuple('varTup', ('frame_offset tx_time valid tx_gain gains slot_bw schedule_seq max_scheds rf_freq')) PatternTuple = namedtuple("PatternTuple", 'owner len offset type bb_freq') gains = None slot_bw = None schedule_seq = None max_scheds = None # this variable will be tricky: It'll be modified as a class level variable, and won't # be sent across during pickle/unpickle operations but will rely on the classes on # remote machines being configured properly _action_space = None sync_space = None num_actions = None def __init__(self, tx_time=None, frame_offset=None, time_ref=None, first_frame_num=None, frame_num_ref=None, valid=None, tx_gain=None, max_schedules=2, action_ind=None, rf_freq=None, slot_bw=0.0, epoch_num=None): if tx_time is not None: self.tx_time = time_spec_t(tx_time) else: self.tx_time = None self.frame_offset = frame_offset self.valid = valid # this is the list of schedule states this schedule object knows about. # The schedules are ordered by first_frame_num self.schedule_seq = SortedCollection(key=itemgetter(2)) self.max_scheds = max_schedules # use a default dict so slots with no initialized gain will use the default tx # gain self.tx_gain = tx_gain self.gains = defaultdict(self.constant_factory(self.tx_gain)) # set default values for all controllable parameters. These are what will be used # if the action space doesn't specify a value self.rf_freq = rf_freq self.slot_bw = slot_bw first_state = (time_ref, frame_num_ref, first_frame_num, action_ind, epoch_num) # only add the initial state if all the necessary params are defined if all( v is not None for v in first_state): self.add_schedule(*first_state) @staticmethod def constant_factory(value): return itertools.repeat(value).next def add_schedule(self, time_ref, frame_num_ref, first_frame_num, action_ind, epoch_num=None): ''' Add a schedule to the end of the schedule queue, and if the queue is over capacity, pop off the oldest element ''' self.schedule_seq.insert((time_spec_t(time_ref).to_tuple(), frame_num_ref, first_frame_num, action_ind, epoch_num)) if len(self.schedule_seq) > self.max_scheds: # find the first element in the list when sorted by frame number self.schedule_seq.remove(self.schedule_seq[0]) def compute_frame(self, frame_num=None): ''' Given a frame number, produce an individual frame configuration ''' if frame_num is None: sched = self.stateTup(*self.schedule_seq[0]) else: try: sched_tup = self.schedule_seq.find_le(frame_num) except ValueError: sched_tup = self.schedule_seq[0] sched = self.stateTup(*sched_tup) #print "Frame num is %i, action ind is %i"%(frame_num, sched.action_ind) #print "schedule sequence is %s"%self.schedule_seq action = self._action_space[sched.action_ind] # if "pattern" not in action: # # TODO: Make a better exception for when there aren't any patterns # raise KeyError("Expected at least one pattern object in the action space") # else: frame_len = action["frame_len"] frame_delta = frame_num - sched.frame_num_ref t0 = time_spec_t(sched.time_ref) + frame_len*frame_delta frame_config = {"frame_len":frame_len, "t0":t0, "t0_frame_num":frame_num, "first_frame_num":sched.first_frame_num, "valid":self.valid, "epoch_num":sched.epoch_num, } # get all the parameters needed for computing each slot in frame_config if "rf_freq" in action: rf_freq = action["rf_freq"] else: rf_freq = self.rf_freq # get the list of gains per slot act_slots = action["slots"] gains = [ self.gains[(s.owner, s.type)] for s in act_slots] slots = [SlotParamTuple(owner=s.owner, len=s.len, offset=s.offset, type=s.type, rf_freq=rf_freq, bb_freq=s.bb_freq, bw=self.slot_bw, tx_gain=gain) for gain, s in zip(gains, act_slots)] frame_config["slots"] = slots for s in slots: if s.type == "beacon": pass #print ("frame at time %s beacon slot at offset %f fr freq %f and " # +"channel %f")%(frame_config["t0"], s.offset, s.rf_freq, s.bb_freq) return frame_config def store_current_config(self): ''' No longer needed ''' pass def store_tx_gain(self, owner, linktype, gain): ''' Update the gain setting for the current and next schedules by owner and link type ''' self.gains[(owner, linktype)] = gain def get_unique_links(self, frame_num): ''' Return a list of unique owner-link type tuples ''' # get the schedule in effect for frame_num try: sched = self.stateTup(*self.schedule_seq.find_le(frame_num)) except ValueError: # didn't find any frames less than or equal frame number, so return an empty # list return list() action = self._action_space[sched.action_ind] unique_links = set() # add links from the 'next' frame config for s in action["pattern"]["slots"]: unique_links.add(self.LinkTuple(s.owner, s.type)) return list(unique_links) def get_uplink_gain(self, owner): ''' Get the uplink gain associated with an owner ''' uplink_gain = self.gains[(owner, "uplink")] return uplink_gain @property def time_ref(self): return time_spec_t(self.schedule_seq[-1][0]) @time_ref.setter def time_ref(self, value): # ignore values here until redesign makes this unnecessary pass def __getstate__(self): ''' load all the instance variables into a namedtuple and then return that as a plain tuple to cut down on the size of the pickled object ''' try: inst_vars = self.__dict__.copy() inst_vars["schedule_seq"] = list(inst_vars["schedule_seq"]) inst_vars["gains"] = dict(inst_vars["gains"]) temp_tup = self.varTup(**inst_vars) except TypeError: found_fields = inst_vars.keys() expected_fields = self.varTup._fields raise TypeError(("The beacon class does not support adding or removing " + "variables when pickling. " + "Found %s, expected %s" % (found_fields, expected_fields))) return tuple(temp_tup) def __setstate__(self,b): ''' load b, which will be a plain tuple, into a namedtuple and then convert that to this instance's __dict__ attribute ''' try: temp_tup = self.varTup(*b) self.__dict__.update(temp_tup._asdict()) self.schedule_seq = SortedCollection(temp_tup.schedule_seq, key=itemgetter(2)) self.gains = defaultdict(self.constant_factory(self.tx_gain)) self.gains.update(temp_tup.gains) except TypeError: raise TypeError(("The beacon class does not support adding or removing " + "variables when pickling")) def __cmp__(self, other): simp_vals_equal = all([ self.__dict__[key] == val for key,val in other.__dict__.iteritems() if (key != "gains") and (key != "schedule_seq")]) gains_equal = dict(self.__dict__["gains"]) == dict(other.__dict__["gains"]) seq_equal = list(self.__dict__["schedule_seq"]) == list(other.__dict__["schedule_seq"]) return all([simp_vals_equal, gains_equal, seq_equal]) def __eq__(self, other): simp_vals_equal = all([ self.__dict__[key] == val for key,val in other.__dict__.iteritems() if (key != "gains") and (key != "schedule_seq")]) gains_equal = dict(self.__dict__["gains"]) == dict(other.__dict__["gains"]) seq_equal = list(self.__dict__["schedule_seq"]) == list(other.__dict__["schedule_seq"]) return all([simp_vals_equal, gains_equal, seq_equal]) def __repr__(self): s = ["PatternFrameSchedule(", "frame_offset=%r"%self.frame_offset, ", tx_time=%r"%self.tx_time, ", valid=%r"%self.valid, ", tx_gain=%r"%self.tx_gain, ", gains=%r"%dict(self.gains), ", slot_bw=%r"%self.slot_bw, ", schedule_seq=%r"%list(self.schedule_seq), ", max_scheds=%r"%self.max_scheds, ", rf_freq=%r"%self.rf_freq, ")"] repr_str = ''.join(s) return repr_str @staticmethod def check_types(slot, fields): # convert each entry to the correct type types_valid = True failed_fields = [] #print "converting row: %s" % row for field_name in slot._fields: if not isinstance(getattr(slot, field_name), fields[field_name]): wrong_type = type(getattr(slot, field_name)) failed_fields.append((field_name, fields[field_name], wrong_type)) types_valid = False return types_valid, failed_fields @staticmethod def load_pattern_set_from_file(pattern_file, set_name, fs): dev_log = logging.getLogger('developer') # sanitize path name and pull apart path from base file name abs_pattern_file = os.path.expandvars(os.path.expanduser(pattern_file)) abs_pattern_file = os.path.abspath(abs_pattern_file) abs_pattern_dir = os.path.dirname(abs_pattern_file) pattern_basename = os.path.basename(abs_pattern_file) if os.path.isdir(abs_pattern_dir): sys.path.append(abs_pattern_dir) else: dev_log.error("pattern directory does not exist: %s",abs_pattern_dir) return False try: sanitized_pattern_file = os.path.splitext(pattern_basename)[0] group_module = __import__(sanitized_pattern_file) dev_log.info("using pattern sets from %s", group_module.__file__) pattern_set = getattr(group_module, set_name) except ImportError: dev_log.error("Could not import %s from directory %s", pattern_basename, abs_pattern_dir) raise ImportError except AttributeError: dev_log.error("Pattern set %s not found in file %s", set_name, group_module.__file__) raise AttributeError slot_fields = dict([("owner",int), ("len",float), ("offset",float), ("type",str), ("rf_freq",float), ("bb_freq",int), ("bw",float), ("tx_gain",float),]) all_rf_freqs_found = True for m, frame in enumerate(pattern_set): # check that the pattern set includes rf_frequency for each action if "rf_freq_ind" not in frame: dev_log.warning("RF frequency index not specified in action number %i in Pattern set %s in file %s", m, set_name, group_module.__file__) all_rf_freqs_found = False for n, slot in enumerate(frame["slots"]): types_valid, failed_fields = PatternFrameSchedule.check_types(slot, slot_fields) if not types_valid: for failure in failed_fields: dev_log.warning("Field %s in Slot %i in frame index %i failed field type validation. Type was %s but should be %s", failure[0], n, m, failure[2], failure[1]) # log an error and raise an exception if there's a missing rf frequency field if not all_rf_freqs_found: dev_log.error("At least one action in the pattern file was missing an rf_freq_ind field") raise AttributeError # sort slots by order of offset for m, frame in enumerate(pattern_set): frame["slots"].sort(key=lambda slot: slot.offset) frame_len_rounded = round(frame["frame_len"]*fs)/fs # # enforce slot/frame boundaries occur at integer samples # # check that frame len is at an integer sample if frame["frame_len"] != frame_len_rounded: dev_log.warn("rounding frame len from %.15f to %.15f", frame["frame_len"], frame_len_rounded) pattern_set[m]["frame_len"] = frame_len_rounded try: # do a limited amount of error checking for ind, frame in enumerate(pattern_set): for num, slot in enumerate(frame["slots"]): offset_rounded = round(slot.offset*fs)/fs len_rounded = round(slot.len*fs)/fs if slot.offset != offset_rounded: dev_log.warn("rounding frame %d slot %d offset from %.15f to %.15f", ind, num, slot.offset,offset_rounded) if slot.len != len_rounded: dev_log.warn("rounding frame %d slot %d len from %.15f to %.15f", ind, num, slot.len, len_rounded) # more precision fun end_of_slot = round( (offset_rounded + len_rounded)*fs)/fs if end_of_slot > frame["frame_len"]: raise InvalidFrameError(("slot %d with offset %f and len %f extends past " + "the end of the frame, len %f") % (num, slot.offset, slot.len, frame["frame_len"])) pattern_set[ind]["slots"][num] = slot._replace(offset=offset_rounded, len=len_rounded) except InvalidFrameError, err: dev_log.error("Invalid Frame: %s", err) raise # self.__class__.pattern_set = deepcopy(pattern_set) return pattern_set