class LogSystem: def __init__(self): self.map = SortedDict(list) self.gra = { 'Year': 5, 'Month':8, 'Day':11, 'Hour': 14, 'Minute':17, 'Second':20, } def put(self, id: int, timestamp: str) -> None: self.map.setdefault(timestamp, []).append(id) def retrieve(self, start: str, end: str, granularity: str) -> List[int]: idx = self.gra[granularity] left = self.map.bisect_left(start[:idx]) result = [] for i in range(left, len(self.map)): # DO NOT USE bisect to find right, it may only granular to year, but date exceeds key = self.map.keys()[i] if key[:idx]>end[:idx]: break result.extend(self.map[key]) return result
def frequency(sample): letters = alpha_only(sample) # encode to washers l2n = alphanum.base64_to_data n2w = washers.data_to_base4 washer_numbers = n2w(l2n(letters)) freq = SortedDict() max_width = max(map(lambda x: len(x), washer_numbers)) for segment in washer_numbers: # normalize widths width = len(segment) for i in range(max_width - width): segment.insert(0, 0) # update frequency for washer in segment: freq.setdefault(washer, 0) freq[washer] += 1 return freq
def get_obspyh5_index(src_file, seeds_only=False): """Scrape the index (only) from an obspyh5 file. :param src_file: Name of file to extract index from :type src_file: str or pathlib.Path :param seeds_only: If True, only get the seed IDs of the traces. Otherwise (default), get full index. :type seeds_only: bool :return: Sorted dictionary with index of waveforms in the file :rtype: sortedcontainers.SortedDict """ # We use SortedSet rather than SortedList for the inner container # because it allows set operations that are useful, such as finding # event IDs for events which are common across multiple stations. assert is_obspyh5(src_file), '{} is not an obspyh5 file'.format(src_file) index = SortedDict() with h5py.File(src_file, mode='r') as h5f: root = h5f['/waveforms'] if seeds_only: return SortedList(root.keys()) # end if for seedid, station_grp in root.items(): for event_grp in station_grp.values(): evid = None for channel in event_grp.values(): evid = channel.attrs['event_id'] break # end for if evid: index.setdefault(seedid, SortedSet()).add(evid) # end if # end for # end for # end with return index
def collect_misses(parsed_lexemes: Iterable) -> Dict: misses = SortedDict() for lexeme in parsed_lexemes: for sublexeme in lexeme: for segment in sublexeme: for sm in segment[1]: if sm.seg_type == 'miss': misses.setdefault(sm.segment.lower(), default=SortedSet()).add(sm.lexeme) return misses
def sort_data_on_timestamp(mDict): sd = SortedDict() for key in mDict: for index in range(len(mDict[key])): if len(mDict[key][index]) == 3: sd.setdefault(int(mDict[key][index][0]), []).\ append((key, mDict[key][index][1], -1, 0)) elif len(mDict[key][index]) == 4: sd.setdefault(int(mDict[key][index][0]), []).\ append((key, mDict[key][index][1], mDict[key][index][3], 1)) return sd
def compute_pagerank(urls, inlinks, outlinks, b=.85, iters=20): """ Return a dictionary mapping each url to its PageRank. The formula is R(u) = (1/N)(1-b) + b * (sum_{w in B_u} R(w) / (|F_w|) Initialize all scores to 1.0. Params: urls.......SortedList of urls (names) inlinks....SortedDict mapping url to list of in links (backlinks) outlinks...Sorteddict mapping url to list of outlinks Returns: A SortedDict mapping url to its final PageRank value (float) >>> urls = SortedList(['a', 'b', 'c']) >>> inlinks = SortedDict({'a': ['c'], 'b': set(['a']), 'c': set(['a', 'b'])}) >>> outlinks = SortedDict({'a': ['b', 'c'], 'b': set(['c']), 'c': set(['a'])}) >>> sorted(compute_pagerank(urls, inlinks, outlinks, b=.5, iters=0).items()) [('a', 1.0), ('b', 1.0), ('c', 1.0)] >>> iter1 = compute_pagerank(urls, inlinks, outlinks, b=.5, iters=1) >>> iter1['a'] # doctest:+ELLIPSIS 0.6666... >>> iter1['b'] # doctest:+ELLIPSIS 0.333... """ ###TODO """ R(u) = (1 - b)/N + b * sum( inlinks of u/outlink-number) """ Ru = SortedDict() Rv = SortedDict() size = len(urls) # Initialize to 1.0 all URL's for k in urls: Ru.setdefault(k, 1.0) # Page Rank definition for i in range(iters): for url in urls: try: Ru[url] = ((1-b)/size) + b * sum([Ru[x]/len(outlinks[x]) for x in inlinks[url] if len(outlinks[x])]) except: pass return Ru
def kWeakestRows(self, mat: List[List[int]], k: int) -> List[int]: dic = SortedDict() for idx, row in enumerate(mat): sm = sum(row) if sm in dic: dic[sm].append(idx) else: dic.setdefault(sm, [idx]) ans = [] k1 = k while k > 0: ls_curr = dic.popitem(0)[1] ans.extend(ls_curr) k -= len(ls_curr) return ans[:k1]
def testfunc(array, size): houses = SortedDict() for i in range(size): tier = array[i] houses.setdefault(tier, []).append(i) dist = 0 oldsasha = 0 olddima = 0 for key in houses.keys(): sasha, dima = houses[key] dist += abs(sasha - oldsasha) +\ abs(dima - olddima) oldsasha = sasha olddima = dima return dist
def read_links(path): """ Read the html pages in the data folder. Create and return two SortedDicts: inlinks: maps from a name to a SortedSet of names that link to it. outlinks: maps from a name to a SortedSet of names that it links to. For example: inlinks['Ada_Lovelace'] = SortedSet(['Charles_Babbage', 'David_Gelernter'], key=None, load=1000) outlinks['Ada_Lovelace'] = SortedSet(['Alan_Turing', 'Charles_Babbage'], key=None, load=1000) You should use the read_names and get_links function above. Params: path...the name of the data directory ('data') Returns: A (inlinks, outlinks) tuple, as defined above (i.e., two SortedDicts) """ ###TODO inlinks = SortedDict() outlinks = SortedDict() names_dict = read_names(path) for page in names_dict: with open(path + os.sep + page, 'r') as fp: html_page = fp.readlines() # Get links temp_links = get_links(names_dict, str(html_page)) # remove self link without checking, may save some time try: temp_links.remove(page) except: pass # Count outlinks outlinks[page] = temp_links inlinks.setdefault(page,SortedSet()) # count inlinks from same single page-name for link in outlinks[page]: inlinks.setdefault(link,SortedSet()).add(page) return inlinks, outlinks
def find_closest_elements(sorted_arrays): min_distance_so_far = float('inf') iters = SortedDict() for idx, sorted_array in enumerate(sorted_arrays): it = iter(sorted_array) first_min = next(it, None) if first_min is not None: iters.setdefault((first_min, idx), default=it) while True: min_value, min_idx = iters.peekitem(index=0)[0] max_value = iters.peekitem()[0][0] min_distance_so_far = min(max_value - min_value, min_distance_so_far) it = iters.popitem(index=0)[1] next_min = next(it, None) if next_min is None: return min_distance_so_far iters.setdefault((next_min, min_idx), it)
def test_SortedDict(self): # construct sorted_dict = SortedDict({'a': 1, 'c': 2, 'b': 3}) print('sorted dict is: ', sorted_dict) # adding key => value pairs sorted_dict['d'] = 3 print('sorted dict after adding an element: ', sorted_dict) # adding element using setdefault() sorted_dict.setdefault('e', 4) print('sorted dict after setdefault(): ', sorted_dict) # using the get function print('using the get function to print the value of a: ', sorted_dict.get('a', 0)) for key in sorted_dict: print('{} -> {}'.format(key, sorted_dict[key]), end=' ') print() # removing all elements from the dict sorted_dict.clear()
def order_asteroids(): polar_asteroids = SortedDict() for a in asteroids: r, t = to_polar((a[0] - max_pos[0], a[1] - max_pos[1])) polar_asteroids.setdefault(t, SortedDict())[r] = a while polar_asteroids: for t, rs in list(polar_asteroids.items()): a = rs.pop(rs.keys()[0]) if not rs: del polar_asteroids[t] yield a
def _wb_init(path: str) -> tuple[SortedDict[str, wordbook_entry], SortedDict[str, list[wordbook_entry]]]: ''' Initializes word book data: returns dicts of by-word matches and by-meaning matches. ''' def old(): age_s = time.time() - os.stat(path).st_mtime return age_s > 60 * 60 * 24 * 10 if not os.path.isfile(path) or old(): import requests print('getting wordbook.', end='') response = requests.get( 'https://docs.google.com/spreadsheets/d/1y8_11RDvuCRyUK_MXj5K7ZjccgCUDapsPDI5PjaEkMw/gviz/tq?tqx=out:csv&sheet=Wordbook') print('.\r', end='') response.raise_for_status() with open(path, 'wb') as f: f.write(response.content) print('getting wordbook... done') with open(resource_filename('oe_edit.resources', 'wordbook.bin'), 'rb') as fbase: with open(path, newline='', encoding='U8') as fcsv: reader = iter(csv.reader(fcsv)) next(reader) kfa = pkind.findall def csv_entry(data): return wordbook_entry([data[0], '; '.join(filter( None, map(str.strip, data[2].split('᛫')))), '|'.join(kfa(data[2]))]) # l = list of entries l = [*map(wordbook_entry, zip(*([iter(pickle.load(fbase))] * 3))), *map(csv_entry, reader)] sl = str.lower wfa = pword.findall word_match = SortedDict([(sl(e.word), e) for e in l]) mean_match = SortedDict() for (k, v) in chain.from_iterable( [[(sl(m[2]), e) for m in wfa(e.meaning)] for e in l]): mean_match.setdefault(k, []).append(v) return word_match, mean_match
def top_k_frequent(words: Iterable[str], k: int) -> Iterable[str]: frequency_list = [] frequency_map = SortedDict() for word in words: frequency_map[word] = frequency_map.setdefault(word, 0) + 1 for word, count in frequency_map.items(): print(f"word:{word},count:{count}") frequency_list.append(WordFrequency(word=word, frequency=count)) frequency_list = sorted(frequency_list, key=lambda x: x[1], reverse=True) frequency_list = order_alphabetically(frequency_list) return [x.word for x in frequency_list][:k]
class ExponentiallyDecayingReservoir(object): def __init__(self, size=_DEFAULT_SIZE, alpha=_DEFAULT_ALPHA): self.size = size self.alpha = alpha self.start_time = time() self.next_scale_time = self.start_time + _RESCALE_THRESHOLD self.values = SortedDict() def _rescale_if_needed(self): now = time() if now > self.next_scale_time: self.next_scale_time = now + _RESCALE_THRESHOLD old_start_time = self.start_time self.start_time = now scaling_factor = exp(-self.alpha * (self.start_time - old_start_time)) self.values = SortedDict( (priority * scaling_factor, _WeightedSample(sample.value, sample.weight * scaling_factor)) for priority, sample in self.values.items() ) def update(self, value): self._rescale_if_needed() timestamp = time() item_weight = exp(self.alpha * (timestamp - self.start_time)) sample = _WeightedSample(value, item_weight) priority = item_weight / random() if len(self.values) < self.size: self.values[priority] = sample else: first_key = next(self.values.iterkeys()) if first_key < priority: self.values.setdefault(priority, sample) del self.values[first_key] def get_snapshot(self): return _WeightedSnapshot(self.values.values())
def brightestPosition(self, lights): sd = SortedDict() # 差分记录开闭区间,将右边改为开区间方便处理 for pos, r in lights: st, ed = pos - r, pos + r + 1 sd.setdefault(st, 0) sd.setdefault(ed, 0) sd[st] += 1 sd[ed] -= 1 res = int(-1e9) tot, max_ = 0, 0 entrys = sd.items() # 遍历 map 得到最优位置 for pos, cnt in entrys: tot += cnt if tot > max_: max_ = tot res = pos return res
class PriorityQueue: def __init__(self): self.__buckets__ = SortedDict() return def enqueue(self, priority, value): bucket = self.__buckets__.setdefault(priority, []) bucket.append(value) return def dequeue(self): if len(self.__buckets__) > 0: (priority, bucket) = self.__buckets__.peekitem(0) value = bucket[0] if len(bucket) > 1: self.__buckets__[priority] = bucket[1:] else: del self.__buckets__[priority] return (priority, value) else: return None def clear(self): self.__buckets__.clear() return def size(self): total = 0 for bucket in self.__buckets__.values(): total += len(bucket) return total def count(self, priority): return len( self.__buckets__[priority]) if priority in self.__buckets__ else 0
from sortedcontainers import SortedDict for case in range(1, 1 + int(raw_input())): N, K = map(int, raw_input().split(' ')) blocks = SortedDict({N: 1}) while K: b, n = blocks.popitem() if n >= K: print "Case #%d: %d %d" % (case, b / 2, (b - 1) / 2) break blocks.setdefault(b / 2, 0) blocks[b / 2] += n blocks.setdefault((b - 1) / 2, 0) blocks[(b - 1) / 2] += n K -= n
s = open(outFile, "w+") with open(i) as f: tag = f.readline() header = tag if tag.strip( ): # strip will remove all leading and trailing whitespace such as '\n' or ' ' by default tag = tag.strip("\n ' '") tag = tag.split()[6].split(',')[0] sd = SortedDict() for line in f: if line[0] != '#': tokens = line.split('\t') if (int(tokens[2]) != OUT_OF_RANGE_CODE): if (int(tokens[0]) >= int(vals[0]) and int(tokens[0]) <= int(vals[1])): sd.setdefault(tokens[0], {}).setdefault( tokens[1], []).append(tokens[2].rstrip('\n')) for x in logs: if (x != i): with open(x) as w: find = w.readline() if find.strip( ): # strip will remove all leading and trailing whitespace such as '\n' or ' ' by default find = find.strip("\n ' '") find = find.split()[6].split(',')[0] for row in w: if row[0] != '#': token = row.split('\t') if (token[1] == tag): if (int(token[2]) != OUT_OF_RANGE_CODE): if (int(token[0]) >= int(vals[0]) and int(token[0]) <= int(vals[1])):
def out_degree(self, node=None, begin=None, end=None, delta=False): """Return the out-degree of a specified node between time begin and end. Parameters ---------- node : Nodes can be, for example, strings or numbers, optional. Nodes must be hashable (and not None) Python objects. begin : int or float, optional (default= beginning of the entire interval graph) Inclusive beginning time of the edge appearing in the interval graph. end : int or float, optional (default= end of the entire interval graph) Non-inclusive ending time of the edge appearing in the interval graph. Returns ------- Integer value of degree of specified node. If no node is specified, returns float mean degree value of graph. If delta is True, return list of tuples. First indicating the time a degree change occurred, Second indicating the degree after the change occured Examples -------- >>> G = IntervalDiGraph() >>> G.add_edge(1, 2, 3, 5) >>> G.add_edge(2, 3, 8, 11) >>> G.degree(2) 1 >>> G.out_degree(2,2) 1 >>> G.out_degree(2,end=8) 0 >>> G.out_degree() 0.666666 >>> G.out_degree(2,delta=True) [(8, 1)] """ # no specified node, return mean degree if node == None: n = 0 l = 0 for node in self.nodes(begin=begin, end=end): n += 1 l += self.out_degree(node, begin=begin, end=end) return l / n # specified node, no degree_change, return degree if delta == False: return len(self.edges(u=node, begin=begin, end=end)) # delta == True, return list of changes if begin == None: begin = self.tree.begin if end == None: end = self.tree.end current_degree = self.out_degree(node, begin=begin, end=begin) sd = SortedDict() output = [] # for each edge determine if the begin and/or end value is in specified time period for edge in self.edges(u=node, begin=begin, end=end): if edge[2] >= begin: # if begin is in specified time period, add to SortedDict, with +1 to indicate begin sd.setdefault((edge[2], 1), []).append((edge[0], edge[1])) if edge[3] < end: # if begin is in specified time period, add to SortedDict, with -1 to indicate begin sd.setdefault((edge[3], -1), []).append((edge[0], edge[1])) for time in sd: for edge in sd[time]: # iterate through SortedDict, only advancing current degree if edge was not counted on init if time[0] != begin: current_degree += time[1] print(time, begin, edge, current_degree) output.append((time[0], current_degree)) return sorted(output)
class OrderBookSide: def __init__(self, asc, events: Queue): self.asc = asc self.events = events self.orders_map = {} self.levels_map = SortedDict() def add_order(self, order): self.levels_map.setdefault(order.price, []).append(order) self.orders_map[order.id] = order def match_order(self, order): remove_list = [] for level in self.__iterate_levels(): if self.__compare_price(order.price, level): self.__match_orders(order, self.levels_map[level], remove_list) if order.is_matched(): self.events.put({ 'name': 'complete', 'order_id': order.id, }) break else: break # Remove orders after the iteration is complete (so we do not modify the list while iterating over) for o in remove_list: self.__remove_order(o) return order def __match_orders(self, order, orders, remove_list): for o in orders: transferred_amount = o.transfer_amount(order) self.events.put({ 'name': 'match', 'amount': transferred_amount, 'order_id': order.id, 'matched_order_id': o.id, }) if o.is_matched(): remove_list.append(o) self.events.put({ 'name': 'complete', 'order_id': o.id, }) if order.is_matched(): break def __remove_order(self, order): level = self.levels_map[order.price] level.remove(order) if len(level) == 0: self.levels_map.pop(order.price) def __compare_price(self, order_price, book_price): if self.asc: return order_price >= book_price else: return order_price <= book_price def __iterate_levels(self): if self.asc: return iter(self.levels_map) else: return reversed(self.levels_map) def cancel_order_by_id(self, order_id): order = self.orders_map.get(order_id) if order is None: return self.__remove_order(order) self.events.put({ 'name': 'cancelled', 'order_id': order.id, 'remaining_amount': order.amount_to_match(), }) def orders(self): return [o for l in self.levels_map for o in self.levels_map[l]]
def collect_changed_files(self, ticket_frame: pd.DataFrame): changed_files = SortedDict() for i, files in enumerate(ticket_frame.ChangedFiles): for f in files: changed_files.setdefault(f, []).append(i) return changed_files
class NetworkEventDataset: """Collection of 3-channel ZNE streams with traces aligned to a fixed time window about seismic P-wave arrival events, *for a given network*. Two indexes are provided. One indexes hierarchically by station code and event ID, yielding a 3-channel ZNE stream per event, so that you can easily gather all traces for a given station by iterating over events. The other index indexes hierarchically by event ID and station code, yielding a 3-channel ZNE stream per station. Using this index you can easily gather all traces for a given event across multiple stations. Preferably each input trace will already have an 'event_id' attribute in its stats. If not, an event ID will be invented based on station identifiers and time window. """ def __init__(self, stream_src, network=None, station=None, location='', ordering='ZNE'): """ Initialize from data source (file or obspy.Stream). Traces are COPIED into the dataset in order to leave input object intact, since many obspy functions mutate traces in-place. All streams in the input data source stream_src are expected to belong to the same network. This is checked as the data is ingested. A discrepant network code is an error condition. :param stream_src: Source of input streams. May be a file name or an Obspy Stream :type stream_src: str, pathlib.Path or obspy.Stream :param network: Network code of streams to load. If stream_src is an Obspy Stream, the \ streams will be filtered to match this network code. :type network: str :param station: Station code of streams to load. If stream_src is an Obspy Stream, the \ streams will be filtered to match this station code. :type station: str :param location: [OPTIONAL] Location code of streams to load. Leave as default (empty string) \ if location code is empty in the data source. :type location: str :param ordering: Channel ordering to be applied to the data after loading. The channel labelling \ must be consistent with the requested ordering - rotation to the coordinate system implied \ by the ordering is *NOT* applied. :type ordering: str :raises AssertionError: If discrepant network code is found in input data """ if isinstance(stream_src, obspy.Stream): net = network sta = station loc = location or None if net or sta or loc: data_src = stream_src.select(net, sta, loc) else: data_src = stream_src # end if elif os.path.isfile(stream_src): data_src = read_h5_stream(stream_src, network, station, location) else: assert False, "Unknown data source {}".format(type(stream_src)) # end if self.network = network # Data in data_src collects all traces together under a single Stream object. # In order to get control over data slicing and traceability in processing, we # break it down into one Stream per ZNE channel triplet of a given event. self.db_sta = SortedDict() for tr in data_src: net, sta, loc, _ = tr.id.split('.') if self.network: assert net == self.network else: self.network = net # end if # Create single copy of the trace to be shared by both dicts. dupe_trace = tr.copy() try: event_id = tr.stats.event_id except AttributeError: event_id = '.'.join([ net, sta, loc, '_'.join([str(tr.stats.starttime), str(tr.stats.endtime)]) ]) # end try self.db_sta.setdefault(sta, SortedDict()).setdefault( event_id, obspy.Stream()).append(dupe_trace) # end for # Index same obspy.Stream instances in event dict. This way, any changes # to a given event stream will be seen by both indexes. self.db_evid = SortedDict() for sta, ev_db in self.db_sta.items(): for evid, stream in ev_db.items(): self.db_evid.setdefault(evid, SortedDict())[sta] = stream # end for # end for # Sort each stream into specific order. if ordering.upper() == 'ZNE': ordinal = zne_order elif ordering.upper() == 'ZRT': ordinal = zrt_order else: ordinal = None # end if if ordinal is not None: self.apply(lambda x: x.traces.sort(key=ordinal)) # end if # end func def __iter__(self): """ Flat iterator. Loops over self.db_sta depth first and returns tuple of keys and matching stream. Equivalent to:: ```Python for sta, ev_db in self.db_sta.items(): for evid, stream in ev_db.items(): yield (sta, evid, stream) ``` """ return ((sta, evid, stream) for sta, ev_db in self.db_sta.items() for evid, stream in ev_db.items()) # end if def __len__(self): """Returns number of streams""" return sum((len(x) for x in self.db_sta.values())) # end func def __repr__(self): """Displays summary string for all streams""" return '\n'.join( (evid + ', ' + str(stream) for _, evid, stream in iter(self))) # end func def num_stations(self): """ Get number of stations in the dataset. :return: Number of stations :rtype: int """ return len(self.db_sta) # end func def station(self, station_code): """ Accessor for events for a given station. :param station_code: Station to get :type station_code: str :return: Event index for station, if station is found :rtype: SortedDict """ return self.db_sta.get(station_code) # end func def num_events(self): """ Get number of events in the dataset. :return: Number of events :rtype: int """ return len(self.db_evid) # end func def event(self, event_id): """ Accessor for stations for a given event. :param event_id: ID of event to look up :type event_id: str :return: Station index for given event, if event ID is found, otherwise None :rtype: SortedDict or NoneType """ return self.db_evid.get(event_id) # end func def curate(self, curator): """ Curate the dataset according to a callable curator. Modifies collection in-place to remove streams that do not satisfy the curation criteria of the callable. Curator call signature must be consitent with:: callable(station_code, event_id, stream) -> bool The callable returns a boolean indicating whether to keep the Stream or not. :param curator: Function or callable delegate to adjudicate whether to keep each given stream. :type curator: Callable :return: None """ # Only need to loop over one db, since they both reference the same underlying Stream instances. PY2 = (sys.version_info[0] == 2) if PY2: from itertools import ifilterfalse as filterfalse # pylint: disable=no-name-in-module, import-outside-toplevel else: from itertools import filterfalse # pylint: disable=import-outside-toplevel # end if discard_items = [ (x[0], x[1]) for x in filterfalse(lambda rec: curator(*rec), iter(self)) ] self.prune(discard_items) # end func def apply(self, _callable): """Apply a callable across all streams. Use to apply uniform processing steps to the whole dataset. :param _callable: Callable object that takes an obspy Stream as input and applies itself to that Stream. \ Expect that stream may be mutated in-place by the callable. :type _callable: Any Callable compatible with the call signature. :return: None """ for _1, _2, stream in iter(self): _callable(stream) # end func def by_station(self): """ Iterate over station sub-dictionaries. :return: Iterable over the stations, each element consisting of pair containing \ (station code, event dict). :rtype: Iterable(tuple) """ return iter(self.db_sta.items()) # end func def by_event(self): """ Iterate over event sub-dictionaries. :return: Iterable over the discrete events, each element consisting of pair containing \ (event id, station dict). :rtype: Iterable(tuple) """ return iter(self.db_evid.items()) # end func def prune(self, items, cull=True): """ Remove a given sequence of (station, event) pairs from the dataset. :param items: Iterable of (station, event) pairs :type items: Iterable(tuple) :param cull: If True, then empty entries in the top level index will be removed. :type cull: boolean :return: None """ for station, event_id in items: self.db_sta[station].pop(event_id) self.db_evid[event_id].pop(station) if cull: if not self.db_sta[station]: self.db_sta.pop(station) # end if if not self.db_evid[event_id]: self.db_evid.pop(event_id) # end if # end if # end for # end func def write(self, output_h5_filename, index_format='event'): """ Write event dataset back out to HDF5 file. :param output_h5_filename: Output file name :type output_h5_filename: str or path :param index_format: Format to use for index. Must be 'event' (default) or 'standard' (obspy default) :type index_format: str :return: True if file was written :rtype: boolean """ assert not os.path.exists( output_h5_filename), 'Output file already exists' if index_format not in ['event', 'standard']: raise ValueError('Index format %s not supported' % index_format) # end if all_stream = obspy.Stream() for sta, evid, stream in iter(self): all_stream += stream # end for if index_format == 'event': write_h5_event_stream(output_h5_filename, all_stream, mode='w') elif index_format == 'standard': all_stream.write(output_h5_filename, format='H5', mode='w') # end if return os.path.isfile(output_h5_filename)
class Track: @staticmethod def id(t): """ Try to parse an object as a Track id """ if isinstance(t, int): return str(t) if isinstance(t, str): return t return t.id def __init__(self): self.id: str = None self.points = SortedDict() self.color: Tuple[int, int, int] = None self.tags: Set[str] = set() def to_dict(self): ret = { 'id': self.id, 'color': self.color, 'tags': list(self.tags), # keep points last for easier manual editing 'points': list(self.points.items()), } return ret @classmethod def from_dict(cls, d): ret = cls() ret.id = d['id'] ret.color = tuple(d['color']) ret.tags.update(x.lower() for x in d.get('tags', ())) ret.points.update(d['points']) return ret def under(self, frame): """ Get all points that occur before (exclusive) the frame """ keys = self.points.irange(maximum=frame, inclusive=(True, False)) # A SortedDict is just a b-list with a dict attached # so this is really the most efficient way to get an irange with values. # https://stackoverflow.com/questions/34099308/iterate-over-a-slice-of-items-in-a-sorteddict return ((k, self.points[k]) for k in keys) def add(self, frame, point): """ Add a point at the frame :raise PointOverrideException: If a point already exists at the frame """ v = self.points.setdefault(frame, point) # won't add the point if the frame is already filled if v is not point: raise PointOverrideException("can't have the same track in two places at the same frame") def del_point_floor(self, frame): """ Deletes the first point that occurs before the frame :returns: the deleted point, or None if no points were deleted """ k = next(iter(self.points.irange(None, frame, reverse=True)), None) if not k: return None v = self.points.pop(k) return v def key_span(self): """ Get the first and last frames of the track, or None if the track is empty """ if not self.points: return None k = self.points.keys() return k[0], k[-1] def value_span(self): """ Get the first and last points of the track, or None if the track is empty """ if not self.points: return None v = self.points.values() return v[0], v[-1] def __str__(self): return f'Track({self.id})' def distance(self): """ :return: the distance of the path, in pixels """ ret = 0 prev = None for x, y in self.points.values(): if prev: ret += ((x - prev[0]) ** 2 + (y - prev[1]) ** 2) ** 0.5 prev = (x, y) return ret def stats(self, **kwargs: Optional[str]): """ Get genric stats about the track :param kwargs: additional tracks to add to the stats. If the value is falsish, the argument is skipped. If it's True, the key name is appended. Otherwise, the key and the value names are appended """ ret = [str(self), ': ', str(len(self.points)), ' points'] if self.points: ret.extend([', frame range: ', str(self.key_span()), ', value range: ', str(self.value_span()), ]) if self.tags: ret.extend([', tags: ', '[', ', '.join(self.tags), ']']) ret.extend([', distance: ', str(int(self.distance())), 'px']) for k, v in kwargs.items(): if not v: continue if v is True: ret.extend([', ', k]) else: ret.extend([', ', k, ': ', str(v)]) return ''.join(ret)
class DotMap(MutableMapping): def __init__(self, *args, **kwargs): self._map = SortedDict() if args: d = args[0] if type(d) is dict: for k, v in self.__call_items(d): if type(v) is dict: v = DotMap(v) self._map[k] = v if kwargs: for k, v in self.__call_items(kwargs): self._map[k] = v @staticmethod def __call_items(obj): if hasattr(obj, 'iteritems') and ismethod(getattr(obj, 'iteritems')): return obj.iteritems() else: return obj.items() def items(self): return self.iteritems() def iteritems(self): return self.__call_items(self._map) def __iter__(self): return self._map.__iter__() def __setitem__(self, k, v): self._map[k] = v def __getitem__(self, k): if k not in self._map: # automatically extend to new DotMap self[k] = DotMap() return self._map[k] def __setattr__(self, k, v): if k == '_map': super(DotMap, self).__setattr__(k, v) else: self[k] = v def __getattr__(self, k): if k == '_map': return self._map else: return self[k] def __delattr__(self, key): return self._map.__delitem__(key) def __contains__(self, k): return self._map.__contains__(k) def __str__(self): items = [] for k, v in self.__call_items(self._map): items.append('{0}={1}'.format(k, repr(v))) out = 'DotMap({0})'.format(', '.join(items)) return out def __repr__(self): return str(self) def to_dict(self): d = {} for k, v in self.items(): if type(v) is DotMap: v = v.to_dict() d[k] = v return d def pprint(self): pprint(self.to_dict()) # proper dict subclassing def values(self): return self._map.values() @staticmethod def parse_other(other): if type(other) is DotMap: return other._map else: return other def __cmp__(self, other): other = DotMap.parse_other(other) return self._map.__cmp__(other) def __eq__(self, other): other = DotMap.parse_other(other) if not isinstance(other, dict): return False return self._map.__eq__(other) def __ge__(self, other): other = DotMap.parse_other(other) return self._map.__ge__(other) def __gt__(self, other): other = DotMap.parse_other(other) return self._map.__gt__(other) def __le__(self, other): other = DotMap.parseOther(other) return self._map.__le__(other) def __lt__(self, other): other = DotMap.parse_other(other) return self._map.__lt__(other) def __ne__(self, other): other = DotMap.parse_other(other) return self._map.__ne__(other) def __delitem__(self, key): return self._map.__delitem__(key) def __len__(self): return self._map.__len__() def copy(self): return self def get(self, key, default=None): return self._map.get(key, default) def has_key(self, key): return key in self._map def iterkeys(self): return self._map.iterkeys() def itervalues(self): return self._map.itervalues() def keys(self): return self._map.keys() def pop(self, key, default=None): return self._map.pop(key, default) def setdefault(self, key, default=None): return self._map.setdefault(key, default) def viewitems(self): if version_info.major == 2 and version_info.minor >= 7: return self._map.viewitems() else: return self._map.items() def viewkeys(self): if version_info.major == 2 and version_info.minor >= 7: return self._map.viewkeys() else: return self._map.keys() def viewvalues(self): if version_info.major == 2 and version_info.minor >= 7: return self._map.viewvalues() else: return self._map.values() @classmethod def fromkeys(cls, seq, value=None): d = DotMap() d._map = SortedDict.fromkeys(seq, value) return d
class Controller(object): def __init__(self, thrift_server, thrift_port): self.transport = TSocket.TSocket(thrift_server, thrift_port) self.transport = TTransport.TBufferedTransport(self.transport) bprotocol = TBinaryProtocol.TBinaryProtocol(self.transport) conn_mgr_protocol = TMultiplexedProtocol.TMultiplexedProtocol( bprotocol, "conn_mgr") self.conn_mgr = conn_mgr_pd_rpc.conn_mgr.Client(conn_mgr_protocol) p4_protocol = TMultiplexedProtocol.TMultiplexedProtocol( bprotocol, "pegasus") self.pegasus = pegasus.p4_pd_rpc.pegasus.Client(p4_protocol) self.devport = devport_mgr_pd_rpc.devport_mgr.Client(conn_mgr_protocol) self.transport.open() self.sess_hdl = self.conn_mgr.client_init() self.dev = 0 self.dev_tgt = DevTarget_t(self.dev, hex_to_i16(0xFFFF)) self.flags = pegasus_register_flags_t(read_hw_sync=True) # keyhash -> ReplicatedKey (sorted in ascending load) self.replicated_keys = SortedDict( lambda x: self.replicated_keys[x].load) self.num_nodes = DEFAULT_NUM_NODES self.num_rkeys = MAX_NRKEYS self.switch_lock = threading.Lock() def install_table_entries(self, tables): # tab_l2_forward self.pegasus.tab_l2_forward_set_default_action__drop( self.sess_hdl, self.dev_tgt) for (mac, port) in tables["tab_l2_forward"].items(): self.pegasus.tab_l2_forward_table_add_with_l2_forward( self.sess_hdl, self.dev_tgt, pegasus_tab_l2_forward_match_spec_t( ethernet_dstAddr=macAddr_to_string(mac)), pegasus_l2_forward_action_spec_t(action_port=port)) # tab_node_forward self.pegasus.tab_node_forward_set_default_action__drop( self.sess_hdl, self.dev_tgt) self.num_nodes = len(tables["tab_node_forward"]) for (node, attrs) in tables["tab_node_forward"].items(): self.pegasus.tab_node_forward_table_add_with_node_forward( self.sess_hdl, self.dev_tgt, pegasus_tab_node_forward_match_spec_t(meta_node=int(node)), pegasus_node_forward_action_spec_t( action_mac_addr=macAddr_to_string(attrs["mac"]), action_ip_addr=ipv4Addr_to_i32(attrs["ip"]), action_udp_addr=attrs["udp"], action_port=attrs["port"])) # reg_n_servers self.pegasus.register_write_reg_n_servers(self.sess_hdl, self.dev_tgt, 0, self.num_nodes) # tab_calc_rset_index for i in range(self.num_rkeys): self.pegasus.tab_calc_rset_index_table_add_with_calc_rset_index( self.sess_hdl, self.dev_tgt, pegasus_tab_calc_rset_index_match_spec_t(meta_rkey_index=i), pegasus_calc_rset_index_action_spec_t(action_base=i * MAX_RSET_SIZE)) # tab_replicated_keys self.all_rkeys = tables["tab_replicated_keys"] for i in range(self.num_rkeys): self.add_rkey(self.all_rkeys[i], i, 0) self.conn_mgr.complete_operations(self.sess_hdl) def reset(self): self.switch_lock.acquire() for keyhash in self.replicated_keys.keys(): self.pegasus.tab_replicated_keys_table_delete_by_match_spec( self.sess_hdl, self.dev_tgt, pegasus_tab_replicated_keys_match_spec_t( pegasus_keyhash=int(keyhash))) self.replicated_keys.clear() for i in range(self.num_rkeys): self.add_rkey(self.all_rkeys[i], i, 0) self.pegasus.register_write_reg_ver_next(self.sess_hdl, self.dev_tgt, 0, 1) self.pegasus.register_write_reg_n_servers(self.sess_hdl, self.dev_tgt, 0, self.num_nodes) self.pegasus.register_write_reg_rr_all_servers(self.sess_hdl, self.dev_tgt, 0, 0) self.conn_mgr.complete_operations(self.sess_hdl) self.switch_lock.release() def add_rkey(self, keyhash, rkey_index, load): if RSET_ALL: self.pegasus.register_write_reg_rset_size(self.sess_hdl, self.dev_tgt, rkey_index, self.num_nodes) bitmap = (2**self.num_nodes) - 1 self.pegasus.register_write_reg_rset_bitmap( self.sess_hdl, self.dev_tgt, rkey_index, bitmap) rset_index = rkey_index << RSET_INDEX_SHIFT for i in range(self.num_nodes): self.pegasus.register_write_reg_rset(self.sess_hdl, self.dev_tgt, rset_index + i, i) else: node = int(keyhash) % self.num_nodes bitmap = 1 << node rset_index = rkey_index << RSET_INDEX_SHIFT self.pegasus.register_write_reg_rset_size(self.sess_hdl, self.dev_tgt, rkey_index, 1) self.pegasus.register_write_reg_rset_bitmap( self.sess_hdl, self.dev_tgt, rkey_index, bitmap) self.pegasus.register_write_reg_rset(self.sess_hdl, self.dev_tgt, rset_index, node) self.pegasus.register_write_reg_rkey_ver_completed( self.sess_hdl, self.dev_tgt, rkey_index, 1) self.pegasus.register_write_reg_rkey_read_counter( self.sess_hdl, self.dev_tgt, rkey_index, 0) self.pegasus.register_write_reg_rkey_write_counter( self.sess_hdl, self.dev_tgt, rkey_index, 0) self.pegasus.register_write_reg_rkey_rate_counter( self.sess_hdl, self.dev_tgt, rkey_index, 0) self.pegasus.register_write_reg_rr_rkey(self.sess_hdl, self.dev_tgt, rkey_index, 0) self.pegasus.tab_replicated_keys_table_add_with_is_rkey( self.sess_hdl, self.dev_tgt, pegasus_tab_replicated_keys_match_spec_t( pegasus_keyhash=int(keyhash)), pegasus_is_rkey_action_spec_t(action_rkey_index=rkey_index)) self.replicated_keys.setdefault( keyhash, ReplicatedKey(index=int(rkey_index), load=load)) def periodic_update(self): self.switch_lock.acquire() # Reset read and write counters self.pegasus.register_reset_all_reg_rkey_read_counter( self.sess_hdl, self.dev_tgt) self.pegasus.register_reset_all_reg_rkey_write_counter( self.sess_hdl, self.dev_tgt) # Read rkey load for rkey in self.replicated_keys.values(): read_value = self.pegasus.register_read_reg_rkey_rate_counter( self.sess_hdl, self.dev_tgt, rkey.index, self.flags) rkey.load = int(read_value[1]) # Reset rkey load self.pegasus.register_reset_all_reg_rkey_rate_counter( self.sess_hdl, self.dev_tgt) self.conn_mgr.complete_operations(self.sess_hdl) self.switch_lock.release() def print_stats(self): self.switch_lock.acquire() # read replicated keys info for (keyhash, rkey) in self.replicated_keys.items(): read_value = self.pegasus.register_read_reg_rkey_rate_counter( self.sess_hdl, self.dev_tgt, rkey.index, self.flags) rkey.load = read_value[0] print "rkey hash", keyhash print "rkey load", rkey.load read_value = self.pegasus.register_read_reg_rkey_ver_completed( self.sess_hdl, self.dev_tgt, rkey.index, self.flags) print "ver completed", read_value[0] read_value = self.pegasus.register_read_reg_rset_size( self.sess_hdl, self.dev_tgt, rkey.index, self.flags) rset_size = int(read_value[0]) print "rset size", rset_size read_value = self.pegasus.register_read_reg_rset_bitmap( self.sess_hdl, self.dev_tgt, rkey.index, self.flags) print "rset bitmap", read_value[0] base = rkey.index * MAX_RSET_SIZE for i in range(rset_size): read_value = self.pegasus.register_read_reg_rset( self.sess_hdl, self.dev_tgt, base + i, self.flags) print "replica", read_value[0] self.conn_mgr.complete_operations(self.sess_hdl) self.switch_lock.release() def run(self): while True: time.sleep(0.1) self.periodic_update() def stop(self): self.transport.close()
def test_setdefault(): mapping = [(val, pos) for pos, val in enumerate(string.ascii_lowercase)] temp = SortedDict(mapping) assert temp.setdefault('a', -1) == 0 assert temp['a'] == 0 assert temp.setdefault('A', -1) == -1
class ImpulseDiGraph(ImpulseGraph): """Base class for directed impulse graphs. The ImpulseDiGraph class allows any hashable object as a node and can associate key/value attribute pairs with each directed edge. Each edge must have one integer, timestamp. Self-loops are allowed. Multiple edges between two nodes are allowed. Parameters ---------- attr : keyword arguments, optional (default= no attributes) Attributes to add to graph as key=value pairs. Examples -------- Create an empty graph structure (a "null impulse graph") with no nodes and no edges. >>> G = dnx.ImpulseDiGraph() G can be grown in several ways. **Nodes:** Add one node at a time: >>> G.add_node(1) Add the nodes from any container (a list, dict, set or even the lines from a file or the nodes from another graph). Add the nodes from any container (a list, dict, set) >>> G.add_nodes_from([2, 3]) >>> G.add_nodes_from(range(100, 110)) **Edges:** G can also be grown by adding edges. This can be considered the primary way to grow G, since nodes with no edge will not appear in G in most cases. See ``G.to_snapshot()``. Add one edge, with timestamp of 10. >>> G.add_edge(1, 2, 10) a list of edges, >>> G.add_edges_from([(1, 2, 10), (1, 3, 11)]) If some edges connect nodes not yet in the graph, the nodes are added automatically. There are no errors when adding nodes or edges that already exist. **Attributes:** Each impulse graph, node, and edge can hold key/value attribute pairs in an associated attribute dictionary (the keys must be hashable). By default these are empty, but can be added or changed using add_edge, add_node. Keep in mind that the edge timestamp is not an attribute of the edge. >>> G = dnx.ImpulseDiGraph(day="Friday") >>> G.graph {'day': 'Friday'} Add node attributes using add_node(), add_nodes_from() >>> G.add_node(1, time='5pm') >>> G.add_nodes_from([3], time='2pm') Add edge attributes using add_edge(), add_edges_from(). >>> G.add_edge(1, 2, 10, weight=4.7 ) >>> G.add_edges_from([(3, 4, 11), (4, 5, 33)], color='red') **Shortcuts:** Here are a couple examples of available shortcuts: >>> 1 in G # check if node in impulse graph during any timestamp True >>> len(G) # number of nodes in the entire impulse graph 5 **Subclasses (Advanced):** Edges in impulse graphs are represented by tuples kept in a SortedDict (http://www.grantjenks.com/docs/sortedcontainers/) keyed by timestamp. The Graph class uses a dict-of-dict-of-dict data structure. The outer dict (node_dict) holds adjacency information keyed by nodes. The next dict (adjlist_dict) represents the adjacency information and holds edge data keyed by interval objects. The inner dict (edge_attr_dict) represents the edge data and holds edge attribute values keyed by attribute names. """ def __init__(self, **attr): """Initialize an impulse graph with edges, name, or graph attributes. Parameters ---------- attr : keyword arguments, optional (default= no attributes) Attributes to add to graph as key=value pairs. Examples -------- >>> G = dnx.ImpulseDiGraph() >>> G = dnx.ImpulseDiGraph(name='my graph') >>> G.graph {'name': 'my graph'} """ self.tree = SortedDict() self.graph = {} # dictionary for graph attributes self._node = {} self._pred = {} # out self._succ = {} # in self.edgeid = 0 self.graph.update(attr) def add_edge(self, u, v, t, **attr): """Add an edge between u and v, at t. The nodes u and v will be automatically added if they are not already in the impulse graph. Edge attributes can be specified with keywords or by directly accessing the edge's attribute dictionary. See examples below. Parameters ---------- u, v : nodes Nodes can be, for example, strings or numbers. Nodes must be hashable (and not None) Python objects. t : timestamp Timestamps can be, for example, strings or numbers. Timestamps must be hashable (and not None) Python objects. attr : keyword arguments, optional Edge data (or labels or objects) can be assigned using keyword arguments. See Also -------- add_edges_from : add a collection of edges Notes ----- Adding an edge that already exists updates the edge data. Timestamps must be the same type across all edges in the impulse graph. Also, to create snapshots, timestamps must be integers. Many NetworkX algorithms designed for weighted graphs use an edge attribute (by default `weight`) to hold a numerical value. Examples -------- The following all add the edge e=(1, 2, 3, 10) to graph G: >>> G = dnx.ImpulseDiGraph() >>> e = (1, 2, 10) >>> G.add_edge(1, 2, 10) # explicit two-node form with timestamp >>> G.add_edge(*e) # single edge as tuple of two nodes and timestamp >>> G.add_edges_from([(1, 2, 10)]) # add edges from iterable container Associate data to edges using keywords: >>> G.add_edge(1, 2, 10 weight=3) >>> G.add_edge(1, 3, 9, weight=7, capacity=15, length=342.7) """ self.tree.setdefault(t, set()).add((u, v)) self._node.setdefault(u, {}) self._node.setdefault(v, {}) self._pred.setdefault(u, {}).setdefault(v, {})[(u, v, t)] = attr self._succ.setdefault(v, {}).setdefault(u, {})[(u, v, t)] = attr def add_edges_from(self, ebunch_to_add, **attr): """Add all the edges in ebunch_to_add. Parameters ---------- ebunch_to_add : container of edges Each edge given in the container will be added to the impulse graph. The edges must be given as as 3-tuples (u, v, t). Timestamp must be orderable and the same type across all edges. attr : keyword arguments, optional Edge data (or labels or objects) can be assigned using keyword arguments. See Also -------- add_edge : add a single edge Notes ----- Adding the same edge (with the same timestamp) twice has no effect but any edge data will be updated when each duplicate edge is added. Examples -------- >>> G = dnx.ImpulseDiGraph() >>> G.add_edges_from([(1, 2, 10), (2, 4, 11)]) # using a list of edge tuples Associate data to edges >>> G.add_edges_from([(1, 2, 10), (2, 4, 11)], weight=3) >>> G.add_edges_from([(3, 4, 19), (1, 4, 3)], label='WN2898') """ for e in ebunch_to_add: if len(e) != 3: raise NetworkXError( "Edge tuple {0} must be a 3-tuple.".format(e)) self.add_edge(e[0], e[1], e[2], **attr) def has_edge(self, u, v, begin=None, end=None, inclusive=(True, True)): """Return True if there exists an edge between u and v in the impulse graph, during the given interval. Parameters ---------- u, v : nodes Nodes can be, for example, strings or numbers. Nodes must be hashable (and not None) Python objects. begin : int or float, optional (default= beginning of the entire impulse graph) end : int or float, optional (default= end of the entire impulse graph) Must be bigger than or equal begin. inclusive: 2-tuple boolean that determines inclusivity of begin and end Examples -------- >>> G = dnx.ImpulseDiGraph() >>> G.add_edges_from([(1, 2, 10), (2, 4, 11)]) >>> G.has_edge(1, 2) True >>> G.has_edge(1, 2, begin=2) True >>> G.has_edge(2, 4, begin=12) False """ if u not in self._pred or v not in self._pred[u]: return False if begin is None and end is None: return True if begin and end and begin > end: raise NetworkXError( "IntervalGraph: interval end must be bigger than or equal to begin: " "begin: {}, end: {}.".format(begin, end)) for iv in self._pred[u][v]: if self.__in_interval(iv[2], begin, end, inclusive=inclusive): return True return False def edges(self, u=None, v=None, begin=None, end=None, inclusive=(True, True), data=False, default=None): """Returns a list of tuples of the ImpulseDiGraph edges. All edges which are present within the given interval. All parameters are optional. `u` and `v` can be thought of as constraints. If no node is defined, all edges within the interval are returned. If one node is defined, all edges which have that node as one end, will be returned, and finally if both nodes are defined then all edges between the two nodes are returned. Parameters ---------- u, v : nodes, optional (default=None) Nodes can be, for example, strings or numbers. Nodes must be hashable (and not None) Python objects. If the node does not exist in the graph, a key error is raised. begin: int or float, optional (default= beginning of the entire impulse graph) end: int or float, optional (default= end of the entire impulse graph) Must be bigger than or equal to begin. inclusive: 2-tuple boolean that determines inclusivity of begin and end data : string or bool, optional (default=False) If True, return 2-tuple (Edge Tuple, dict of attributes). If False, return just the Edge Tuples. If string (name of the attribute), return 2-tuple (Edge Tuple, attribute value). default : value, optional (default=None) Default Value to be used for edges that don't have the requested attribute. Only relevant if `data` is a string (name of an attribute). Returns ------- List of Edge Tuples An edge tuple has the following format: (u, v, edge_id, timestamp) When called, if `data` is False, a list of edge tuples. If `data` is True, a list of 2-tuples: (Edge Tuple, dict of attribute(s) with values), If `data` is a string, a list of 2-tuples (Edge Tuple, attribute value). Examples -------- To get a list of all edges: >>> G = dnx.ImpulseDiGraph() >>> G.add_edges_from([(1, 2, 10), (2, 4, 11), (6, 4, 19), (2, 4, 15)]) >>> G.edges() [(1, 2, 10), (2, 4, 11), (2, 4, 15), (6, 4, 19)] To get edges which appear in a specific interval: >>> G.edges(begin=10) [(1, 2, 10), (2, 4, 11), (2, 4, 15), (6, 4, 19)] >>> G.edges(end=11) [(1, 2, 10), (2, 4, 11)] >>> G.edges(begin=11, end=15) [(2, 4, 11), (2, 4, 15)] To get edges with either of the two nodes being defined: >>> G.edges(u=2) [(2, 4, 11), (2, 4, 15)] >>> G.edges(u=2, begin=11) [(2, 4, 11), (2, 4, 15)] >>> G.edges(u=2, v=4, end=11) [(2, 4, 11)] >>> G.edges(u=1, v=6) [] To get a list of edges with data: >>> G = dnx.ImpulseDiGraph() >>> G.add_edge(1, 3, 4, weight=8, height=18) >>> G.add_edge(1, 2, 10, weight=10) >>> G.add_edge(2, 6, 10) >>> G.edges(data="weight") [((1, 3, 4), 8), ((1, 2, 10), 10), ((2, 6, 10), None)] >>> G.edges(data="weight", default=5) [((1, 3, 4), 8), ((1, 2, 10), 10), ((2, 6, 10), 5)] >>> G.edges(data=True) [((1, 3, 4), {'weight': 8, 'height': 18}), ((1, 2, 10), {'weight': 10}), ((2, 6, 10), {})] >>> G.edges(u=1, begin=2, end=9, data="weight") [((1, 3, 4), 8)] """ if begin is None: inclusive = (True, inclusive[1]) if end is None: inclusive = (inclusive[0], True) if u is None and v is None: if begin is not None and end is not None and begin > end: raise NetworkXError( "IntervalGraph: interval end must be bigger than or equal to begin: " "begin: {}, end: {}.".format(begin, end)) iedges = [iv for iv in self.__search_tree(begin, end, inclusive)] else: # Node filtering if u is not None and v is not None: if u not in self._pred: return [] if v not in self._pred[u]: return [] iedges = self._pred[u][v] elif u is not None: if u not in self._pred: return [] iedges = [iv for v in self._pred[u] for iv in self._pred[u][v]] else: if v not in self._succ: return [] iedges = [iv for u in self._succ[v] for iv in self._succ[v][u]] # Interval filtering if begin is not None and end is not None and begin > end: raise NetworkXError( "IntervalGraph: interval end must be bigger than or equal to begin: " "begin: {}, end: {}.".format(begin, end)) iedges = [ iv for iv in iedges if self.__in_interval(iv[2], begin, end, inclusive=inclusive) ] if data is False: return [edge for edge in iedges] if data is True: return [(edge, self._pred[edge[0]][edge[1]][edge]) for edge in iedges] return [(edge, self._pred[edge[0]][edge[1]][edge][data]) if data in self._pred[edge[0]][edge[1]][edge] else (edge, default) for edge in iedges] def remove_edge(self, u, v, begin=None, end=None, inclusive=(True, True)): """Remove the edge between u and v in the impulse graph, during the given interval. Quiet if the specified edge is not present. Parameters ---------- u, v : nodes Nodes can be, for example, strings or numbers. Nodes must be hashable (and not None) Python objects. begin : int or float, optional (default= beginning of the entire impulse graph) end : int or float, optional (default= end of the entire impulse graph + 1) Must be bigger than or equal to begin. inclusive: 2-tuple boolean that determines inclusivity of begin and end Examples -------- >>> G = dnx.ImpulseDiGraph() >>> G.add_edges_from([(1, 2, 10), (2, 4, 11), (6, 4, 9), (1, 2, 15)]) >>> G.remove_edge(1, 2) >>> G.has_edge(1, 2) False >>> G = dnx.ImpulseDiGraph() >>> G.add_edges_from([(1, 2, 10), (2, 4, 11), (6, 4, 9), (1, 2, 15)]) >>> G.remove_edge(1, 2, begin=2, end=11) >>> G.has_edge(1, 2, begin=2, end=11) False >>> G.has_edge(1, 2) True """ if u not in self._pred or v not in self._pred[u]: return iedges_to_remove = [] # remove every edge between u and v if begin is None and end is None: for iv in self._pred[u][v]: iedges_to_remove.append(iv) else: for iv in self._pred[u][v]: if self.__in_interval(iv[2], begin, end): iedges_to_remove.append(iv) # removing found iedges for edge in iedges_to_remove: self.__remove_iedge(edge) # clean up empty dictionaries if len(self._pred[u][v]) == 0: self._pred[u].pop(v, None) if len(self._succ[v][u]) == 0: self._succ[v].pop(u, None) if len(self._pred[u]) == 0: self._pred.pop(u, None) if len(self._succ[v]) == 0: self._succ.pop(v, None) def degree(self, node=None, begin=None, end=None, delta=False, inclusive=(True, True)): """Return the sum of in and out degree of a specified node between time begin and end. Parameters ---------- node : Nodes can be, for example, strings or numbers. Nodes must be hashable (and not None) Python objects. begin : int or float, optional (default= beginning of the entire impulse graph) Inclusive beginning time of the edge appearing in the impulse graph. end : int or float, optional (default= end of the entire impulse graph) Non-inclusive ending time of the edge appearing in the impulse graph. delta : boolean, optional (default= False) Returns list of 2-tuples, first element is the timestamp, second is the node of changing degree. inclusive : 2-tuple boolean that determines inclusivity of begin and end Returns ------- Integer value of degree of specified node. Examples -------- >>> G = dnx.ImpulseDiGraph() >>> G.add_edge(1, 2, 3) >>> G.add_edge(2, 3, 8) >>> G.degree(2) 2 >>> G.degree(2, 4) 1 >>> G.degree(2, end=8) 2 >>> G.degree() 1.33333 >>> G.degree(2, delta=True) [(3, 1), (8, 1)] """ # no specified node, return mean degree if node == None: n = 0 l = 0 for node in self.nodes(begin=begin, end=end, inclusive=inclusive): n += 1 l += self.degree(node, begin=begin, end=end, inclusive=inclusive) return l / n # specified node, no degree_change, return degree if delta == False: return len(self.edges(u=node, begin=begin, end=end, inclusive=inclusive)) + \ len(self.edges(v=node, begin=begin, end=end, inclusive=inclusive)) # delta == True, return list of changes if begin == None: begin = list(self.tree.keys())[0] if end == None: end = list(self.tree.keys())[-1] d = {} output = [] # for each edge determine if the begin and/or end value is in specified time period for edge in self.edges(u=node, begin=begin, end=end, inclusive=(True, True)): d.setdefault(edge[2], []).append((edge[0], edge[1])) for edge in self.edges(v=node, begin=begin, end=end, inclusive=(True, True)): d.setdefault(edge[2], []).append((edge[0], edge[1])) # for each time in Dict add to output list the len of each value for time in d: output.append((time, len(d[time]))) return sorted(output) def in_degree(self, node=None, begin=None, end=None, delta=False, inclusive=(True, True)): """Return the in-degree of a specified node between time begin and end. Parameters ---------- node : Nodes can be, for example, strings or numbers. Nodes must be hashable (and not None) Python objects. begin : int or float, optional (default= beginning of the entire impulse graph) Inclusive beginning time of the edge appearing in the impulse graph. end : int or float, optional (default= end of the entire impulse graph) Non-inclusive ending time of the edge appearing in the impulse graph. delta : boolean, optional (default= False) Returns list of 2-tuples, first element is the timestamp, second is the node of changing degree. inclusive : 2-tuple boolean that determines inclusivity of begin and end Returns ------- Integer value of in-degree of specified node. Examples -------- >>> G = dnx.ImpulseDiGraph() >>> G.add_edge(1, 2, 3) >>> G.add_edge(2, 3, 8) >>> G.in_degree(2) 1 >>> G.in_degree(2, 4) 0 >>> G.in_degree(2, end=8) 1 >>> G.in_degree() 0.66666 >>> G.in_degree(2, delta=True) [(3, 1)] """ # no specified node, return mean degree if node == None: n = 0 l = 0 for node in self.nodes(begin=begin, end=end, inclusive=inclusive): n += 1 l += self.in_degree(node, begin=begin, end=end, inclusive=inclusive) return l / n # specified node, no degree_change, return degree if delta == False: return len( self.edges(v=node, begin=begin, end=end, inclusive=inclusive)) # delta == True, return list of changes if begin == None: begin = list(self.tree.keys())[0] if end == None: end = list(self.tree.keys())[-1] d = {} output = [] # for each edge determine if the begin and/or end value is in specified time period for edge in self.edges(v=node, begin=begin, end=end, inclusive=(True, True)): d.setdefault(edge[2], []).append((edge[0], edge[1])) # for each time in Dict add to output list the len of each value for time in d: output.append((time, len(d[time]))) return output def out_degree(self, node=None, begin=None, end=None, delta=False, inclusive=(True, True)): """Return the out-degree of a specified node between time begin and end. Parameters ---------- node : Nodes can be, for example, strings or numbers. Nodes must be hashable (and not None) Python objects. begin : int or float, optional (default= beginning of the entire impulse graph) Inclusive beginning time of the edge appearing in the impulse graph. end : int or float, optional (default= end of the entire impulse graph) Non-inclusive ending time of the edge appearing in the impulse graph. delta : boolean, optional (default= False) Returns list of 2-tuples, first element is the timestamp, second is the node of changing degree. inclusive : 2-tuple boolean that determines inclusivity of begin and end Returns ------- Integer value of out-degree of specified node. Examples -------- >>> G = dnx.ImpulseDiGraph() >>> G.add_edge(1, 2, 3) >>> G.add_edge(2, 3, 8) >>> G.out_degree(2) 1 >>> G.out_degree(2, 2) 1 >>> G.out_degree(2, end=8) 1 >>> G.out_degree() 0.66666 >>> G.out_degree(2, delta=True) [(8, 1)] """ # no specified node, return mean degree if node == None: n = 0 l = 0 for node in self.nodes(begin=begin, end=end, inclusive=inclusive): n += 1 l += self.in_degree(node, begin=begin, end=end, inclusive=inclusive) return l / n # specified node, no degree_change, return degree if delta == False: return len( self.edges(u=node, begin=begin, end=end, inclusive=inclusive)) # delta == True, return list of changes if begin == None: begin = list(self.tree.keys())[0] if end == None: end = list(self.tree.keys())[-1] d = {} output = [] # for each edge determine if the begin and/or end value is in specified time period for edge in self.edges(u=node, begin=begin, end=end, inclusive=(True, True)): d.setdefault(edge[2], []).append((edge[0], edge[1])) # for each time in Dict add to output list the len of each value for time in d: output.append((time, len(d[time]))) return output def to_networkx_graph(self, begin=None, end=None, inclusive=(True, False), multigraph=False, edge_data=False, edge_timestamp_data=False, node_data=False): """Return a networkx Graph or MultiGraph which includes all the nodes and edges which have timestamps within the given interval. Wrapper function for ImpulseGraph.to_subgraph. Refer to ImpulseGraph.to_subgraph for full description. """ return self.to_subgraph(begin=begin, end=end, inclusive=inclusive, multigraph=multigraph, edge_data=edge_data, edge_timestamp_data=edge_timestamp_data, node_data=node_data) def to_subgraph(self, begin, end, inclusive=(True, False), multigraph=False, edge_data=False, edge_timestamp_data=False, node_data=False): """Return a networkx Graph or MultiGraph which includes all the nodes and edges which have timestamps within the given interval. Parameters ---------- begin: int or float end: int or float Must be bigger than or equal to begin. inclusive: 2-tuple boolean that determines inclusivity of begin and end multigraph: bool, optional (default= False) If True, a networkx MultiGraph will be returned. If False, networkx Graph. edge_data: bool, optional (default= False) If True, edges will keep their attributes. edge_timestamp_data: bool, optional (default= False) If True, each edge's attribute will also include its timestamp data. If `edge_data= True` and there already exist edge attributes named timestamp it will be overwritten. node_data : bool, optional (default= False) if True, each node's attributes will be included. See Also -------- to_snapshots : divide the impulse graph to snapshots Notes ----- If multigraph= False, and edge_data=True or edge_interval_data=True, in case there are multiple edges, only one will show with one of the edge's attributes. Note: nodes with no edges will not appear in any subgraph. Examples -------- >>> G = dnx.ImpulseGraph() >>> G.add_edges_from([(1, 2, 10), (2, 4, 11), (6, 4, 19), (2, 4, 15)]) >>> H = G.to_subgraph(4, 12) >>> type(H) <class 'networkx.classes.graph.DiGraph'> >>> list(H.edges(data=True)) [(1, 2, {}), (2, 4, {})] >>> H = G.to_subgraph(10, 12, edge_timestamp_data=True) >>> type(H) <class 'networkx.classes.graph.DiGraph'> >>> list(H.edges(data=True)) [(1, 2, {'timestamp': 10}), (2, 4, {'timestamp': 11})] >>> M = G.to_subgraph(4, 12, multigraph=True, edge_timestamp_data=True) >>> type(M) <class 'networkx.classes.multigraph.MultiDiGraph'> >>> list(M.edges(data=True)) [(1, 2, {'timestamp': 10}), (2, 4, {'timestamp': 11})] """ iedges = self.__search_tree(begin, end, inclusive=inclusive) if multigraph: G = MultiDiGraph() else: G = DiGraph() if edge_data and edge_timestamp_data: G.add_edges_from((iedge[0], iedge[1], dict(self._pred[iedge[0]][iedge[1]][iedge], timestamp=iedge[3])) for iedge in iedges) elif edge_data: G.add_edges_from( (iedge[0], iedge[1], self._pred[iedge[0]][iedge[1]][iedge]) for iedge in iedges) elif edge_timestamp_data: G.add_edges_from((iedge[0], iedge[1], { 'timestamp': iedge[3] }) for iedge in iedges) else: G.add_edges_from((iedge[0], iedge[1]) for iedge in iedges) if node_data: G.add_nodes_from((n, self._node[n].copy()) for n in G.nodes) return G def __remove_iedge(self, iedge): """Remove the impulse edge from the impulse graph. Quiet if the specified edge is not present. Parameters ---------- iedge : Edge Tuple (u,v,eid,t) Edge to be removed. """ try: self.tree[iedge[2]].remove((iedge[0], iedge[1])) del self._pred[iedge[0]][iedge[1]][iedge] del self._succ[iedge[1]][iedge[0]][iedge] except: return def __validate_interval(self, begin=None, end=None): """Returns validated begin and end. Raises an exception if begin is larger than end. Parameters ---------- begin : int or float, optional end : int or float, optional """ if (begin is not None and end is not None) and begin > end: raise NetworkXError( "ImpulseDiGraph: interval end must be bigger than or equal to begin: " "begin: {}, end: {}.".format(begin, end)) return begin, end def __search_tree(self, begin=None, end=None, inclusive=(True, True)): """if begin and end are equal performs a point search on the tree, otherwise an interval search is performed. Parameters ---------- begin: int or float, optional (default= beginning of the entire impulse graph) end: int or float, optional (default= end of the entire impulse graph) Must be bigger than or equal begin. inclusive: 2-tuple boolean that determines inclusivity of begin and end """ begin, end = self.__validate_interval(begin, end) if begin is not None and begin == end and begin in self.tree: for edge in self.tree[begin]: yield (*edge, begin) for t in self.tree.irange(begin, end, inclusive=inclusive): for edge in self.tree[t]: yield (*edge, t) def __in_interval(self, t, begin, end, inclusive=(True, True)): """ Parameters ---------- t: int or float, timestamp begin: int or float Beginning time of Interval. end: int or float Ending time of Interval. Must be bigger than or equal begin. inclusive: 2-tuple boolean that determines inclusivity of begin and end Returns ------- Returns True if t is in the interval (begin,end). Otherwise False. """ if begin is None: begin = float('-inf') if end is None: end = float('inf') if inclusive == (True, True): return begin <= t <= end if inclusive == (True, False): return begin <= t < end if inclusive == (False, True): return begin < t <= end if inclusive == (False, False): return begin < t < end @staticmethod def load_from_txt(path, delimiter=" ", nodetype=int, timestamptype=float, order=('u', 'v', 't'), comments="#"): """Read impulse graph in from path. Timestamps must be integers or floats. Nodes can be any hashable objects. Edge Attributes can be assigned with in the following format: Key=Value Parameters ---------- path : string or file Filename to read. nodetype : Python type, optional (default= int) Convert nodes to this type. timestamptype : Python type, optional (default= float) Convert timestamp to this type. This must be an orderable type, ideally int or float. Other orderable types have not been fully tested. order : Python 3-tuple, optional (default= ('u', 'v', 't')) This must be a 3-tuple containing strings 'u', 'v', and 't'. 'u' specifies the starting node, 'v' the ending node, and 't' the timestamp. comments : string, optional Marker for comment lines delimiter : string, optional Separator for node labels. The default is whitespace. Cannot be =. Returns ------- G: ImpulseGraph The graph corresponding to the lines in edge list. Examples -------- >>> G=dnx.ImpulseGraph.load_from_txt("my_dygraph.txt") The optional nodetype is a function to convert node strings to nodetype. For example >>> G=dnx.ImpulseGraph.load_from_txt("my_dygraph.txt", nodetype=int) will attempt to convert all nodes to integer type. Since nodes must be hashable, the function nodetype must return hashable types (e.g. int, float, str, frozenset - or tuples of those, etc.) """ G = ImpulseDiGraph() if delimiter == '=': raise ValueError("Delimiter cannot be =.") if len( order ) != 3 or 'u' not in order or 'v' not in order or 't' not in order: raise ValueError( "Order must be a 3-tuple containing strings 'u', 'v', and 't'." ) with open(path, 'r') as file: for line in file: p = line.find(comments) if p >= 0: line = line[:p] if not len(line): continue line = re.split(delimiter + '+', line.strip()) u = line[order.index('u')] v = line[order.index('v')] t = line[order.index('t')] edgedata = {} for data in line[3:]: key, value = data.split('=') try: value = float(value) except: pass edgedata[key] = value if nodetype is not int: try: u = nodetype(u) v = nodetype(v) except: raise TypeError( "Failed to convert node to {0}".format(nodetype)) else: try: u = int(u) v = int(v) except: pass try: t = timestamptype(t) except: raise TypeError( "Failed to convert interval time to {}".format( timestamptype)) G.add_edge(u, v, t, **edgedata) return G