class LogSystem:
def __init__(self):
self.map = SortedDict(list)
self.gra = {
'Year': 5,
'Month':8,
'Day':11,
'Hour': 14,
'Minute':17,
'Second':20,
}
def put(self, id: int, timestamp: str) -> None:
self.map.setdefault(timestamp, []).append(id)
def retrieve(self, start: str, end: str, granularity: str) -> List[int]:
idx = self.gra[granularity]
left = self.map.bisect_left(start[:idx])
result = []
for i in range(left, len(self.map)): # DO NOT USE bisect to find right, it may only granular to year, but date exceeds
key = self.map.keys()[i]
if key[:idx]>end[:idx]: break
result.extend(self.map[key])
return result
def frequency(sample):
letters = alpha_only(sample)
# encode to washers
l2n = alphanum.base64_to_data
n2w = washers.data_to_base4
washer_numbers = n2w(l2n(letters))
freq = SortedDict()
max_width = max(map(lambda x: len(x), washer_numbers))
for segment in washer_numbers:
# normalize widths
width = len(segment)
for i in range(max_width - width):
segment.insert(0, 0)
# update frequency
for washer in segment:
freq.setdefault(washer, 0)
freq[washer] += 1
return freq
def get_obspyh5_index(src_file, seeds_only=False):
"""Scrape the index (only) from an obspyh5 file.
:param src_file: Name of file to extract index from
:type src_file: str or pathlib.Path
:param seeds_only: If True, only get the seed IDs of the traces. Otherwise (default), get full index.
:type seeds_only: bool
:return: Sorted dictionary with index of waveforms in the file
:rtype: sortedcontainers.SortedDict
"""
# We use SortedSet rather than SortedList for the inner container
# because it allows set operations that are useful, such as finding
# event IDs for events which are common across multiple stations.
assert is_obspyh5(src_file), '{} is not an obspyh5 file'.format(src_file)
index = SortedDict()
with h5py.File(src_file, mode='r') as h5f:
root = h5f['/waveforms']
if seeds_only:
return SortedList(root.keys())
# end if
for seedid, station_grp in root.items():
for event_grp in station_grp.values():
evid = None
for channel in event_grp.values():
evid = channel.attrs['event_id']
break
# end for
if evid:
index.setdefault(seedid, SortedSet()).add(evid)
# end if
# end for
# end for
# end with
return index
def collect_misses(parsed_lexemes: Iterable) -> Dict:
misses = SortedDict()
for lexeme in parsed_lexemes:
for sublexeme in lexeme:
for segment in sublexeme:
for sm in segment[1]:
if sm.seg_type == 'miss':
misses.setdefault(sm.segment.lower(),
default=SortedSet()).add(sm.lexeme)
return misses
def sort_data_on_timestamp(mDict):
sd = SortedDict()
for key in mDict:
for index in range(len(mDict[key])):
if len(mDict[key][index]) == 3:
sd.setdefault(int(mDict[key][index][0]), []).\
append((key, mDict[key][index][1], -1, 0))
elif len(mDict[key][index]) == 4:
sd.setdefault(int(mDict[key][index][0]), []).\
append((key, mDict[key][index][1], mDict[key][index][3], 1))
return sd
def compute_pagerank(urls, inlinks, outlinks, b=.85, iters=20):
""" Return a dictionary mapping each url to its PageRank.
The formula is R(u) = (1/N)(1-b) + b * (sum_{w in B_u} R(w) / (|F_w|)
Initialize all scores to 1.0.
Params:
urls.......SortedList of urls (names)
inlinks....SortedDict mapping url to list of in links (backlinks)
outlinks...Sorteddict mapping url to list of outlinks
Returns:
A SortedDict mapping url to its final PageRank value (float)
>>> urls = SortedList(['a', 'b', 'c'])
>>> inlinks = SortedDict({'a': ['c'], 'b': set(['a']), 'c': set(['a', 'b'])})
>>> outlinks = SortedDict({'a': ['b', 'c'], 'b': set(['c']), 'c': set(['a'])})
>>> sorted(compute_pagerank(urls, inlinks, outlinks, b=.5, iters=0).items())
[('a', 1.0), ('b', 1.0), ('c', 1.0)]
>>> iter1 = compute_pagerank(urls, inlinks, outlinks, b=.5, iters=1)
>>> iter1['a'] # doctest:+ELLIPSIS
0.6666...
>>> iter1['b'] # doctest:+ELLIPSIS
0.333...
"""
###TODO
"""
R(u) = (1 - b)/N + b * sum( inlinks of u/outlink-number)
"""
Ru = SortedDict()
Rv = SortedDict()
size = len(urls)
# Initialize to 1.0 all URL's
for k in urls:
Ru.setdefault(k, 1.0)
# Page Rank definition
for i in range(iters):
for url in urls:
try:
Ru[url] = ((1-b)/size) + b * sum([Ru[x]/len(outlinks[x]) for x in inlinks[url] if len(outlinks[x])])
except:
pass
return Ru
def kWeakestRows(self, mat: List[List[int]], k: int) -> List[int]:
dic = SortedDict()
for idx, row in enumerate(mat):
sm = sum(row)
if sm in dic:
dic[sm].append(idx)
else:
dic.setdefault(sm, [idx])
ans = []
k1 = k
while k > 0:
ls_curr = dic.popitem(0)[1]
ans.extend(ls_curr)
k -= len(ls_curr)
return ans[:k1]
def testfunc(array, size):
houses = SortedDict()
for i in range(size):
tier = array[i]
houses.setdefault(tier, []).append(i)
dist = 0
oldsasha = 0
olddima = 0
for key in houses.keys():
sasha, dima = houses[key]
dist += abs(sasha - oldsasha) +\
abs(dima - olddima)
oldsasha = sasha
olddima = dima
return dist
def read_links(path):
"""
Read the html pages in the data folder. Create and return two SortedDicts:
inlinks: maps from a name to a SortedSet of names that link to it.
outlinks: maps from a name to a SortedSet of names that it links to.
For example:
inlinks['Ada_Lovelace'] = SortedSet(['Charles_Babbage', 'David_Gelernter'], key=None, load=1000)
outlinks['Ada_Lovelace'] = SortedSet(['Alan_Turing', 'Charles_Babbage'], key=None, load=1000)
You should use the read_names and get_links function above.
Params:
path...the name of the data directory ('data')
Returns:
A (inlinks, outlinks) tuple, as defined above (i.e., two SortedDicts)
"""
###TODO
inlinks = SortedDict()
outlinks = SortedDict()
names_dict = read_names(path)
for page in names_dict:
with open(path + os.sep + page, 'r') as fp:
html_page = fp.readlines()
# Get links
temp_links = get_links(names_dict, str(html_page))
# remove self link without checking, may save some time
try:
temp_links.remove(page)
except:
pass
# Count outlinks
outlinks[page] = temp_links
inlinks.setdefault(page,SortedSet())
# count inlinks from same single page-name
for link in outlinks[page]:
inlinks.setdefault(link,SortedSet()).add(page)
return inlinks, outlinks
def find_closest_elements(sorted_arrays):
min_distance_so_far = float('inf')
iters = SortedDict()
for idx, sorted_array in enumerate(sorted_arrays):
it = iter(sorted_array)
first_min = next(it, None)
if first_min is not None:
iters.setdefault((first_min, idx), default=it)
while True:
min_value, min_idx = iters.peekitem(index=0)[0]
max_value = iters.peekitem()[0][0]
min_distance_so_far = min(max_value - min_value, min_distance_so_far)
it = iters.popitem(index=0)[1]
next_min = next(it, None)
if next_min is None:
return min_distance_so_far
iters.setdefault((next_min, min_idx), it)
def test_SortedDict(self):
# construct
sorted_dict = SortedDict({'a': 1, 'c': 2, 'b': 3})
print('sorted dict is: ', sorted_dict)
# adding key => value pairs
sorted_dict['d'] = 3
print('sorted dict after adding an element: ', sorted_dict)
# adding element using setdefault()
sorted_dict.setdefault('e', 4)
print('sorted dict after setdefault(): ', sorted_dict)
# using the get function
print('using the get function to print the value of a: ', sorted_dict.get('a', 0))
for key in sorted_dict:
print('{} -> {}'.format(key, sorted_dict[key]), end=' ')
print()
# removing all elements from the dict
sorted_dict.clear()
def order_asteroids():
polar_asteroids = SortedDict()
for a in asteroids:
r, t = to_polar((a[0] - max_pos[0], a[1] - max_pos[1]))
polar_asteroids.setdefault(t, SortedDict())[r] = a
while polar_asteroids:
for t, rs in list(polar_asteroids.items()):
a = rs.pop(rs.keys()[0])
if not rs:
del polar_asteroids[t]
yield a
def _wb_init(path: str) -> tuple[SortedDict[str,
wordbook_entry], SortedDict[str, list[wordbook_entry]]]:
'''
Initializes word book data: returns dicts of by-word matches and by-meaning matches.
'''
def old():
age_s = time.time() - os.stat(path).st_mtime
return age_s > 60 * 60 * 24 * 10
if not os.path.isfile(path) or old():
import requests
print('getting wordbook.', end='')
response = requests.get(
'https://docs.google.com/spreadsheets/d/1y8_11RDvuCRyUK_MXj5K7ZjccgCUDapsPDI5PjaEkMw/gviz/tq?tqx=out:csv&sheet=Wordbook')
print('.\r', end='')
response.raise_for_status()
with open(path, 'wb') as f:
f.write(response.content)
print('getting wordbook... done')
with open(resource_filename('oe_edit.resources', 'wordbook.bin'), 'rb') as fbase:
with open(path, newline='', encoding='U8') as fcsv:
reader = iter(csv.reader(fcsv))
next(reader)
kfa = pkind.findall
def csv_entry(data):
return wordbook_entry([data[0], '; '.join(filter(
None, map(str.strip, data[2].split('᛫')))), '|'.join(kfa(data[2]))])
# l = list of entries
l = [*map(wordbook_entry, zip(*([iter(pickle.load(fbase))] * 3))),
*map(csv_entry, reader)]
sl = str.lower
wfa = pword.findall
word_match = SortedDict([(sl(e.word), e) for e in l])
mean_match = SortedDict()
for (k, v) in chain.from_iterable(
[[(sl(m[2]), e) for m in wfa(e.meaning)] for e in l]):
mean_match.setdefault(k, []).append(v)
return word_match, mean_match
def top_k_frequent(words: Iterable[str], k: int) -> Iterable[str]:
frequency_list = []
frequency_map = SortedDict()
for word in words:
frequency_map[word] = frequency_map.setdefault(word, 0) + 1
for word, count in frequency_map.items():
print(f"word:{word},count:{count}")
frequency_list.append(WordFrequency(word=word, frequency=count))
frequency_list = sorted(frequency_list, key=lambda x: x[1], reverse=True)
frequency_list = order_alphabetically(frequency_list)
return [x.word for x in frequency_list][:k]
class ExponentiallyDecayingReservoir(object):
def __init__(self, size=_DEFAULT_SIZE, alpha=_DEFAULT_ALPHA):
self.size = size
self.alpha = alpha
self.start_time = time()
self.next_scale_time = self.start_time + _RESCALE_THRESHOLD
self.values = SortedDict()
def _rescale_if_needed(self):
now = time()
if now > self.next_scale_time:
self.next_scale_time = now + _RESCALE_THRESHOLD
old_start_time = self.start_time
self.start_time = now
scaling_factor = exp(-self.alpha * (self.start_time - old_start_time))
self.values = SortedDict(
(priority * scaling_factor, _WeightedSample(sample.value, sample.weight * scaling_factor))
for priority, sample in self.values.items()
)
def update(self, value):
self._rescale_if_needed()
timestamp = time()
item_weight = exp(self.alpha * (timestamp - self.start_time))
sample = _WeightedSample(value, item_weight)
priority = item_weight / random()
if len(self.values) < self.size:
self.values[priority] = sample
else:
first_key = next(self.values.iterkeys())
if first_key < priority:
self.values.setdefault(priority, sample)
del self.values[first_key]
def get_snapshot(self):
return _WeightedSnapshot(self.values.values())
def brightestPosition(self, lights):
sd = SortedDict()
# 差分记录开闭区间，将右边改为开区间方便处理
for pos, r in lights:
st, ed = pos - r, pos + r + 1
sd.setdefault(st, 0)
sd.setdefault(ed, 0)
sd[st] += 1
sd[ed] -= 1
res = int(-1e9)
tot, max_ = 0, 0
entrys = sd.items()
# 遍历 map 得到最优位置
for pos, cnt in entrys:
tot += cnt
if tot > max_:
max_ = tot
res = pos
return res
class PriorityQueue:
def __init__(self):
self.__buckets__ = SortedDict()
return
def enqueue(self, priority, value):
bucket = self.__buckets__.setdefault(priority, [])
bucket.append(value)
return
def dequeue(self):
if len(self.__buckets__) > 0:
(priority, bucket) = self.__buckets__.peekitem(0)
value = bucket[0]
if len(bucket) > 1:
self.__buckets__[priority] = bucket[1:]
else:
del self.__buckets__[priority]
return (priority, value)
else:
return None
def clear(self):
self.__buckets__.clear()
return
def size(self):
total = 0
for bucket in self.__buckets__.values():
total += len(bucket)
return total
def count(self, priority):
return len(
self.__buckets__[priority]) if priority in self.__buckets__ else 0
from sortedcontainers import SortedDict
for case in range(1, 1 + int(raw_input())):
N, K = map(int, raw_input().split(' '))
blocks = SortedDict({N: 1})
while K:
b, n = blocks.popitem()
if n >= K:
print "Case #%d: %d %d" % (case, b / 2, (b - 1) / 2)
break
blocks.setdefault(b / 2, 0)
blocks[b / 2] += n
blocks.setdefault((b - 1) / 2, 0)
blocks[(b - 1) / 2] += n
K -= n
s = open(outFile, "w+")
with open(i) as f:
tag = f.readline()
header = tag
if tag.strip(
): # strip will remove all leading and trailing whitespace such as '\n' or ' ' by default
tag = tag.strip("\n ' '")
tag = tag.split()[6].split(',')[0]
sd = SortedDict()
for line in f:
if line[0] != '#':
tokens = line.split('\t')
if (int(tokens[2]) != OUT_OF_RANGE_CODE):
if (int(tokens[0]) >= int(vals[0])
and int(tokens[0]) <= int(vals[1])):
sd.setdefault(tokens[0], {}).setdefault(
tokens[1], []).append(tokens[2].rstrip('\n'))
for x in logs:
if (x != i):
with open(x) as w:
find = w.readline()
if find.strip(
): # strip will remove all leading and trailing whitespace such as '\n' or ' ' by default
find = find.strip("\n ' '")
find = find.split()[6].split(',')[0]
for row in w:
if row[0] != '#':
token = row.split('\t')
if (token[1] == tag):
if (int(token[2]) != OUT_OF_RANGE_CODE):
if (int(token[0]) >= int(vals[0])
and int(token[0]) <= int(vals[1])):
def out_degree(self, node=None, begin=None, end=None, delta=False):
"""Return the out-degree of a specified node between time begin and end.
Parameters
----------
node : Nodes can be, for example, strings or numbers, optional.
Nodes must be hashable (and not None) Python objects.
begin : int or float, optional (default= beginning of the entire interval graph)
Inclusive beginning time of the edge appearing in the interval graph.
end : int or float, optional (default= end of the entire interval graph)
Non-inclusive ending time of the edge appearing in the interval graph.
Returns
-------
Integer value of degree of specified node.
If no node is specified, returns float mean degree value of graph.
If delta is True, return list of tuples.
First indicating the time a degree change occurred,
Second indicating the degree after the change occured
Examples
--------
>>> G = IntervalDiGraph()
>>> G.add_edge(1, 2, 3, 5)
>>> G.add_edge(2, 3, 8, 11)
>>> G.degree(2)
1
>>> G.out_degree(2,2)
1
>>> G.out_degree(2,end=8)
0
>>> G.out_degree()
0.666666
>>> G.out_degree(2,delta=True)
[(8, 1)]
"""
# no specified node, return mean degree
if node == None:
n = 0
l = 0
for node in self.nodes(begin=begin, end=end):
n += 1
l += self.out_degree(node, begin=begin, end=end)
return l / n
# specified node, no degree_change, return degree
if delta == False:
return len(self.edges(u=node, begin=begin, end=end))
# delta == True, return list of changes
if begin == None:
begin = self.tree.begin
if end == None:
end = self.tree.end
current_degree = self.out_degree(node, begin=begin, end=begin)
sd = SortedDict()
output = []
# for each edge determine if the begin and/or end value is in specified time period
for edge in self.edges(u=node, begin=begin, end=end):
if edge[2] >= begin:
# if begin is in specified time period, add to SortedDict, with +1 to indicate begin
sd.setdefault((edge[2], 1), []).append((edge[0], edge[1]))
if edge[3] < end:
# if begin is in specified time period, add to SortedDict, with -1 to indicate begin
sd.setdefault((edge[3], -1), []).append((edge[0], edge[1]))
for time in sd:
for edge in sd[time]:
# iterate through SortedDict, only advancing current degree if edge was not counted on init
if time[0] != begin:
current_degree += time[1]
print(time, begin, edge, current_degree)
output.append((time[0], current_degree))
return sorted(output)
class OrderBookSide:
def __init__(self, asc, events: Queue):
self.asc = asc
self.events = events
self.orders_map = {}
self.levels_map = SortedDict()
def add_order(self, order):
self.levels_map.setdefault(order.price, []).append(order)
self.orders_map[order.id] = order
def match_order(self, order):
remove_list = []
for level in self.__iterate_levels():
if self.__compare_price(order.price, level):
self.__match_orders(order, self.levels_map[level], remove_list)
if order.is_matched():
self.events.put({
'name': 'complete',
'order_id': order.id,
})
break
else:
break
# Remove orders after the iteration is complete (so we do not modify the list while iterating over)
for o in remove_list:
self.__remove_order(o)
return order
def __match_orders(self, order, orders, remove_list):
for o in orders:
transferred_amount = o.transfer_amount(order)
self.events.put({
'name': 'match',
'amount': transferred_amount,
'order_id': order.id,
'matched_order_id': o.id,
})
if o.is_matched():
remove_list.append(o)
self.events.put({
'name': 'complete',
'order_id': o.id,
})
if order.is_matched():
break
def __remove_order(self, order):
level = self.levels_map[order.price]
level.remove(order)
if len(level) == 0:
self.levels_map.pop(order.price)
def __compare_price(self, order_price, book_price):
if self.asc:
return order_price >= book_price
else:
return order_price <= book_price
def __iterate_levels(self):
if self.asc:
return iter(self.levels_map)
else:
return reversed(self.levels_map)
def cancel_order_by_id(self, order_id):
order = self.orders_map.get(order_id)
if order is None:
return
self.__remove_order(order)
self.events.put({
'name': 'cancelled',
'order_id': order.id,
'remaining_amount': order.amount_to_match(),
})
def orders(self):
return [o for l in self.levels_map for o in self.levels_map[l]]
def collect_changed_files(self, ticket_frame: pd.DataFrame):
changed_files = SortedDict()
for i, files in enumerate(ticket_frame.ChangedFiles):
for f in files:
changed_files.setdefault(f, []).append(i)
return changed_files
class NetworkEventDataset:
"""Collection of 3-channel ZNE streams with traces aligned to a fixed time window about
seismic P-wave arrival events, *for a given network*.
Two indexes are provided. One indexes hierarchically by station code and
event ID, yielding a 3-channel ZNE stream per event, so that you can easily gather all
traces for a given station by iterating over events.
The other index indexes hierarchically by event ID and station code, yielding a
3-channel ZNE stream per station. Using this index you can easily gather all traces
for a given event across multiple stations.
Preferably each input trace will already have an 'event_id' attribute in its stats. If
not, an event ID will be invented based on station identifiers and time window.
"""
def __init__(self,
stream_src,
network=None,
station=None,
location='',
ordering='ZNE'):
"""
Initialize from data source (file or obspy.Stream). Traces are COPIED into
the dataset in order to leave input object intact, since many obspy functions
mutate traces in-place.
All streams in the input data source stream_src are expected to belong to the same network.
This is checked as the data is ingested. A discrepant network code is an error condition.
:param stream_src: Source of input streams. May be a file name or an Obspy Stream
:type stream_src: str, pathlib.Path or obspy.Stream
:param network: Network code of streams to load. If stream_src is an Obspy Stream, the \
streams will be filtered to match this network code.
:type network: str
:param station: Station code of streams to load. If stream_src is an Obspy Stream, the \
streams will be filtered to match this station code.
:type station: str
:param location: [OPTIONAL] Location code of streams to load. Leave as default (empty string) \
if location code is empty in the data source.
:type location: str
:param ordering: Channel ordering to be applied to the data after loading. The channel labelling \
must be consistent with the requested ordering - rotation to the coordinate system implied \
by the ordering is *NOT* applied.
:type ordering: str
:raises AssertionError: If discrepant network code is found in input data
"""
if isinstance(stream_src, obspy.Stream):
net = network
sta = station
loc = location or None
if net or sta or loc:
data_src = stream_src.select(net, sta, loc)
else:
data_src = stream_src
# end if
elif os.path.isfile(stream_src):
data_src = read_h5_stream(stream_src, network, station, location)
else:
assert False, "Unknown data source {}".format(type(stream_src))
# end if
self.network = network
# Data in data_src collects all traces together under a single Stream object.
# In order to get control over data slicing and traceability in processing, we
# break it down into one Stream per ZNE channel triplet of a given event.
self.db_sta = SortedDict()
for tr in data_src:
net, sta, loc, _ = tr.id.split('.')
if self.network:
assert net == self.network
else:
self.network = net
# end if
# Create single copy of the trace to be shared by both dicts.
dupe_trace = tr.copy()
try:
event_id = tr.stats.event_id
except AttributeError:
event_id = '.'.join([
net, sta, loc,
'_'.join([str(tr.stats.starttime),
str(tr.stats.endtime)])
])
# end try
self.db_sta.setdefault(sta, SortedDict()).setdefault(
event_id, obspy.Stream()).append(dupe_trace)
# end for
# Index same obspy.Stream instances in event dict. This way, any changes
# to a given event stream will be seen by both indexes.
self.db_evid = SortedDict()
for sta, ev_db in self.db_sta.items():
for evid, stream in ev_db.items():
self.db_evid.setdefault(evid, SortedDict())[sta] = stream
# end for
# end for
# Sort each stream into specific order.
if ordering.upper() == 'ZNE':
ordinal = zne_order
elif ordering.upper() == 'ZRT':
ordinal = zrt_order
else:
ordinal = None
# end if
if ordinal is not None:
self.apply(lambda x: x.traces.sort(key=ordinal))
# end if
# end func
def __iter__(self):
"""
Flat iterator. Loops over self.db_sta depth first and returns tuple of keys and matching stream.
Equivalent to::
```Python
for sta, ev_db in self.db_sta.items():
for evid, stream in ev_db.items():
yield (sta, evid, stream)
```
"""
return ((sta, evid, stream) for sta, ev_db in self.db_sta.items()
for evid, stream in ev_db.items())
# end if
def __len__(self):
"""Returns number of streams"""
return sum((len(x) for x in self.db_sta.values()))
# end func
def __repr__(self):
"""Displays summary string for all streams"""
return '\n'.join(
(evid + ', ' + str(stream) for _, evid, stream in iter(self)))
# end func
def num_stations(self):
"""
Get number of stations in the dataset.
:return: Number of stations
:rtype: int
"""
return len(self.db_sta)
# end func
def station(self, station_code):
"""
Accessor for events for a given station.
:param station_code: Station to get
:type station_code: str
:return: Event index for station, if station is found
:rtype: SortedDict
"""
return self.db_sta.get(station_code)
# end func
def num_events(self):
"""
Get number of events in the dataset.
:return: Number of events
:rtype: int
"""
return len(self.db_evid)
# end func
def event(self, event_id):
"""
Accessor for stations for a given event.
:param event_id: ID of event to look up
:type event_id: str
:return: Station index for given event, if event ID is found, otherwise None
:rtype: SortedDict or NoneType
"""
return self.db_evid.get(event_id)
# end func
def curate(self, curator):
"""
Curate the dataset according to a callable curator. Modifies collection in-place to remove
streams that do not satisfy the curation criteria of the callable.
Curator call signature must be consitent with::
callable(station_code, event_id, stream) -> bool
The callable returns a boolean indicating whether to keep the Stream or not.
:param curator: Function or callable delegate to adjudicate whether to keep each given stream.
:type curator: Callable
:return: None
"""
# Only need to loop over one db, since they both reference the same underlying Stream instances.
PY2 = (sys.version_info[0] == 2)
if PY2:
from itertools import ifilterfalse as filterfalse # pylint: disable=no-name-in-module, import-outside-toplevel
else:
from itertools import filterfalse # pylint: disable=import-outside-toplevel
# end if
discard_items = [
(x[0], x[1])
for x in filterfalse(lambda rec: curator(*rec), iter(self))
]
self.prune(discard_items)
# end func
def apply(self, _callable):
"""Apply a callable across all streams. Use to apply uniform processing steps to the whole dataset.
:param _callable: Callable object that takes an obspy Stream as input and applies itself to that Stream. \
Expect that stream may be mutated in-place by the callable.
:type _callable: Any Callable compatible with the call signature.
:return: None
"""
for _1, _2, stream in iter(self):
_callable(stream)
# end func
def by_station(self):
"""
Iterate over station sub-dictionaries.
:return: Iterable over the stations, each element consisting of pair containing \
(station code, event dict).
:rtype: Iterable(tuple)
"""
return iter(self.db_sta.items())
# end func
def by_event(self):
"""
Iterate over event sub-dictionaries.
:return: Iterable over the discrete events, each element consisting of pair containing \
(event id, station dict).
:rtype: Iterable(tuple)
"""
return iter(self.db_evid.items())
# end func
def prune(self, items, cull=True):
"""
Remove a given sequence of (station, event) pairs from the dataset.
:param items: Iterable of (station, event) pairs
:type items: Iterable(tuple)
:param cull: If True, then empty entries in the top level index will be removed.
:type cull: boolean
:return: None
"""
for station, event_id in items:
self.db_sta[station].pop(event_id)
self.db_evid[event_id].pop(station)
if cull:
if not self.db_sta[station]:
self.db_sta.pop(station)
# end if
if not self.db_evid[event_id]:
self.db_evid.pop(event_id)
# end if
# end if
# end for
# end func
def write(self, output_h5_filename, index_format='event'):
"""
Write event dataset back out to HDF5 file.
:param output_h5_filename: Output file name
:type output_h5_filename: str or path
:param index_format: Format to use for index. Must be 'event' (default) or 'standard' (obspy default)
:type index_format: str
:return: True if file was written
:rtype: boolean
"""
assert not os.path.exists(
output_h5_filename), 'Output file already exists'
if index_format not in ['event', 'standard']:
raise ValueError('Index format %s not supported' % index_format)
# end if
all_stream = obspy.Stream()
for sta, evid, stream in iter(self):
all_stream += stream
# end for
if index_format == 'event':
write_h5_event_stream(output_h5_filename, all_stream, mode='w')
elif index_format == 'standard':
all_stream.write(output_h5_filename, format='H5', mode='w')
# end if
return os.path.isfile(output_h5_filename)
class Track:
@staticmethod
def id(t):
"""
Try to parse an object as a Track id
"""
if isinstance(t, int):
return str(t)
if isinstance(t, str):
return t
return t.id
def __init__(self):
self.id: str = None
self.points = SortedDict()
self.color: Tuple[int, int, int] = None
self.tags: Set[str] = set()
def to_dict(self):
ret = {
'id': self.id,
'color': self.color,
'tags': list(self.tags),
# keep points last for easier manual editing
'points': list(self.points.items()),
}
return ret
@classmethod
def from_dict(cls, d):
ret = cls()
ret.id = d['id']
ret.color = tuple(d['color'])
ret.tags.update(x.lower() for x in d.get('tags', ()))
ret.points.update(d['points'])
return ret
def under(self, frame):
"""
Get all points that occur before (exclusive) the frame
"""
keys = self.points.irange(maximum=frame, inclusive=(True, False))
# A SortedDict is just a b-list with a dict attached
# so this is really the most efficient way to get an irange with values.
# https://stackoverflow.com/questions/34099308/iterate-over-a-slice-of-items-in-a-sorteddict
return ((k, self.points[k]) for k in keys)
def add(self, frame, point):
"""
Add a point at the frame
:raise PointOverrideException: If a point already exists at the frame
"""
v = self.points.setdefault(frame, point) # won't add the point if the frame is already filled
if v is not point:
raise PointOverrideException("can't have the same track in two places at the same frame")
def del_point_floor(self, frame):
"""
Deletes the first point that occurs before the frame
:returns: the deleted point, or None if no points were deleted
"""
k = next(iter(self.points.irange(None, frame, reverse=True)), None)
if not k:
return None
v = self.points.pop(k)
return v
def key_span(self):
"""
Get the first and last frames of the track, or None if the track is empty
"""
if not self.points:
return None
k = self.points.keys()
return k[0], k[-1]
def value_span(self):
"""
Get the first and last points of the track, or None if the track is empty
"""
if not self.points:
return None
v = self.points.values()
return v[0], v[-1]
def __str__(self):
return f'Track({self.id})'
def distance(self):
"""
:return: the distance of the path, in pixels
"""
ret = 0
prev = None
for x, y in self.points.values():
if prev:
ret += ((x - prev[0]) ** 2 + (y - prev[1]) ** 2) ** 0.5
prev = (x, y)
return ret
def stats(self, **kwargs: Optional[str]):
"""
Get genric stats about the track
:param kwargs: additional tracks to add to the stats. If the value is falsish, the argument is skipped.
If it's True, the key name is appended.
Otherwise, the key and the value names are appended
"""
ret = [str(self), ': ', str(len(self.points)), ' points']
if self.points:
ret.extend([', frame range: ', str(self.key_span()), ', value range: ', str(self.value_span()), ])
if self.tags:
ret.extend([', tags: ', '[', ', '.join(self.tags), ']'])
ret.extend([', distance: ', str(int(self.distance())), 'px'])
for k, v in kwargs.items():
if not v:
continue
if v is True:
ret.extend([', ', k])
else:
ret.extend([', ', k, ': ', str(v)])
return ''.join(ret)
class DotMap(MutableMapping):
def __init__(self, *args, **kwargs):
self._map = SortedDict()
if args:
d = args[0]
if type(d) is dict:
for k, v in self.__call_items(d):
if type(v) is dict:
v = DotMap(v)
self._map[k] = v
if kwargs:
for k, v in self.__call_items(kwargs):
self._map[k] = v
@staticmethod
def __call_items(obj):
if hasattr(obj, 'iteritems') and ismethod(getattr(obj, 'iteritems')):
return obj.iteritems()
else:
return obj.items()
def items(self):
return self.iteritems()
def iteritems(self):
return self.__call_items(self._map)
def __iter__(self):
return self._map.__iter__()
def __setitem__(self, k, v):
self._map[k] = v
def __getitem__(self, k):
if k not in self._map:
# automatically extend to new DotMap
self[k] = DotMap()
return self._map[k]
def __setattr__(self, k, v):
if k == '_map':
super(DotMap, self).__setattr__(k, v)
else:
self[k] = v
def __getattr__(self, k):
if k == '_map':
return self._map
else:
return self[k]
def __delattr__(self, key):
return self._map.__delitem__(key)
def __contains__(self, k):
return self._map.__contains__(k)
def __str__(self):
items = []
for k, v in self.__call_items(self._map):
items.append('{0}={1}'.format(k, repr(v)))
out = 'DotMap({0})'.format(', '.join(items))
return out
def __repr__(self):
return str(self)
def to_dict(self):
d = {}
for k, v in self.items():
if type(v) is DotMap:
v = v.to_dict()
d[k] = v
return d
def pprint(self):
pprint(self.to_dict())
# proper dict subclassing
def values(self):
return self._map.values()
@staticmethod
def parse_other(other):
if type(other) is DotMap:
return other._map
else:
return other
def __cmp__(self, other):
other = DotMap.parse_other(other)
return self._map.__cmp__(other)
def __eq__(self, other):
other = DotMap.parse_other(other)
if not isinstance(other, dict):
return False
return self._map.__eq__(other)
def __ge__(self, other):
other = DotMap.parse_other(other)
return self._map.__ge__(other)
def __gt__(self, other):
other = DotMap.parse_other(other)
return self._map.__gt__(other)
def __le__(self, other):
other = DotMap.parseOther(other)
return self._map.__le__(other)
def __lt__(self, other):
other = DotMap.parse_other(other)
return self._map.__lt__(other)
def __ne__(self, other):
other = DotMap.parse_other(other)
return self._map.__ne__(other)
def __delitem__(self, key):
return self._map.__delitem__(key)
def __len__(self):
return self._map.__len__()
def copy(self):
return self
def get(self, key, default=None):
return self._map.get(key, default)
def has_key(self, key):
return key in self._map
def iterkeys(self):
return self._map.iterkeys()
def itervalues(self):
return self._map.itervalues()
def keys(self):
return self._map.keys()
def pop(self, key, default=None):
return self._map.pop(key, default)
def setdefault(self, key, default=None):
return self._map.setdefault(key, default)
def viewitems(self):
if version_info.major == 2 and version_info.minor >= 7:
return self._map.viewitems()
else:
return self._map.items()
def viewkeys(self):
if version_info.major == 2 and version_info.minor >= 7:
return self._map.viewkeys()
else:
return self._map.keys()
def viewvalues(self):
if version_info.major == 2 and version_info.minor >= 7:
return self._map.viewvalues()
else:
return self._map.values()
@classmethod
def fromkeys(cls, seq, value=None):
d = DotMap()
d._map = SortedDict.fromkeys(seq, value)
return d
class Controller(object):
def __init__(self, thrift_server, thrift_port):
self.transport = TSocket.TSocket(thrift_server, thrift_port)
self.transport = TTransport.TBufferedTransport(self.transport)
bprotocol = TBinaryProtocol.TBinaryProtocol(self.transport)
conn_mgr_protocol = TMultiplexedProtocol.TMultiplexedProtocol(
bprotocol, "conn_mgr")
self.conn_mgr = conn_mgr_pd_rpc.conn_mgr.Client(conn_mgr_protocol)
p4_protocol = TMultiplexedProtocol.TMultiplexedProtocol(
bprotocol, "pegasus")
self.pegasus = pegasus.p4_pd_rpc.pegasus.Client(p4_protocol)
self.devport = devport_mgr_pd_rpc.devport_mgr.Client(conn_mgr_protocol)
self.transport.open()
self.sess_hdl = self.conn_mgr.client_init()
self.dev = 0
self.dev_tgt = DevTarget_t(self.dev, hex_to_i16(0xFFFF))
self.flags = pegasus_register_flags_t(read_hw_sync=True)
# keyhash -> ReplicatedKey (sorted in ascending load)
self.replicated_keys = SortedDict(
lambda x: self.replicated_keys[x].load)
self.num_nodes = DEFAULT_NUM_NODES
self.num_rkeys = MAX_NRKEYS
self.switch_lock = threading.Lock()
def install_table_entries(self, tables):
# tab_l2_forward
self.pegasus.tab_l2_forward_set_default_action__drop(
self.sess_hdl, self.dev_tgt)
for (mac, port) in tables["tab_l2_forward"].items():
self.pegasus.tab_l2_forward_table_add_with_l2_forward(
self.sess_hdl, self.dev_tgt,
pegasus_tab_l2_forward_match_spec_t(
ethernet_dstAddr=macAddr_to_string(mac)),
pegasus_l2_forward_action_spec_t(action_port=port))
# tab_node_forward
self.pegasus.tab_node_forward_set_default_action__drop(
self.sess_hdl, self.dev_tgt)
self.num_nodes = len(tables["tab_node_forward"])
for (node, attrs) in tables["tab_node_forward"].items():
self.pegasus.tab_node_forward_table_add_with_node_forward(
self.sess_hdl, self.dev_tgt,
pegasus_tab_node_forward_match_spec_t(meta_node=int(node)),
pegasus_node_forward_action_spec_t(
action_mac_addr=macAddr_to_string(attrs["mac"]),
action_ip_addr=ipv4Addr_to_i32(attrs["ip"]),
action_udp_addr=attrs["udp"],
action_port=attrs["port"]))
# reg_n_servers
self.pegasus.register_write_reg_n_servers(self.sess_hdl, self.dev_tgt,
0, self.num_nodes)
# tab_calc_rset_index
for i in range(self.num_rkeys):
self.pegasus.tab_calc_rset_index_table_add_with_calc_rset_index(
self.sess_hdl, self.dev_tgt,
pegasus_tab_calc_rset_index_match_spec_t(meta_rkey_index=i),
pegasus_calc_rset_index_action_spec_t(action_base=i *
MAX_RSET_SIZE))
# tab_replicated_keys
self.all_rkeys = tables["tab_replicated_keys"]
for i in range(self.num_rkeys):
self.add_rkey(self.all_rkeys[i], i, 0)
self.conn_mgr.complete_operations(self.sess_hdl)
def reset(self):
self.switch_lock.acquire()
for keyhash in self.replicated_keys.keys():
self.pegasus.tab_replicated_keys_table_delete_by_match_spec(
self.sess_hdl, self.dev_tgt,
pegasus_tab_replicated_keys_match_spec_t(
pegasus_keyhash=int(keyhash)))
self.replicated_keys.clear()
for i in range(self.num_rkeys):
self.add_rkey(self.all_rkeys[i], i, 0)
self.pegasus.register_write_reg_ver_next(self.sess_hdl, self.dev_tgt,
0, 1)
self.pegasus.register_write_reg_n_servers(self.sess_hdl, self.dev_tgt,
0, self.num_nodes)
self.pegasus.register_write_reg_rr_all_servers(self.sess_hdl,
self.dev_tgt, 0, 0)
self.conn_mgr.complete_operations(self.sess_hdl)
self.switch_lock.release()
def add_rkey(self, keyhash, rkey_index, load):
if RSET_ALL:
self.pegasus.register_write_reg_rset_size(self.sess_hdl,
self.dev_tgt, rkey_index,
self.num_nodes)
bitmap = (2**self.num_nodes) - 1
self.pegasus.register_write_reg_rset_bitmap(
self.sess_hdl, self.dev_tgt, rkey_index, bitmap)
rset_index = rkey_index << RSET_INDEX_SHIFT
for i in range(self.num_nodes):
self.pegasus.register_write_reg_rset(self.sess_hdl,
self.dev_tgt,
rset_index + i, i)
else:
node = int(keyhash) % self.num_nodes
bitmap = 1 << node
rset_index = rkey_index << RSET_INDEX_SHIFT
self.pegasus.register_write_reg_rset_size(self.sess_hdl,
self.dev_tgt, rkey_index,
1)
self.pegasus.register_write_reg_rset_bitmap(
self.sess_hdl, self.dev_tgt, rkey_index, bitmap)
self.pegasus.register_write_reg_rset(self.sess_hdl, self.dev_tgt,
rset_index, node)
self.pegasus.register_write_reg_rkey_ver_completed(
self.sess_hdl, self.dev_tgt, rkey_index, 1)
self.pegasus.register_write_reg_rkey_read_counter(
self.sess_hdl, self.dev_tgt, rkey_index, 0)
self.pegasus.register_write_reg_rkey_write_counter(
self.sess_hdl, self.dev_tgt, rkey_index, 0)
self.pegasus.register_write_reg_rkey_rate_counter(
self.sess_hdl, self.dev_tgt, rkey_index, 0)
self.pegasus.register_write_reg_rr_rkey(self.sess_hdl, self.dev_tgt,
rkey_index, 0)
self.pegasus.tab_replicated_keys_table_add_with_is_rkey(
self.sess_hdl, self.dev_tgt,
pegasus_tab_replicated_keys_match_spec_t(
pegasus_keyhash=int(keyhash)),
pegasus_is_rkey_action_spec_t(action_rkey_index=rkey_index))
self.replicated_keys.setdefault(
keyhash, ReplicatedKey(index=int(rkey_index), load=load))
def periodic_update(self):
self.switch_lock.acquire()
# Reset read and write counters
self.pegasus.register_reset_all_reg_rkey_read_counter(
self.sess_hdl, self.dev_tgt)
self.pegasus.register_reset_all_reg_rkey_write_counter(
self.sess_hdl, self.dev_tgt)
# Read rkey load
for rkey in self.replicated_keys.values():
read_value = self.pegasus.register_read_reg_rkey_rate_counter(
self.sess_hdl, self.dev_tgt, rkey.index, self.flags)
rkey.load = int(read_value[1])
# Reset rkey load
self.pegasus.register_reset_all_reg_rkey_rate_counter(
self.sess_hdl, self.dev_tgt)
self.conn_mgr.complete_operations(self.sess_hdl)
self.switch_lock.release()
def print_stats(self):
self.switch_lock.acquire()
# read replicated keys info
for (keyhash, rkey) in self.replicated_keys.items():
read_value = self.pegasus.register_read_reg_rkey_rate_counter(
self.sess_hdl, self.dev_tgt, rkey.index, self.flags)
rkey.load = read_value[0]
print "rkey hash", keyhash
print "rkey load", rkey.load
read_value = self.pegasus.register_read_reg_rkey_ver_completed(
self.sess_hdl, self.dev_tgt, rkey.index, self.flags)
print "ver completed", read_value[0]
read_value = self.pegasus.register_read_reg_rset_size(
self.sess_hdl, self.dev_tgt, rkey.index, self.flags)
rset_size = int(read_value[0])
print "rset size", rset_size
read_value = self.pegasus.register_read_reg_rset_bitmap(
self.sess_hdl, self.dev_tgt, rkey.index, self.flags)
print "rset bitmap", read_value[0]
base = rkey.index * MAX_RSET_SIZE
for i in range(rset_size):
read_value = self.pegasus.register_read_reg_rset(
self.sess_hdl, self.dev_tgt, base + i, self.flags)
print "replica", read_value[0]
self.conn_mgr.complete_operations(self.sess_hdl)
self.switch_lock.release()
def run(self):
while True:
time.sleep(0.1)
self.periodic_update()
def stop(self):
self.transport.close()
def test_setdefault():
mapping = [(val, pos) for pos, val in enumerate(string.ascii_lowercase)]
temp = SortedDict(mapping)
assert temp.setdefault('a', -1) == 0
assert temp['a'] == 0
assert temp.setdefault('A', -1) == -1
class ImpulseDiGraph(ImpulseGraph):
"""Base class for directed impulse graphs.
The ImpulseDiGraph class allows any hashable object as a node
and can associate key/value attribute pairs with each directed edge.
Each edge must have one integer, timestamp.
Self-loops are allowed.
Multiple edges between two nodes are allowed.
Parameters
----------
attr : keyword arguments, optional (default= no attributes)
Attributes to add to graph as key=value pairs.
Examples
--------
Create an empty graph structure (a "null impulse graph") with no nodes and
no edges.
>>> G = dnx.ImpulseDiGraph()
G can be grown in several ways.
**Nodes:**
Add one node at a time:
>>> G.add_node(1)
Add the nodes from any container (a list, dict, set or
even the lines from a file or the nodes from another graph).
Add the nodes from any container (a list, dict, set)
>>> G.add_nodes_from([2, 3])
>>> G.add_nodes_from(range(100, 110))
**Edges:**
G can also be grown by adding edges. This can be considered
the primary way to grow G, since nodes with no edge will not
appear in G in most cases. See ``G.to_snapshot()``.
Add one edge, with timestamp of 10.
>>> G.add_edge(1, 2, 10)
a list of edges,
>>> G.add_edges_from([(1, 2, 10), (1, 3, 11)])
If some edges connect nodes not yet in the graph, the nodes
are added automatically. There are no errors when adding
nodes or edges that already exist.
**Attributes:**
Each impulse graph, node, and edge can hold key/value attribute pairs
in an associated attribute dictionary (the keys must be hashable).
By default these are empty, but can be added or changed using
add_edge, add_node.
Keep in mind that the edge timestamp is not an attribute of the edge.
>>> G = dnx.ImpulseDiGraph(day="Friday")
>>> G.graph
{'day': 'Friday'}
Add node attributes using add_node(), add_nodes_from()
>>> G.add_node(1, time='5pm')
>>> G.add_nodes_from([3], time='2pm')
Add edge attributes using add_edge(), add_edges_from().
>>> G.add_edge(1, 2, 10, weight=4.7 )
>>> G.add_edges_from([(3, 4, 11), (4, 5, 33)], color='red')
**Shortcuts:**
Here are a couple examples of available shortcuts:
>>> 1 in G # check if node in impulse graph during any timestamp
True
>>> len(G) # number of nodes in the entire impulse graph
5
**Subclasses (Advanced):**
Edges in impulse graphs are represented by tuples kept in a SortedDict
(http://www.grantjenks.com/docs/sortedcontainers/) keyed by timestamp.
The Graph class uses a dict-of-dict-of-dict data structure.
The outer dict (node_dict) holds adjacency information keyed by nodes.
The next dict (adjlist_dict) represents the adjacency information and holds
edge data keyed by interval objects. The inner dict (edge_attr_dict) represents
the edge data and holds edge attribute values keyed by attribute names.
"""
def __init__(self, **attr):
"""Initialize an impulse graph with edges, name, or graph attributes.
Parameters
----------
attr : keyword arguments, optional (default= no attributes)
Attributes to add to graph as key=value pairs.
Examples
--------
>>> G = dnx.ImpulseDiGraph()
>>> G = dnx.ImpulseDiGraph(name='my graph')
>>> G.graph
{'name': 'my graph'}
"""
self.tree = SortedDict()
self.graph = {} # dictionary for graph attributes
self._node = {}
self._pred = {} # out
self._succ = {} # in
self.edgeid = 0
self.graph.update(attr)
def add_edge(self, u, v, t, **attr):
"""Add an edge between u and v, at t.
The nodes u and v will be automatically added if they are
not already in the impulse graph.
Edge attributes can be specified with keywords or by directly
accessing the edge's attribute dictionary. See examples below.
Parameters
----------
u, v : nodes
Nodes can be, for example, strings or numbers.
Nodes must be hashable (and not None) Python objects.
t : timestamp
Timestamps can be, for example, strings or numbers.
Timestamps must be hashable (and not None) Python objects.
attr : keyword arguments, optional
Edge data (or labels or objects) can be assigned using
keyword arguments.
See Also
--------
add_edges_from : add a collection of edges
Notes
-----
Adding an edge that already exists updates the edge data.
Timestamps must be the same type across all edges in the impulse graph.
Also, to create snapshots, timestamps must be integers.
Many NetworkX algorithms designed for weighted graphs use
an edge attribute (by default `weight`) to hold a numerical value.
Examples
--------
The following all add the edge e=(1, 2, 3, 10) to graph G:
>>> G = dnx.ImpulseDiGraph()
>>> e = (1, 2, 10)
>>> G.add_edge(1, 2, 10) # explicit two-node form with timestamp
>>> G.add_edge(*e) # single edge as tuple of two nodes and timestamp
>>> G.add_edges_from([(1, 2, 10)]) # add edges from iterable container
Associate data to edges using keywords:
>>> G.add_edge(1, 2, 10 weight=3)
>>> G.add_edge(1, 3, 9, weight=7, capacity=15, length=342.7)
"""
self.tree.setdefault(t, set()).add((u, v))
self._node.setdefault(u, {})
self._node.setdefault(v, {})
self._pred.setdefault(u, {}).setdefault(v, {})[(u, v, t)] = attr
self._succ.setdefault(v, {}).setdefault(u, {})[(u, v, t)] = attr
def add_edges_from(self, ebunch_to_add, **attr):
"""Add all the edges in ebunch_to_add.
Parameters
----------
ebunch_to_add : container of edges
Each edge given in the container will be added to the
impulse graph. The edges must be given as as 3-tuples (u, v, t).
Timestamp must be orderable and the same type across all edges.
attr : keyword arguments, optional
Edge data (or labels or objects) can be assigned using
keyword arguments.
See Also
--------
add_edge : add a single edge
Notes
-----
Adding the same edge (with the same timestamp) twice has no effect
but any edge data will be updated when each duplicate edge is added.
Examples
--------
>>> G = dnx.ImpulseDiGraph()
>>> G.add_edges_from([(1, 2, 10), (2, 4, 11)]) # using a list of edge tuples
Associate data to edges
>>> G.add_edges_from([(1, 2, 10), (2, 4, 11)], weight=3)
>>> G.add_edges_from([(3, 4, 19), (1, 4, 3)], label='WN2898')
"""
for e in ebunch_to_add:
if len(e) != 3:
raise NetworkXError(
"Edge tuple {0} must be a 3-tuple.".format(e))
self.add_edge(e[0], e[1], e[2], **attr)
def has_edge(self, u, v, begin=None, end=None, inclusive=(True, True)):
"""Return True if there exists an edge between u and v
in the impulse graph, during the given interval.
Parameters
----------
u, v : nodes
Nodes can be, for example, strings or numbers.
Nodes must be hashable (and not None) Python objects.
begin : int or float, optional (default= beginning of the entire impulse graph)
end : int or float, optional (default= end of the entire impulse graph)
Must be bigger than or equal begin.
inclusive: 2-tuple boolean that determines inclusivity of begin and end
Examples
--------
>>> G = dnx.ImpulseDiGraph()
>>> G.add_edges_from([(1, 2, 10), (2, 4, 11)])
>>> G.has_edge(1, 2)
True
>>> G.has_edge(1, 2, begin=2)
True
>>> G.has_edge(2, 4, begin=12)
False
"""
if u not in self._pred or v not in self._pred[u]:
return False
if begin is None and end is None:
return True
if begin and end and begin > end:
raise NetworkXError(
"IntervalGraph: interval end must be bigger than or equal to begin: "
"begin: {}, end: {}.".format(begin, end))
for iv in self._pred[u][v]:
if self.__in_interval(iv[2], begin, end, inclusive=inclusive):
return True
return False
def edges(self,
u=None,
v=None,
begin=None,
end=None,
inclusive=(True, True),
data=False,
default=None):
"""Returns a list of tuples of the ImpulseDiGraph edges.
All edges which are present within the given interval.
All parameters are optional. `u` and `v` can be thought of as constraints.
If no node is defined, all edges within the interval are returned.
If one node is defined, all edges which have that node as one end,
will be returned, and finally if both nodes are defined then all
edges between the two nodes are returned.
Parameters
----------
u, v : nodes, optional (default=None)
Nodes can be, for example, strings or numbers.
Nodes must be hashable (and not None) Python objects.
If the node does not exist in the graph, a key error is raised.
begin: int or float, optional (default= beginning of the entire impulse graph)
end: int or float, optional (default= end of the entire impulse graph)
Must be bigger than or equal to begin.
inclusive: 2-tuple boolean that determines inclusivity of begin and end
data : string or bool, optional (default=False)
If True, return 2-tuple (Edge Tuple, dict of attributes).
If False, return just the Edge Tuples.
If string (name of the attribute), return 2-tuple (Edge Tuple, attribute value).
default : value, optional (default=None)
Default Value to be used for edges that don't have the requested attribute.
Only relevant if `data` is a string (name of an attribute).
Returns
-------
List of Edge Tuples
An edge tuple has the following format: (u, v, edge_id, timestamp)
When called, if `data` is False, a list of edge tuples.
If `data` is True, a list of 2-tuples: (Edge Tuple, dict of attribute(s) with values),
If `data` is a string, a list of 2-tuples (Edge Tuple, attribute value).
Examples
--------
To get a list of all edges:
>>> G = dnx.ImpulseDiGraph()
>>> G.add_edges_from([(1, 2, 10), (2, 4, 11), (6, 4, 19), (2, 4, 15)])
>>> G.edges()
[(1, 2, 10), (2, 4, 11), (2, 4, 15), (6, 4, 19)]
To get edges which appear in a specific interval:
>>> G.edges(begin=10)
[(1, 2, 10), (2, 4, 11), (2, 4, 15), (6, 4, 19)]
>>> G.edges(end=11)
[(1, 2, 10), (2, 4, 11)]
>>> G.edges(begin=11, end=15)
[(2, 4, 11), (2, 4, 15)]
To get edges with either of the two nodes being defined:
>>> G.edges(u=2)
[(2, 4, 11), (2, 4, 15)]
>>> G.edges(u=2, begin=11)
[(2, 4, 11), (2, 4, 15)]
>>> G.edges(u=2, v=4, end=11)
[(2, 4, 11)]
>>> G.edges(u=1, v=6)
[]
To get a list of edges with data:
>>> G = dnx.ImpulseDiGraph()
>>> G.add_edge(1, 3, 4, weight=8, height=18)
>>> G.add_edge(1, 2, 10, weight=10)
>>> G.add_edge(2, 6, 10)
>>> G.edges(data="weight")
[((1, 3, 4), 8), ((1, 2, 10), 10), ((2, 6, 10), None)]
>>> G.edges(data="weight", default=5)
[((1, 3, 4), 8), ((1, 2, 10), 10), ((2, 6, 10), 5)]
>>> G.edges(data=True)
[((1, 3, 4), {'weight': 8, 'height': 18}), ((1, 2, 10), {'weight': 10}), ((2, 6, 10), {})]
>>> G.edges(u=1, begin=2, end=9, data="weight")
[((1, 3, 4), 8)]
"""
if begin is None:
inclusive = (True, inclusive[1])
if end is None:
inclusive = (inclusive[0], True)
if u is None and v is None:
if begin is not None and end is not None and begin > end:
raise NetworkXError(
"IntervalGraph: interval end must be bigger than or equal to begin: "
"begin: {}, end: {}.".format(begin, end))
iedges = [iv for iv in self.__search_tree(begin, end, inclusive)]
else:
# Node filtering
if u is not None and v is not None:
if u not in self._pred:
return []
if v not in self._pred[u]:
return []
iedges = self._pred[u][v]
elif u is not None:
if u not in self._pred:
return []
iedges = [iv for v in self._pred[u] for iv in self._pred[u][v]]
else:
if v not in self._succ:
return []
iedges = [iv for u in self._succ[v] for iv in self._succ[v][u]]
# Interval filtering
if begin is not None and end is not None and begin > end:
raise NetworkXError(
"IntervalGraph: interval end must be bigger than or equal to begin: "
"begin: {}, end: {}.".format(begin, end))
iedges = [
iv for iv in iedges
if self.__in_interval(iv[2], begin, end, inclusive=inclusive)
]
if data is False:
return [edge for edge in iedges]
if data is True:
return [(edge, self._pred[edge[0]][edge[1]][edge])
for edge in iedges]
return [(edge, self._pred[edge[0]][edge[1]][edge][data]) if data
in self._pred[edge[0]][edge[1]][edge] else (edge, default)
for edge in iedges]
def remove_edge(self, u, v, begin=None, end=None, inclusive=(True, True)):
"""Remove the edge between u and v in the impulse graph,
during the given interval.
Quiet if the specified edge is not present.
Parameters
----------
u, v : nodes
Nodes can be, for example, strings or numbers.
Nodes must be hashable (and not None) Python objects.
begin : int or float, optional (default= beginning of the entire impulse graph)
end : int or float, optional (default= end of the entire impulse graph + 1)
Must be bigger than or equal to begin.
inclusive: 2-tuple boolean that determines inclusivity of begin and end
Examples
--------
>>> G = dnx.ImpulseDiGraph()
>>> G.add_edges_from([(1, 2, 10), (2, 4, 11), (6, 4, 9), (1, 2, 15)])
>>> G.remove_edge(1, 2)
>>> G.has_edge(1, 2)
False
>>> G = dnx.ImpulseDiGraph()
>>> G.add_edges_from([(1, 2, 10), (2, 4, 11), (6, 4, 9), (1, 2, 15)])
>>> G.remove_edge(1, 2, begin=2, end=11)
>>> G.has_edge(1, 2, begin=2, end=11)
False
>>> G.has_edge(1, 2)
True
"""
if u not in self._pred or v not in self._pred[u]:
return
iedges_to_remove = []
# remove every edge between u and v
if begin is None and end is None:
for iv in self._pred[u][v]:
iedges_to_remove.append(iv)
else:
for iv in self._pred[u][v]:
if self.__in_interval(iv[2], begin, end):
iedges_to_remove.append(iv)
# removing found iedges
for edge in iedges_to_remove:
self.__remove_iedge(edge)
# clean up empty dictionaries
if len(self._pred[u][v]) == 0:
self._pred[u].pop(v, None)
if len(self._succ[v][u]) == 0:
self._succ[v].pop(u, None)
if len(self._pred[u]) == 0:
self._pred.pop(u, None)
if len(self._succ[v]) == 0:
self._succ.pop(v, None)
def degree(self,
node=None,
begin=None,
end=None,
delta=False,
inclusive=(True, True)):
"""Return the sum of in and out degree of a specified node between time begin and end.
Parameters
----------
node : Nodes can be, for example, strings or numbers.
Nodes must be hashable (and not None) Python objects.
begin : int or float, optional (default= beginning of the entire impulse graph)
Inclusive beginning time of the edge appearing in the impulse graph.
end : int or float, optional (default= end of the entire impulse graph)
Non-inclusive ending time of the edge appearing in the impulse graph.
delta : boolean, optional (default= False)
Returns list of 2-tuples, first element is the timestamp, second is the node of changing degree.
inclusive : 2-tuple boolean that determines inclusivity of begin and end
Returns
-------
Integer value of degree of specified node.
Examples
--------
>>> G = dnx.ImpulseDiGraph()
>>> G.add_edge(1, 2, 3)
>>> G.add_edge(2, 3, 8)
>>> G.degree(2)
2
>>> G.degree(2, 4)
1
>>> G.degree(2, end=8)
2
>>> G.degree()
1.33333
>>> G.degree(2, delta=True)
[(3, 1), (8, 1)]
"""
# no specified node, return mean degree
if node == None:
n = 0
l = 0
for node in self.nodes(begin=begin, end=end, inclusive=inclusive):
n += 1
l += self.degree(node,
begin=begin,
end=end,
inclusive=inclusive)
return l / n
# specified node, no degree_change, return degree
if delta == False:
return len(self.edges(u=node, begin=begin, end=end, inclusive=inclusive)) + \
len(self.edges(v=node, begin=begin, end=end, inclusive=inclusive))
# delta == True, return list of changes
if begin == None:
begin = list(self.tree.keys())[0]
if end == None:
end = list(self.tree.keys())[-1]
d = {}
output = []
# for each edge determine if the begin and/or end value is in specified time period
for edge in self.edges(u=node,
begin=begin,
end=end,
inclusive=(True, True)):
d.setdefault(edge[2], []).append((edge[0], edge[1]))
for edge in self.edges(v=node,
begin=begin,
end=end,
inclusive=(True, True)):
d.setdefault(edge[2], []).append((edge[0], edge[1]))
# for each time in Dict add to output list the len of each value
for time in d:
output.append((time, len(d[time])))
return sorted(output)
def in_degree(self,
node=None,
begin=None,
end=None,
delta=False,
inclusive=(True, True)):
"""Return the in-degree of a specified node between time begin and end.
Parameters
----------
node : Nodes can be, for example, strings or numbers.
Nodes must be hashable (and not None) Python objects.
begin : int or float, optional (default= beginning of the entire impulse graph)
Inclusive beginning time of the edge appearing in the impulse graph.
end : int or float, optional (default= end of the entire impulse graph)
Non-inclusive ending time of the edge appearing in the impulse graph.
delta : boolean, optional (default= False)
Returns list of 2-tuples, first element is the timestamp, second is the node of changing degree.
inclusive : 2-tuple boolean that determines inclusivity of begin and end
Returns
-------
Integer value of in-degree of specified node.
Examples
--------
>>> G = dnx.ImpulseDiGraph()
>>> G.add_edge(1, 2, 3)
>>> G.add_edge(2, 3, 8)
>>> G.in_degree(2)
1
>>> G.in_degree(2, 4)
0
>>> G.in_degree(2, end=8)
1
>>> G.in_degree()
0.66666
>>> G.in_degree(2, delta=True)
[(3, 1)]
"""
# no specified node, return mean degree
if node == None:
n = 0
l = 0
for node in self.nodes(begin=begin, end=end, inclusive=inclusive):
n += 1
l += self.in_degree(node,
begin=begin,
end=end,
inclusive=inclusive)
return l / n
# specified node, no degree_change, return degree
if delta == False:
return len(
self.edges(v=node, begin=begin, end=end, inclusive=inclusive))
# delta == True, return list of changes
if begin == None:
begin = list(self.tree.keys())[0]
if end == None:
end = list(self.tree.keys())[-1]
d = {}
output = []
# for each edge determine if the begin and/or end value is in specified time period
for edge in self.edges(v=node,
begin=begin,
end=end,
inclusive=(True, True)):
d.setdefault(edge[2], []).append((edge[0], edge[1]))
# for each time in Dict add to output list the len of each value
for time in d:
output.append((time, len(d[time])))
return output
def out_degree(self,
node=None,
begin=None,
end=None,
delta=False,
inclusive=(True, True)):
"""Return the out-degree of a specified node between time begin and end.
Parameters
----------
node : Nodes can be, for example, strings or numbers.
Nodes must be hashable (and not None) Python objects.
begin : int or float, optional (default= beginning of the entire impulse graph)
Inclusive beginning time of the edge appearing in the impulse graph.
end : int or float, optional (default= end of the entire impulse graph)
Non-inclusive ending time of the edge appearing in the impulse graph.
delta : boolean, optional (default= False)
Returns list of 2-tuples, first element is the timestamp, second is the node of changing degree.
inclusive : 2-tuple boolean that determines inclusivity of begin and end
Returns
-------
Integer value of out-degree of specified node.
Examples
--------
>>> G = dnx.ImpulseDiGraph()
>>> G.add_edge(1, 2, 3)
>>> G.add_edge(2, 3, 8)
>>> G.out_degree(2)
1
>>> G.out_degree(2, 2)
1
>>> G.out_degree(2, end=8)
1
>>> G.out_degree()
0.66666
>>> G.out_degree(2, delta=True)
[(8, 1)]
"""
# no specified node, return mean degree
if node == None:
n = 0
l = 0
for node in self.nodes(begin=begin, end=end, inclusive=inclusive):
n += 1
l += self.in_degree(node,
begin=begin,
end=end,
inclusive=inclusive)
return l / n
# specified node, no degree_change, return degree
if delta == False:
return len(
self.edges(u=node, begin=begin, end=end, inclusive=inclusive))
# delta == True, return list of changes
if begin == None:
begin = list(self.tree.keys())[0]
if end == None:
end = list(self.tree.keys())[-1]
d = {}
output = []
# for each edge determine if the begin and/or end value is in specified time period
for edge in self.edges(u=node,
begin=begin,
end=end,
inclusive=(True, True)):
d.setdefault(edge[2], []).append((edge[0], edge[1]))
# for each time in Dict add to output list the len of each value
for time in d:
output.append((time, len(d[time])))
return output
def to_networkx_graph(self,
begin=None,
end=None,
inclusive=(True, False),
multigraph=False,
edge_data=False,
edge_timestamp_data=False,
node_data=False):
"""Return a networkx Graph or MultiGraph which includes all the nodes and
edges which have timestamps within the given interval.
Wrapper function for ImpulseGraph.to_subgraph. Refer to ImpulseGraph.to_subgraph for full description.
"""
return self.to_subgraph(begin=begin,
end=end,
inclusive=inclusive,
multigraph=multigraph,
edge_data=edge_data,
edge_timestamp_data=edge_timestamp_data,
node_data=node_data)
def to_subgraph(self,
begin,
end,
inclusive=(True, False),
multigraph=False,
edge_data=False,
edge_timestamp_data=False,
node_data=False):
"""Return a networkx Graph or MultiGraph which includes all the nodes and
edges which have timestamps within the given interval.
Parameters
----------
begin: int or float
end: int or float
Must be bigger than or equal to begin.
inclusive: 2-tuple boolean that determines inclusivity of begin and end
multigraph: bool, optional (default= False)
If True, a networkx MultiGraph will be returned. If False, networkx Graph.
edge_data: bool, optional (default= False)
If True, edges will keep their attributes.
edge_timestamp_data: bool, optional (default= False)
If True, each edge's attribute will also include its timestamp data.
If `edge_data= True` and there already exist edge attributes named timestamp
it will be overwritten.
node_data : bool, optional (default= False)
if True, each node's attributes will be included.
See Also
--------
to_snapshots : divide the impulse graph to snapshots
Notes
-----
If multigraph= False, and edge_data=True or edge_interval_data=True,
in case there are multiple edges, only one will show with one of the edge's attributes.
Note: nodes with no edges will not appear in any subgraph.
Examples
--------
>>> G = dnx.ImpulseGraph()
>>> G.add_edges_from([(1, 2, 10), (2, 4, 11), (6, 4, 19), (2, 4, 15)])
>>> H = G.to_subgraph(4, 12)
>>> type(H)
<class 'networkx.classes.graph.DiGraph'>
>>> list(H.edges(data=True))
[(1, 2, {}), (2, 4, {})]
>>> H = G.to_subgraph(10, 12, edge_timestamp_data=True)
>>> type(H)
<class 'networkx.classes.graph.DiGraph'>
>>> list(H.edges(data=True))
[(1, 2, {'timestamp': 10}), (2, 4, {'timestamp': 11})]
>>> M = G.to_subgraph(4, 12, multigraph=True, edge_timestamp_data=True)
>>> type(M)
<class 'networkx.classes.multigraph.MultiDiGraph'>
>>> list(M.edges(data=True))
[(1, 2, {'timestamp': 10}), (2, 4, {'timestamp': 11})]
"""
iedges = self.__search_tree(begin, end, inclusive=inclusive)
if multigraph:
G = MultiDiGraph()
else:
G = DiGraph()
if edge_data and edge_timestamp_data:
G.add_edges_from((iedge[0], iedge[1],
dict(self._pred[iedge[0]][iedge[1]][iedge],
timestamp=iedge[3])) for iedge in iedges)
elif edge_data:
G.add_edges_from(
(iedge[0], iedge[1], self._pred[iedge[0]][iedge[1]][iedge])
for iedge in iedges)
elif edge_timestamp_data:
G.add_edges_from((iedge[0], iedge[1], {
'timestamp': iedge[3]
}) for iedge in iedges)
else:
G.add_edges_from((iedge[0], iedge[1]) for iedge in iedges)
if node_data:
G.add_nodes_from((n, self._node[n].copy()) for n in G.nodes)
return G
def __remove_iedge(self, iedge):
"""Remove the impulse edge from the impulse graph.
Quiet if the specified edge is not present.
Parameters
----------
iedge : Edge Tuple (u,v,eid,t)
Edge to be removed.
"""
try:
self.tree[iedge[2]].remove((iedge[0], iedge[1]))
del self._pred[iedge[0]][iedge[1]][iedge]
del self._succ[iedge[1]][iedge[0]][iedge]
except:
return
def __validate_interval(self, begin=None, end=None):
"""Returns validated begin and end.
Raises an exception if begin is larger than end.
Parameters
----------
begin : int or float, optional
end : int or float, optional
"""
if (begin is not None and end is not None) and begin > end:
raise NetworkXError(
"ImpulseDiGraph: interval end must be bigger than or equal to begin: "
"begin: {}, end: {}.".format(begin, end))
return begin, end
def __search_tree(self, begin=None, end=None, inclusive=(True, True)):
"""if begin and end are equal performs a point search on the tree,
otherwise an interval search is performed.
Parameters
----------
begin: int or float, optional (default= beginning of the entire impulse graph)
end: int or float, optional (default= end of the entire impulse graph)
Must be bigger than or equal begin.
inclusive: 2-tuple boolean that determines inclusivity of begin and end
"""
begin, end = self.__validate_interval(begin, end)
if begin is not None and begin == end and begin in self.tree:
for edge in self.tree[begin]:
yield (*edge, begin)
for t in self.tree.irange(begin, end, inclusive=inclusive):
for edge in self.tree[t]:
yield (*edge, t)
def __in_interval(self, t, begin, end, inclusive=(True, True)):
"""
Parameters
----------
t: int or float, timestamp
begin: int or float
Beginning time of Interval.
end: int or float
Ending time of Interval.
Must be bigger than or equal begin.
inclusive: 2-tuple boolean that determines inclusivity of begin and end
Returns
-------
Returns True if t is in the interval (begin,end). Otherwise False.
"""
if begin is None:
begin = float('-inf')
if end is None:
end = float('inf')
if inclusive == (True, True):
return begin <= t <= end
if inclusive == (True, False):
return begin <= t < end
if inclusive == (False, True):
return begin < t <= end
if inclusive == (False, False):
return begin < t < end
@staticmethod
def load_from_txt(path,
delimiter=" ",
nodetype=int,
timestamptype=float,
order=('u', 'v', 't'),
comments="#"):
"""Read impulse graph in from path.
Timestamps must be integers or floats.
Nodes can be any hashable objects.
Edge Attributes can be assigned with in the following format: Key=Value
Parameters
----------
path : string or file
Filename to read.
nodetype : Python type, optional (default= int)
Convert nodes to this type.
timestamptype : Python type, optional (default= float)
Convert timestamp to this type.
This must be an orderable type, ideally int or float. Other orderable types have not been fully tested.
order : Python 3-tuple, optional (default= ('u', 'v', 't'))
This must be a 3-tuple containing strings 'u', 'v', and 't'. 'u' specifies the starting node, 'v' the ending node, and 't' the timestamp.
comments : string, optional
Marker for comment lines
delimiter : string, optional
Separator for node labels. The default is whitespace. Cannot be =.
Returns
-------
G: ImpulseGraph
The graph corresponding to the lines in edge list.
Examples
--------
>>> G=dnx.ImpulseGraph.load_from_txt("my_dygraph.txt")
The optional nodetype is a function to convert node strings to nodetype.
For example
>>> G=dnx.ImpulseGraph.load_from_txt("my_dygraph.txt", nodetype=int)
will attempt to convert all nodes to integer type.
Since nodes must be hashable, the function nodetype must return hashable
types (e.g. int, float, str, frozenset - or tuples of those, etc.)
"""
G = ImpulseDiGraph()
if delimiter == '=':
raise ValueError("Delimiter cannot be =.")
if len(
order
) != 3 or 'u' not in order or 'v' not in order or 't' not in order:
raise ValueError(
"Order must be a 3-tuple containing strings 'u', 'v', and 't'."
)
with open(path, 'r') as file:
for line in file:
p = line.find(comments)
if p >= 0:
line = line[:p]
if not len(line):
continue
line = re.split(delimiter + '+', line.strip())
u = line[order.index('u')]
v = line[order.index('v')]
t = line[order.index('t')]
edgedata = {}
for data in line[3:]:
key, value = data.split('=')
try:
value = float(value)
except:
pass
edgedata[key] = value
if nodetype is not int:
try:
u = nodetype(u)
v = nodetype(v)
except:
raise TypeError(
"Failed to convert node to {0}".format(nodetype))
else:
try:
u = int(u)
v = int(v)
except:
pass
try:
t = timestamptype(t)
except:
raise TypeError(
"Failed to convert interval time to {}".format(
timestamptype))
G.add_edge(u, v, t, **edgedata)
return G
