def _parse_flags(self, cmd_flag_map, args):
"""Expects arguments after the initial command and subcommands (i.e.,
the second item returned from _parse_subcmds)
Returns a tuple of (multidict of flag names to parsed and validated values, remaining_args).
Raises on unknown subcommands.
"""
flag_value_map = OMD()
ff_path_res_map = OrderedDict()
ff_path_seen = set()
orig_args = args
while args:
arg = args[0]
if not arg or arg[0] != '-' or arg == '-' or arg == '--':
# posargs or post_posargs beginning ('-' is a conventional pos arg for stdin)
break
flag, value, args = self._parse_single_flag(cmd_flag_map, args)
flag_value_map.add(flag.name, value)
if flag is self.flagfile_flag:
self._parse_flagfile(cmd_flag_map,
value,
res_map=ff_path_res_map)
for path, ff_flag_value_map in ff_path_res_map.items():
if path in ff_path_seen:
continue
flag_value_map.update_extend(ff_flag_value_map)
ff_path_seen.add(path)
return flag_value_map, ff_path_res_map, args
def find_instruction_order(instructions: OrderedMultiDict):
starting_instruction = set(instructions.keys(multi=True)).difference(
set(instructions.values(multi=True)))
worker_manager = WorkerManager(offset=0)
instruction_order = IndexedSet()
current_instructions = list()
for index, value in enumerate(sorted(starting_instruction)):
current_instructions.append(value)
worker_manager.add(value)
current_instructions = sorted(current_instructions)
while current_instructions:
index = -1
for i, current_instruction in enumerate(current_instructions):
if not {
key
for key, value in instructions.items(multi=True)
if value == current_instruction
}.difference(instruction_order):
instruction_order.add(current_instruction)
worker_manager.remove(current_instruction)
for n, value in enumerate(
instructions.getlist(current_instruction)):
if value not in current_instructions:
current_instructions.append(value)
index = i
break
if index != -1:
current_instructions.pop(index)
current_instructions = sorted(current_instructions)
for instruction in current_instructions:
worker_manager.add(instruction)
print("".join(instruction_order))
print(worker_manager.start_time)
class EventEmitter2(object):
def __init__(self):
"""
"""
self.events = OrderedMultiDict()
self.on = self.add_listener
self.off = self.remove_listener
# TODO: Add listener to remove
def add_listener(self, event, listener, count=0):
if not (isinstance(listener, FunctionType)) or (isinstance(listener, BuiltinFunctionType)):
raise Exception("Invalid Listener: %s" % (str(listener)))
_event = re.compile(event)
_listener = {"handler": listener, "calls": 0, "calls_left": count}
self.events.add(_event, _listener)
return True
def emit(self, event, kwargs):
for pattern, listener in self.events.iteritems(multi=True):
if pattern.match(event):
if not listener["calls_left"]:
log.debug("Removing Listener: %s on Pattern: %s") % (
listener, pattern)
self.remove_listener(pattern, listener)
listener["calls"] += 1
listener["calls_left"] -= 1
yield listener["handler"](**kwargs)
def remove_listener(self, pattern, listener):
pattern = re.compile(pattern)
listeners = self.events.getlist(pattern)
for pattern, _listener in self.events.iteritems(multi=True):
if _listener['handler'] == listener:
listener = _listener
break
listeners = self.events.getlist(pattern)
listeners.remove(listener)
if len(listeners):
self.events.update({pattern: listeners})
else:
self.events._remove(pattern)
return True
def on_any(self, listener):
raise NotImplementedError
def off_any(self, listener):
raise NotImplementedError
def once(self, event, listener):
self.add_listener(event, listener, left=1)
def many(self, event, listener, left):
self.add_listener(event, listener, left)
def validate(self):
self._call_custom_hook('pre_validate')
dup_id_map = {}
eid_map = OMD([(e.entry_root, e) for e in self.entries])
for eid in eid_map:
elist = eid_map.getlist(eid)
if len(elist) > 1:
dup_id_map[eid] = elist
if dup_id_map:
raise ValueError('duplicate entry IDs detected: %r' % dup_id_map)
self._call_custom_hook('post_validate')
class EventEmitter2(object):
def __init__(self):
"""
"""
self.events = OrderedMultiDict()
self.on = self.add_listener
self.off = self.remove_listener
# TODO: Add listener to remove
def add_listener(self, event, listener, count=0):
if (isinstance(listener, BuiltinFunctionType)):
raise Exception("Invalid Listener: %s" % (str(listener)))
_event = re.compile(event)
_listener = {"handler": listener, "calls": 0, "calls_left": count}
self.events.add(_event, _listener)
return True
def emit(self, event, kwargs):
for pattern, listener in self.events.iteritems(multi=True):
if pattern.match(event):
if not listener["calls_left"]:
log.debug("Removing Listener: %s on Pattern: %s") % (
listener, pattern)
self.remove_listener(pattern, listener)
listener["calls"] += 1
listener["calls_left"] -= 1
yield listener["handler"](**kwargs)
def remove_listener(self, pattern, listener):
pattern = re.compile(pattern)
listeners = self.events.getlist(pattern)
for pattern, _listener in self.events.iteritems(multi=True):
if _listener['handler'] == listener:
listener = _listener
break
listeners = self.events.getlist(pattern)
listeners.remove(listener)
if len(listeners):
self.events.update({pattern: listeners})
else:
self.events._remove(pattern)
return True
def on_any(self, listener):
raise NotImplementedError
def off_any(self, listener):
raise NotImplementedError
def once(self, event, listener):
self.add_listener(event, listener, left=1)
def many(self, event, listener, left):
self.add_listener(event, listener, left)
def validate(self):
self._call_custom_hook('pre_validate')
dup_id_map = {}
eid_map = OMD([(e.entry_root, e) for e in self.entries])
for eid in eid_map:
elist = eid_map.getlist(eid)
if len(elist) > 1:
dup_id_map[eid] = elist
if dup_id_map:
raise ValueError('duplicate entry IDs detected: %r' % dup_id_map)
self._call_custom_hook('post_validate')
def __init__(self, event_list=None, policy=None, marker=None):
if event_list is None:
event_list = []
if policy is not None:
for event in event_list:
event._policy = policy
self._marker = marker
self._by_uuid = OrderedDict(
zip(map(lambda e: e.uuid, event_list), event_list))
self._by_message_id = OrderedMultiDict(
zip(map(lambda e: e.message.id, self._by_uuid.values()), self._by_uuid.values()))
def read_instructions() -> OrderedMultiDict[str, str]:
instructions = OrderedMultiDict()
instruction_regex = re.compile(r"\s(\w)\s")
with open(INPUT_FILE, "r") as f_handle:
for line in f_handle:
if line:
key, value = instruction_regex.findall(line.rstrip())
if key not in instructions:
instructions[key] = value
else:
instructions.add(key, value)
return instructions
def gen_connected_components(self):
"""Generate every connected oomponents"""
# Initialize every ETH node with a distinct number
connected_components = OrderedMultiDict(
(nodeID, i) for i, (nodeID, n) in enumerate(node.items())
if n['protocol'] == 'ETH')
untraversed = self.switches.copy()
while untraversed:
# choos a switch as the root of the connected component
root = untraversed.pop()
# and set its number as the id
ccID = connected_components[root]
# switches of the same connected_components
cc_switch = {root}
while cc_switch:
# current node
cur = cc_switch.pop()
for l in self.adj[cur]:
_, neighbor, _, _, _, _, _, _, _ = l
connected_components[neighbor] = ccID
if neighbor in untraversed:
cc_switch.add(neighbor)
untraversed.remove(neighbor)
self.connected_components = connected_components
def cs_scrape_properties(csid, props=None):
'''Retrieve some of the experimental and predicted chemical properties that
are not surfaced in the ChemSpider web API.'''
try:
r = requests.get('http://www.chemspider.com/Chemical-Structure.{0}.html'.format(csid))
r.raise_for_status()
except Exception as e:
print(e)
return None
data = OMD([('CSID', csid)])
doc = BeautifulSoup(r.text, 'lxml')
# Experimental and predicted properties ("Experimental data" tab):
props_tabs = doc.find(id='suppInfoTab')
try:
props_found = props_tabs.find_all(class_='user_data_property_name')
for p in props_found:
prop_name = p.get_text().strip(': \r\n')
if props is not None and prop_name not in props:
continue
li = p.find_parent('li')
values = li.find_all('td')
for i in values:
value = i.get_text().strip()
data.add(prop_name, value)
except AttributeError:
pass
# ACD/Labs predicted properties:
acd_tab = doc.find(id='acdLabsTab')
try:
acd_props = acd_tab.find_all(class_='prop_title')
for p in acd_props:
prop_name = p.get_text().strip(': \r\n')
v = p.find_next_sibling('td')
value = v.get_text().strip()
if props is not None and prop_name not in props:
continue
# Make sure all predicted properties are obviously named
if prop_name.startswith('ACD/') is False:
prop_name = 'ACD/' + prop_name
data.add(prop_name, value)
except AttributeError:
pass
# EPI Suite results, as a blob to process later:
# Sometimes only returns part of the text, for some reason.
if props is not None and 'EPI Suite' not in props:
pass
else:
epi_tab = doc.find(id='epiTab')
epi_blob = epi_tab.get_text().strip() if epi_tab else None
data.add('EPI Suite', epi_blob)
return data
def analyze_samples():
outcome = OrderedMultiDict()
opcode = Opcode()
methods = [
func
for func in getmembers(Opcode, predicate=isfunction)
if func[0] != "__init__"
]
count = 0
for before, instruction, after in read_inputs():
outcome.clear()
for opcode_method in methods:
if getattr(opcode, opcode_method[0])(before, instruction) == after:
outcome.add(instruction[0], opcode_method[0])
if len(outcome.getlist(instruction[0])) >= 3:
count += 1
#print(outcome)
print(count)
def epi_suite_values(epi_blob):
'''Extracts some information as key-value pairs from EPI Suite output.
Pretty rough (work in progress).'''
data = OMD()
try:
lines = epi_blob.split('\n')
for i in lines:
j = i.strip()
if j.startswith('Log Kow (') or j.startswith('Log BCF'):
data.update(dict_from_line(i, '='))
if j.startswith('Henrys LC') or\
j.startswith('Log Koa (KOAWIN') or\
j.startswith('Log Koa (experimental') or\
j.startswith('Ready Biodegradability Prediction'):
data.update(dict_from_line(i))
if 'Fugacity' in epi_blob:
model_table = epi_blob.split('Level III Fugacity Model:', 1)[1]
data.add('Level III Fugacity Model', model_table)
except:
pass
return data
def gen_connected_components(self):
"""
Detect all the cut edges / cut endpoints (Tarjan Algorithm).
Split the graph into different connected components.
"""
# ----- temporary variables
# temp: endpoints haven't been traversed
self._tmp_untraversed = self.switches.copy()
# temp: order of a node in the DFS
self._tmp_order = defaultdict(lambda: INFTY)
# temp: the lowest depth a node is connected to
self._tmp_low = defaultdict(lambda: INFTY)
# temp: parent of a node in the DFS
self._tmp_parent = defaultdict(lambda: -1)
# Initialize every ETH node with a distinct number
node_connected_components = OrderedMultiDict(
(nodeID, i) for i, (nodeID, n) in enumerate(node.items())
if n['protocol'] == 'ETH')
link_connected_components = OrderedMultiDict()
# Initialize every ETH non-switch as type 2
entity_type = OrderedMultiDict(
(nodeID, 't2') for nodeID, n in node.items()
if n['type'] != 'SWITCH' and n['protocol'] == 'ETH')
while self._tmp_untraversed:
# temp: all links of a connected components
self._tmp_links = set()
# temp: all endpoints of a connected components
self._tmp_nodes = set()
# ----- special links/endpoints
############################# Explanation ##########################
"""
We categorize all the links and endpoints by 2 dimensions , so there
are 2 * 2 = 4 categories in total. Suppose a link/endpoint is down,
then there would be 4 possible results:
New Routes are needed
No | Yes
|
No --- | Type I
Some traffic will not |
be reachable ----------------+----------------
|
Yes Type II | Type III
|
I'll use an example to explain these categories, and why we care
about these scenarios:
"""
###################################################################
# #
# A #
# | #
# | #
# | #
# 1 ----- 2 #
# \ / #
# \ / #
# \ / #
# 3 #
# | #
# | #
# 4 #
# / \ #
# / \ #
# / \ #
# C ----- 5 6 ------┐ #
# | | | #
# | | B #
# A,B,C - DEVICES | | | #
# 1 ~ 8 - SWITCHES 7 ----- 8 ------┘ #
# #
###################################################################
"""
Type I: When the link/endpoint is down, new routes are needed, all
traffic are still reachable
- Endpoints: 2, 6, 7, 8
- Links: 1-2, 1-3, 2-3, 4-5, 5-7, 7-8, 8-6, 6-4, 8-B, 6-B
When one of these links/endpoints is down, the network will function
as good, but traffic should take failover routes. Net Configurator
will generate alternative routes for all of these scenarios.
In the algorithm, such links/endpoints are not cut edges / cut
vertices.
------
Type II: When the link/endpoint is down, every traffic will still
take the primary route, while some of them will not be reachable.
- Endpoints: A, B, C
- Links: 3-4, 1-A, 5-C
When one of these links/endpoints is down, some of the traffic will
be dropped somewhere, but if a traffic is still reachable, then they
will still take the primary route, so we will not generate new routes
for these scenarios to reduce the ARL table size.
In the algorithm, such links are cut edges; such endpoints are cut
vertices, but not all cut vertices are type II endpoints. To be more
precise, if an endpoint E connects to exactly one another endpoint,
or all the other endpoints that E connects to are not connected
w.r.t. any other endpoints except for E itself, then E is a type II
endpoint.
Note that in this example, if endpoint 5 is down, traffic between A
and B will not need a failover route, but 5 is not categorized as a
type II endpoint, since 4 and 7 are connected w.r.t. some other
links. Actually, we are not able to categorize 5 as type II to save
memory, since connected components detection and link/node
categorization step is done before routes are calculated. Due to
that reason, 1 and 5 are not type II endpoints.
I'm not sure if type II should exist. Although we can save memory
in such scenarios, unreachable traffic will still take up the
bandwidth and resources of links/switches until they are dropped at
the broken link/switch unless applications on top of IP Layer stops
to send these traffic. Maybe we will decide to use more memory, but
save runtime resources. There is a tradeoff.
------
Type III: When the endpoint is down, some traffic will take failover
routes while some may not and some traffic will be unreachable.
- Endpoints: 1, 3, 4, 5
When one of these endpoints is down, some traffic will be dropped
and even some traffic are still reachable, they may need a failover
route. In this scenario, we need to generate failover routes.
In the algorithm, if an endpoint E connects to some endpoints that
connect to each other w.r.t. links other than E, then E is a
type III endpoint.
Note that no links are categorized as type III, since if a link is
down, either all the devices are still connected in the same
connected component, or some are disconnected but we could use the
primary route and drop unreachable traffics somewhere.
-----
It will never happen that when a link/endpoint is down, no traffics
are dropped and no failover route is needed. In this toy example,
switch 2 is such an endpoint, but we still categorize it as type I
because connected components detection and link/node categorization
step is done before routes are calculated.
When we are calculating the shortest route tree, all the switches and
links that not appears in the tree will be recorded. They will be a
special part of type I that no failover routes will be generated for
these links/switches. Endpoint 2, 7, 8 and links 1-2, 2-3, 5-7, 6-8,
7-8, 8-B are such links/switches in this example.
"""
# temp: non-cut links
self._tmp_type1_links = set()
# temp: cut links
self._tmp_type2_links = set()
# temp: non-cut endpoints
self._tmp_type1_nodes = set()
# temp: type 2 cut endpoints
self._tmp_type2_nodes = set()
# temp: type 3 cut endpoints
self._tmp_type3_nodes = set()
# order counter
self.__order = 0
# randomly choose an endpoint as DFS root
DFS_ROOT = self._tmp_untraversed.pop()
self.bridge_update(DFS_ROOT)
self._tmp_type1_links = self._tmp_links - self._tmp_type2_links
ccID = node_connected_components[DFS_ROOT]
node_connected_components.update({
nodeID: ccID for nodeID in self._tmp_nodes
})
link_connected_components.update({
linkID: ccID for linkID in self._tmp_links
})
entity_type.update({
nodeID: 't1' for nodeID in self._tmp_type1_nodes
})
entity_type.update({
nodeID: 't2' for nodeID in self._tmp_type2_nodes
})
entity_type.update({
nodeID: 't3' for nodeID in self._tmp_type3_nodes
})
entity_type.update({
linkID: 't1' for linkID in self._tmp_type1_links
})
entity_type.update({
linkID: 't2' for linkID in self._tmp_type2_links
})
self._tmp_untraversed -= self._tmp_nodes
self.node_connected_components = node_connected_components
self.link_connected_components = link_connected_components
self.entity_type = entity_type
class EventContext(object):
def __init__(self, event_list=None, policy=None, marker=None):
if event_list is None:
event_list = []
if policy is not None:
for event in event_list:
event._policy = policy
self._marker = marker
self._by_uuid = OrderedDict(
zip(map(lambda e: e.uuid, event_list), event_list))
self._by_id = OrderedMultiDict(
zip(map(lambda e: e.id, self._by_uuid.values()),
self._by_uuid.values()))
def events(self):
return list(self._by_uuid.values())
def __iter__(self):
return iter(self._by_uuid.values())
def _set_marker(self, marker):
self._marker = marker
return self
def filter(self, payload):
if hasattr(payload, "id") and payload.id in self._by_id:
events = self._by_id.getlist(payload.id)[:]
return EventContext(events, marker=self._marker)
else:
return EventContext()
def last(self, payload=None):
if payload is None:
return next(reversed(self._by_uuid.values()), None)
elif hasattr(payload, "id"):
return self._by_id.get(payload.id)
else:
raise RuntimeError(
"EventContext.last() payload argument doesn't have an 'id' attribute"
)
def __bool__(self):
return len(self._by_uuid) > 0
__nonzero__ = __bool__
def _marker_prefix_str(self):
return self._marker.begin() if self._marker is not None else ""
def _marker_suffix_str(self):
return self._marker.end() if self._marker is not None else ""
def _to_str(self):
ret = ""
if len(self._by_uuid.values()) > 1:
ret += "["
for i, event in enumerate(self._by_uuid.values()):
ret += self._marker_prefix_str()
ret += str(event)
ret += self._marker_suffix_str()
if i != (len(self._by_uuid.values()) - 1):
ret += ","
if len(self._by_uuid.values()) > 1:
ret += "]"
return ret
def __str__(self):
return EventMarker.color_string(_format_olympe_dsl(self._to_str()))
class ArsdkEventContext(object):
def __init__(self, event_list=None, policy=None, marker=None):
if event_list is None:
event_list = []
if policy is not None:
for event in event_list:
event._policy = policy
self._marker = marker
self._by_uuid = OrderedDict(
zip(map(lambda e: e.uuid, event_list), event_list))
self._by_message_id = OrderedMultiDict(
zip(map(lambda e: e.message.id, self._by_uuid.values()), self._by_uuid.values()))
def events(self):
return list(self._by_uuid.values())
def __iter__(self):
return iter(self._by_uuid.values())
def _set_marker(self, marker):
self._marker = marker
return self
def filter(self, message):
if message.id in self._by_message_id:
events = self._by_message_id.getlist(message.id)[:]
return ArsdkEventContext(events, marker=self._marker)
else:
return ArsdkEventContext()
def last(self, message=None):
if message is None:
return next(reversed(self._by_uuid.values()), None)
else:
return self._by_message_id.get(message.id)
def __bool__(self):
return len(self._by_uuid) > 0
__nonzero__ = __bool__
def _marker_prefix_str(self):
return self._marker.begin() if self._marker is not None else ""
def _marker_suffix_str(self):
return self._marker.end() if self._marker is not None else ""
def _to_str(self):
ret = ""
if len(self._by_uuid.values()) > 1:
ret += "["
for i, message_event in enumerate(self._by_uuid.values()):
ret += self._marker_prefix_str()
ret += str(message_event)
ret += self._marker_suffix_str()
if i != (len(self._by_uuid.values()) - 1):
ret += ","
if len(self._by_uuid.values()) > 1:
ret += "]"
return ret
def __str__(self):
return EventMarker.color_string(_format_olympe_dsl(self._to_str()))
def __init__(self):
"""
"""
self.events = OrderedMultiDict()
self.on = self.add_listener
self.off = self.remove_listener
def __init__(self):
"""
"""
self.events = OrderedMultiDict()
self.on = self.add_listener
self.off = self.remove_listener