def _parse_flags(self, cmd_flag_map, args): """Expects arguments after the initial command and subcommands (i.e., the second item returned from _parse_subcmds) Returns a tuple of (multidict of flag names to parsed and validated values, remaining_args). Raises on unknown subcommands. """ flag_value_map = OMD() ff_path_res_map = OrderedDict() ff_path_seen = set() orig_args = args while args: arg = args[0] if not arg or arg[0] != '-' or arg == '-' or arg == '--': # posargs or post_posargs beginning ('-' is a conventional pos arg for stdin) break flag, value, args = self._parse_single_flag(cmd_flag_map, args) flag_value_map.add(flag.name, value) if flag is self.flagfile_flag: self._parse_flagfile(cmd_flag_map, value, res_map=ff_path_res_map) for path, ff_flag_value_map in ff_path_res_map.items(): if path in ff_path_seen: continue flag_value_map.update_extend(ff_flag_value_map) ff_path_seen.add(path) return flag_value_map, ff_path_res_map, args
def find_instruction_order(instructions: OrderedMultiDict): starting_instruction = set(instructions.keys(multi=True)).difference( set(instructions.values(multi=True))) worker_manager = WorkerManager(offset=0) instruction_order = IndexedSet() current_instructions = list() for index, value in enumerate(sorted(starting_instruction)): current_instructions.append(value) worker_manager.add(value) current_instructions = sorted(current_instructions) while current_instructions: index = -1 for i, current_instruction in enumerate(current_instructions): if not { key for key, value in instructions.items(multi=True) if value == current_instruction }.difference(instruction_order): instruction_order.add(current_instruction) worker_manager.remove(current_instruction) for n, value in enumerate( instructions.getlist(current_instruction)): if value not in current_instructions: current_instructions.append(value) index = i break if index != -1: current_instructions.pop(index) current_instructions = sorted(current_instructions) for instruction in current_instructions: worker_manager.add(instruction) print("".join(instruction_order)) print(worker_manager.start_time)
class EventEmitter2(object): def __init__(self): """ """ self.events = OrderedMultiDict() self.on = self.add_listener self.off = self.remove_listener # TODO: Add listener to remove def add_listener(self, event, listener, count=0): if not (isinstance(listener, FunctionType)) or (isinstance(listener, BuiltinFunctionType)): raise Exception("Invalid Listener: %s" % (str(listener))) _event = re.compile(event) _listener = {"handler": listener, "calls": 0, "calls_left": count} self.events.add(_event, _listener) return True def emit(self, event, kwargs): for pattern, listener in self.events.iteritems(multi=True): if pattern.match(event): if not listener["calls_left"]: log.debug("Removing Listener: %s on Pattern: %s") % ( listener, pattern) self.remove_listener(pattern, listener) listener["calls"] += 1 listener["calls_left"] -= 1 yield listener["handler"](**kwargs) def remove_listener(self, pattern, listener): pattern = re.compile(pattern) listeners = self.events.getlist(pattern) for pattern, _listener in self.events.iteritems(multi=True): if _listener['handler'] == listener: listener = _listener break listeners = self.events.getlist(pattern) listeners.remove(listener) if len(listeners): self.events.update({pattern: listeners}) else: self.events._remove(pattern) return True def on_any(self, listener): raise NotImplementedError def off_any(self, listener): raise NotImplementedError def once(self, event, listener): self.add_listener(event, listener, left=1) def many(self, event, listener, left): self.add_listener(event, listener, left)
def validate(self): self._call_custom_hook('pre_validate') dup_id_map = {} eid_map = OMD([(e.entry_root, e) for e in self.entries]) for eid in eid_map: elist = eid_map.getlist(eid) if len(elist) > 1: dup_id_map[eid] = elist if dup_id_map: raise ValueError('duplicate entry IDs detected: %r' % dup_id_map) self._call_custom_hook('post_validate')
class EventEmitter2(object): def __init__(self): """ """ self.events = OrderedMultiDict() self.on = self.add_listener self.off = self.remove_listener # TODO: Add listener to remove def add_listener(self, event, listener, count=0): if (isinstance(listener, BuiltinFunctionType)): raise Exception("Invalid Listener: %s" % (str(listener))) _event = re.compile(event) _listener = {"handler": listener, "calls": 0, "calls_left": count} self.events.add(_event, _listener) return True def emit(self, event, kwargs): for pattern, listener in self.events.iteritems(multi=True): if pattern.match(event): if not listener["calls_left"]: log.debug("Removing Listener: %s on Pattern: %s") % ( listener, pattern) self.remove_listener(pattern, listener) listener["calls"] += 1 listener["calls_left"] -= 1 yield listener["handler"](**kwargs) def remove_listener(self, pattern, listener): pattern = re.compile(pattern) listeners = self.events.getlist(pattern) for pattern, _listener in self.events.iteritems(multi=True): if _listener['handler'] == listener: listener = _listener break listeners = self.events.getlist(pattern) listeners.remove(listener) if len(listeners): self.events.update({pattern: listeners}) else: self.events._remove(pattern) return True def on_any(self, listener): raise NotImplementedError def off_any(self, listener): raise NotImplementedError def once(self, event, listener): self.add_listener(event, listener, left=1) def many(self, event, listener, left): self.add_listener(event, listener, left)
def __init__(self, event_list=None, policy=None, marker=None): if event_list is None: event_list = [] if policy is not None: for event in event_list: event._policy = policy self._marker = marker self._by_uuid = OrderedDict( zip(map(lambda e: e.uuid, event_list), event_list)) self._by_message_id = OrderedMultiDict( zip(map(lambda e: e.message.id, self._by_uuid.values()), self._by_uuid.values()))
def read_instructions() -> OrderedMultiDict[str, str]: instructions = OrderedMultiDict() instruction_regex = re.compile(r"\s(\w)\s") with open(INPUT_FILE, "r") as f_handle: for line in f_handle: if line: key, value = instruction_regex.findall(line.rstrip()) if key not in instructions: instructions[key] = value else: instructions.add(key, value) return instructions
def gen_connected_components(self): """Generate every connected oomponents""" # Initialize every ETH node with a distinct number connected_components = OrderedMultiDict( (nodeID, i) for i, (nodeID, n) in enumerate(node.items()) if n['protocol'] == 'ETH') untraversed = self.switches.copy() while untraversed: # choos a switch as the root of the connected component root = untraversed.pop() # and set its number as the id ccID = connected_components[root] # switches of the same connected_components cc_switch = {root} while cc_switch: # current node cur = cc_switch.pop() for l in self.adj[cur]: _, neighbor, _, _, _, _, _, _, _ = l connected_components[neighbor] = ccID if neighbor in untraversed: cc_switch.add(neighbor) untraversed.remove(neighbor) self.connected_components = connected_components
def cs_scrape_properties(csid, props=None): '''Retrieve some of the experimental and predicted chemical properties that are not surfaced in the ChemSpider web API.''' try: r = requests.get('http://www.chemspider.com/Chemical-Structure.{0}.html'.format(csid)) r.raise_for_status() except Exception as e: print(e) return None data = OMD([('CSID', csid)]) doc = BeautifulSoup(r.text, 'lxml') # Experimental and predicted properties ("Experimental data" tab): props_tabs = doc.find(id='suppInfoTab') try: props_found = props_tabs.find_all(class_='user_data_property_name') for p in props_found: prop_name = p.get_text().strip(': \r\n') if props is not None and prop_name not in props: continue li = p.find_parent('li') values = li.find_all('td') for i in values: value = i.get_text().strip() data.add(prop_name, value) except AttributeError: pass # ACD/Labs predicted properties: acd_tab = doc.find(id='acdLabsTab') try: acd_props = acd_tab.find_all(class_='prop_title') for p in acd_props: prop_name = p.get_text().strip(': \r\n') v = p.find_next_sibling('td') value = v.get_text().strip() if props is not None and prop_name not in props: continue # Make sure all predicted properties are obviously named if prop_name.startswith('ACD/') is False: prop_name = 'ACD/' + prop_name data.add(prop_name, value) except AttributeError: pass # EPI Suite results, as a blob to process later: # Sometimes only returns part of the text, for some reason. if props is not None and 'EPI Suite' not in props: pass else: epi_tab = doc.find(id='epiTab') epi_blob = epi_tab.get_text().strip() if epi_tab else None data.add('EPI Suite', epi_blob) return data
def analyze_samples(): outcome = OrderedMultiDict() opcode = Opcode() methods = [ func for func in getmembers(Opcode, predicate=isfunction) if func[0] != "__init__" ] count = 0 for before, instruction, after in read_inputs(): outcome.clear() for opcode_method in methods: if getattr(opcode, opcode_method[0])(before, instruction) == after: outcome.add(instruction[0], opcode_method[0]) if len(outcome.getlist(instruction[0])) >= 3: count += 1 #print(outcome) print(count)
def epi_suite_values(epi_blob): '''Extracts some information as key-value pairs from EPI Suite output. Pretty rough (work in progress).''' data = OMD() try: lines = epi_blob.split('\n') for i in lines: j = i.strip() if j.startswith('Log Kow (') or j.startswith('Log BCF'): data.update(dict_from_line(i, '=')) if j.startswith('Henrys LC') or\ j.startswith('Log Koa (KOAWIN') or\ j.startswith('Log Koa (experimental') or\ j.startswith('Ready Biodegradability Prediction'): data.update(dict_from_line(i)) if 'Fugacity' in epi_blob: model_table = epi_blob.split('Level III Fugacity Model:', 1)[1] data.add('Level III Fugacity Model', model_table) except: pass return data
def gen_connected_components(self): """ Detect all the cut edges / cut endpoints (Tarjan Algorithm). Split the graph into different connected components. """ # ----- temporary variables # temp: endpoints haven't been traversed self._tmp_untraversed = self.switches.copy() # temp: order of a node in the DFS self._tmp_order = defaultdict(lambda: INFTY) # temp: the lowest depth a node is connected to self._tmp_low = defaultdict(lambda: INFTY) # temp: parent of a node in the DFS self._tmp_parent = defaultdict(lambda: -1) # Initialize every ETH node with a distinct number node_connected_components = OrderedMultiDict( (nodeID, i) for i, (nodeID, n) in enumerate(node.items()) if n['protocol'] == 'ETH') link_connected_components = OrderedMultiDict() # Initialize every ETH non-switch as type 2 entity_type = OrderedMultiDict( (nodeID, 't2') for nodeID, n in node.items() if n['type'] != 'SWITCH' and n['protocol'] == 'ETH') while self._tmp_untraversed: # temp: all links of a connected components self._tmp_links = set() # temp: all endpoints of a connected components self._tmp_nodes = set() # ----- special links/endpoints ############################# Explanation ########################## """ We categorize all the links and endpoints by 2 dimensions , so there are 2 * 2 = 4 categories in total. Suppose a link/endpoint is down, then there would be 4 possible results: New Routes are needed No | Yes | No --- | Type I Some traffic will not | be reachable ----------------+---------------- | Yes Type II | Type III | I'll use an example to explain these categories, and why we care about these scenarios: """ ################################################################### # # # A # # | # # | # # | # # 1 ----- 2 # # \ / # # \ / # # \ / # # 3 # # | # # | # # 4 # # / \ # # / \ # # / \ # # C ----- 5 6 ------┐ # # | | | # # | | B # # A,B,C - DEVICES | | | # # 1 ~ 8 - SWITCHES 7 ----- 8 ------┘ # # # ################################################################### """ Type I: When the link/endpoint is down, new routes are needed, all traffic are still reachable - Endpoints: 2, 6, 7, 8 - Links: 1-2, 1-3, 2-3, 4-5, 5-7, 7-8, 8-6, 6-4, 8-B, 6-B When one of these links/endpoints is down, the network will function as good, but traffic should take failover routes. Net Configurator will generate alternative routes for all of these scenarios. In the algorithm, such links/endpoints are not cut edges / cut vertices. ------ Type II: When the link/endpoint is down, every traffic will still take the primary route, while some of them will not be reachable. - Endpoints: A, B, C - Links: 3-4, 1-A, 5-C When one of these links/endpoints is down, some of the traffic will be dropped somewhere, but if a traffic is still reachable, then they will still take the primary route, so we will not generate new routes for these scenarios to reduce the ARL table size. In the algorithm, such links are cut edges; such endpoints are cut vertices, but not all cut vertices are type II endpoints. To be more precise, if an endpoint E connects to exactly one another endpoint, or all the other endpoints that E connects to are not connected w.r.t. any other endpoints except for E itself, then E is a type II endpoint. Note that in this example, if endpoint 5 is down, traffic between A and B will not need a failover route, but 5 is not categorized as a type II endpoint, since 4 and 7 are connected w.r.t. some other links. Actually, we are not able to categorize 5 as type II to save memory, since connected components detection and link/node categorization step is done before routes are calculated. Due to that reason, 1 and 5 are not type II endpoints. I'm not sure if type II should exist. Although we can save memory in such scenarios, unreachable traffic will still take up the bandwidth and resources of links/switches until they are dropped at the broken link/switch unless applications on top of IP Layer stops to send these traffic. Maybe we will decide to use more memory, but save runtime resources. There is a tradeoff. ------ Type III: When the endpoint is down, some traffic will take failover routes while some may not and some traffic will be unreachable. - Endpoints: 1, 3, 4, 5 When one of these endpoints is down, some traffic will be dropped and even some traffic are still reachable, they may need a failover route. In this scenario, we need to generate failover routes. In the algorithm, if an endpoint E connects to some endpoints that connect to each other w.r.t. links other than E, then E is a type III endpoint. Note that no links are categorized as type III, since if a link is down, either all the devices are still connected in the same connected component, or some are disconnected but we could use the primary route and drop unreachable traffics somewhere. ----- It will never happen that when a link/endpoint is down, no traffics are dropped and no failover route is needed. In this toy example, switch 2 is such an endpoint, but we still categorize it as type I because connected components detection and link/node categorization step is done before routes are calculated. When we are calculating the shortest route tree, all the switches and links that not appears in the tree will be recorded. They will be a special part of type I that no failover routes will be generated for these links/switches. Endpoint 2, 7, 8 and links 1-2, 2-3, 5-7, 6-8, 7-8, 8-B are such links/switches in this example. """ # temp: non-cut links self._tmp_type1_links = set() # temp: cut links self._tmp_type2_links = set() # temp: non-cut endpoints self._tmp_type1_nodes = set() # temp: type 2 cut endpoints self._tmp_type2_nodes = set() # temp: type 3 cut endpoints self._tmp_type3_nodes = set() # order counter self.__order = 0 # randomly choose an endpoint as DFS root DFS_ROOT = self._tmp_untraversed.pop() self.bridge_update(DFS_ROOT) self._tmp_type1_links = self._tmp_links - self._tmp_type2_links ccID = node_connected_components[DFS_ROOT] node_connected_components.update({ nodeID: ccID for nodeID in self._tmp_nodes }) link_connected_components.update({ linkID: ccID for linkID in self._tmp_links }) entity_type.update({ nodeID: 't1' for nodeID in self._tmp_type1_nodes }) entity_type.update({ nodeID: 't2' for nodeID in self._tmp_type2_nodes }) entity_type.update({ nodeID: 't3' for nodeID in self._tmp_type3_nodes }) entity_type.update({ linkID: 't1' for linkID in self._tmp_type1_links }) entity_type.update({ linkID: 't2' for linkID in self._tmp_type2_links }) self._tmp_untraversed -= self._tmp_nodes self.node_connected_components = node_connected_components self.link_connected_components = link_connected_components self.entity_type = entity_type
class EventContext(object): def __init__(self, event_list=None, policy=None, marker=None): if event_list is None: event_list = [] if policy is not None: for event in event_list: event._policy = policy self._marker = marker self._by_uuid = OrderedDict( zip(map(lambda e: e.uuid, event_list), event_list)) self._by_id = OrderedMultiDict( zip(map(lambda e: e.id, self._by_uuid.values()), self._by_uuid.values())) def events(self): return list(self._by_uuid.values()) def __iter__(self): return iter(self._by_uuid.values()) def _set_marker(self, marker): self._marker = marker return self def filter(self, payload): if hasattr(payload, "id") and payload.id in self._by_id: events = self._by_id.getlist(payload.id)[:] return EventContext(events, marker=self._marker) else: return EventContext() def last(self, payload=None): if payload is None: return next(reversed(self._by_uuid.values()), None) elif hasattr(payload, "id"): return self._by_id.get(payload.id) else: raise RuntimeError( "EventContext.last() payload argument doesn't have an 'id' attribute" ) def __bool__(self): return len(self._by_uuid) > 0 __nonzero__ = __bool__ def _marker_prefix_str(self): return self._marker.begin() if self._marker is not None else "" def _marker_suffix_str(self): return self._marker.end() if self._marker is not None else "" def _to_str(self): ret = "" if len(self._by_uuid.values()) > 1: ret += "[" for i, event in enumerate(self._by_uuid.values()): ret += self._marker_prefix_str() ret += str(event) ret += self._marker_suffix_str() if i != (len(self._by_uuid.values()) - 1): ret += "," if len(self._by_uuid.values()) > 1: ret += "]" return ret def __str__(self): return EventMarker.color_string(_format_olympe_dsl(self._to_str()))
class ArsdkEventContext(object): def __init__(self, event_list=None, policy=None, marker=None): if event_list is None: event_list = [] if policy is not None: for event in event_list: event._policy = policy self._marker = marker self._by_uuid = OrderedDict( zip(map(lambda e: e.uuid, event_list), event_list)) self._by_message_id = OrderedMultiDict( zip(map(lambda e: e.message.id, self._by_uuid.values()), self._by_uuid.values())) def events(self): return list(self._by_uuid.values()) def __iter__(self): return iter(self._by_uuid.values()) def _set_marker(self, marker): self._marker = marker return self def filter(self, message): if message.id in self._by_message_id: events = self._by_message_id.getlist(message.id)[:] return ArsdkEventContext(events, marker=self._marker) else: return ArsdkEventContext() def last(self, message=None): if message is None: return next(reversed(self._by_uuid.values()), None) else: return self._by_message_id.get(message.id) def __bool__(self): return len(self._by_uuid) > 0 __nonzero__ = __bool__ def _marker_prefix_str(self): return self._marker.begin() if self._marker is not None else "" def _marker_suffix_str(self): return self._marker.end() if self._marker is not None else "" def _to_str(self): ret = "" if len(self._by_uuid.values()) > 1: ret += "[" for i, message_event in enumerate(self._by_uuid.values()): ret += self._marker_prefix_str() ret += str(message_event) ret += self._marker_suffix_str() if i != (len(self._by_uuid.values()) - 1): ret += "," if len(self._by_uuid.values()) > 1: ret += "]" return ret def __str__(self): return EventMarker.color_string(_format_olympe_dsl(self._to_str()))
def __init__(self): """ """ self.events = OrderedMultiDict() self.on = self.add_listener self.off = self.remove_listener